Adds Python package management to depot_tools.

package_management.py

+#!/usr/bin/env python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Package Management
+
+This module is used to bring in external Python dependencies via eggs from the
+PyPi repository. It is intended to work with any version of Python from 2.4
+through 2.6 and beyond, as well as on any OS and hardware.
+
+The approach is to create a site directory in depot_tools root which will
+contain those packages that are listed as dependencies in the module variable
+PACKAGES. Since we can't guarantee that setuptools is available in all
+distributions this module also contains the ability to bootstrap the site
+directory by manually downloading and loading setuptools. Once setuptools is
+available it uses that to install the other packages in the traditional
+manner.
+
+Use is simple:
+
+  import package_management
+
+  # Before any imports from the site directory, call this. This only needs
+  # to be called in one place near the beginning of the program.
+  package_management.SetupSiteDirectory()
+
+  # If 'SetupSiteDirectory' fails it will complain with an error message but
+  # continue happily. Expect ImportErrors when trying to import any third
+  # party modules from the site directory.
+
+  import some_third_party_module
+
+  ... etc ...
+"""
+
+import cStringIO
+import os
+import re
+import shutil
+import site
+import subprocess
+import sys
+import tempfile
+import urllib
+
+
+# This is the version of setuptools that we will download if the local
+# python distribution does not include one.
+SETUPTOOLS = ('setuptools', '0.6c11')
+
+# These are the packages that are to be installed in the site directory.
+# easy_install makes it so that the most recently installed version of a
+# package is the one that takes precedence, even if a newer version exists
+# in the site directory. This allows us to blindly install these one on top
+# of the other without worrying about whats already installed.
+#
+# NOTE: If we are often rolling these dependencies then users' site
+#     directories will grow monotonically. We could be purging any orphaned
+#     packages using the tools provided by pkg_resources.
+PACKAGES = (('logilab-common', '0.57.1'),
+            ('logilab-astng', '0.23.1'),
+            ('pylint', '0.25.1'))
+
+# This is the root directory of the depot_tools installation.
+ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
+
+# This is the path of the site directory we will use. We make this
+# python version specific so that this will continue to work even if the
+# python version is rolled.
+SITE_DIR = os.path.join(ROOT_DIR, 'site-packages-py%s' % sys.version[0:3])

[Sigurður Ásgeirsson, 2011/12/17 15:26:03]

format from sys.version_info, perhaps?

[chrisha, 2012/01/06 21:21:28]

Done.

+
+# This status file is created the last time PACKAGES were rolled successfully.
+# It is used to determine if packages need to be rolled by comparing against
+# the age of __file__.
+LAST_ROLLED = os.path.join(SITE_DIR, 'last_rolled.txt')
+
+
+class Error(Exception):
+  """The base class for all module errors."""
+  pass
+
+
+class InstallError(Error):
+  """Thrown if an installation is unable to complete."""
+  pass
+
+
+class Package(object):
+  """A package represents a release of a project.
+
+  We use this as a lightweight version of pkg_resources, allowing us to
+  perform an endrun around setuptools for the purpose of bootstrapping. Its

[Sigurður Ásgeirsson, 2011/12/17 15:26:03]

nit: endrun->end-run?

[chrisha, 2012/01/06 21:21:28]

Done.

+  functionality is very limited.
+
+  Attributes:
+    name: the name of the package.
+    version: the version of the package.
+    safe_name: the safe name of the package.
+    safe_version: the safe version string of the package.
+    file_name: the filename-safe name of the package.
+    file_version: the filename-safe version string of the package.
+  """
+
+  def __init__(self, name, version):
+    """Initialize this package.
+
+    Args:
+      name: the name of the package.
+      version: the version of the package.
+    """
+    self.name = name
+    self.version = version
+    self.safe_name = Package._MakeSafeName(self.name)
+    self.safe_version = Package._MakeSafeVersion(self.version)
+    self.file_name = Package._MakeSafeForFilename(self.safe_name)
+    self.file_version = Package._MakeSafeForFilename(self.safe_version)
+
+  @staticmethod
+  def _MakeSafeName(name):
+    """Makes a safe package name.
+
+    Returns:
+      The package name cleaned as per pkg_resources.
+    """
+    return re.sub('[^A-Za-z0-9]+', '-', name)
+
+  @staticmethod
+  def _MakeSafeVersion(version):
+    """Makes a safe package version string.
+
+    Returns:
+      The version string cleaned as per pkg_resources.
+    """
+    version = re.sub('\s+', '.', version)
+    return re.sub('[^A-Za-z0-9\.]+', '-', version)
+
+  @staticmethod
+  def _MakeSafeForFilename(safe_name_or_version):
+    """Makes a safe name or safe version safe to use in a file name.

[M-A Ruel, 2011/12/19 21:04:23]

Stylistic note;

I'm not a fan of functions with a longer docstring that then implementation,
especially when the implementation is a simple re.sub() call.

[chrisha, 2012/01/06 21:21:28]

Noted ;)

+
+    Args:
+      safe_name_or_version: a safe name or string as returned by
+          GetSafeName() or GetSafeVersion().
+
+    Returns:
+      The safe name or version escaped as per pkg_resources.
+    """
+    return re.sub('-', '_', safe_name_or_version)
+
+  def GetAsRequirementString(self):
+    """Builds an easy_install requirements string representing this package.
+
+    Returns:
+      A requirement string that can be used with easy_install.
+    """
+    return '%s==%s' % (self.name, self.version)
+
+  def GetFilename(self, extension=None):
+    """Builds a filename for this package using the setuptools convention.
+
+    The following url discusses the filename format:
+
+      http://svn.python.org/projects/sandbox/trunk/setuptools/doc/formats.txt
+
+    Args:
+      extension: If None, returns a basename. Otherwise, uses the provided
+          extension.
+
+    Returns:
+      The filename for this package according to the setuptools convention.
+    """
+    filename = '%s-%s-py%s' % (self.file_name, self.file_version,
+                               sys.version[0:3])
+
+    if extension:
+      if extension[0] != '.':
+        filename += '.'
+      filename += extension
+
+    return filename
+
+  def GetPyPiUrl(self, extension):
+    """Returns the URL where this package lives on PyPI.
+
+    Returns:
+      A string representing the HTTP URL where this package is hosted at
+      pypi.python.org.
+    """
+    return 'http://pypi.python.org/packages/2.6/%c/%s/%s' % (
+        self.file_name[0], self.file_name, self.GetFilename(extension))
+
+  def DownloadEgg(self, dest_dir, overwrite=False):
+    """Downloads the EGG for this URL.
+
+    Args:
+      dest_dir: The directory where the EGG should be written.
+      overwite: If True the destination path will be overwritten even if
+          it already exists. Defaults to False.
+
+    Returns:
+      The path to the written EGG.
+
+    Raises:
+      Error: if dest_dir doesn't exist, the EGG is unable to be written,
+          the URL doesn't exist, or the server returned an error, or the
+          transmission was interrupted.
+    """
+    if not os.path.exists(dest_dir):
+      raise Error('Path does not exist: %s' % dest_dir)
+
+    if not os.path.isdir(dest_dir):
+      raise Error('Path is not a directory: %s' % dest_dir)
+
+    filename = os.path.abspath(os.path.join(dest_dir, self.GetFilename('egg')))
+    if os.path.exists(filename):
+      if os.path.isdir(filename):
+        raise Error('Path is a directory: %s' % filename)
+      if not overwrite:
+        return filename
+
+    url = self.GetPyPiUrl('egg')
+
+    try:
+      (path, headers) = urllib.urlretrieve(url, filename)
+
+      # If the path where the object was downloaded to does not match the

[M-A Ruel, 2011/12/19 21:04:23]

That may happen?

[chrisha, 2012/01/06 21:21:28]

There's some mention of caching; if a particular file has already been
downloaded via urllib, and is redownloaded, it will return the path to the
original download (so say the docs). It is very doubtful (somebody else in this
process would have had to have downloaded the same file previously), but
apparently possible.

Per the docs:

"If the URL points to a local file, or a valid cached copy of the object exists,
the object is not copied."

Come to think of it, there's no need for us to make a copy only to delete it
later on. I'll simply return the path given me by urlretrieve.

+      # location to which we wanted to write it, copy it.
+      if path != filename:
+        shutil.copyfile(path, filename)
+
+    except IOError, e:
+      raise Error, sys.exc_info()[1], sys.exc_info()[2]

[M-A Ruel, 2011/12/19 21:04:23]

Use the new format;
raise Error(sys.exc_info()[1], sys.exc_info()[2])

or whatever want to achieve.

[chrisha, 2012/01/06 21:21:28]

That's not quite equivalent. I want to recast the error type, but keep the same
value and traceback. This is the only way I know how to do that.

+    except urllib.ContentTooShortError, e:
+      raise Error, sys.exc_info()[1], sys.exc_info()[2]
+
+    return filename
+
+
+def AddToPythonPath(path):
+  """Adds the provided path to the head of PYTHONPATH and sys.path.
+
+  Args:
+    path: the path to add.

[M-A Ruel, 2011/12/19 21:04:23]

A path is a path, don't document that.

Silly style reviewers.

[chrisha, 2012/01/06 21:21:28]

Done.

+  """
+  if path not in sys.path:
+    sys.path.insert(0, path)
+
+  paths = os.environ.get('PYTHONPATH', '').split(os.pathsep)
+  if path not in paths:
+    paths.insert(0, path)
+    os.environ['PYTHONPATH'] = os.pathsep.join(paths)
+
+
+def RemoveFromPythonPath(path):
+  """Removes the provided path from PYTHONPATH and sys.path.
+
+  Args:
+    path: the path to remove.
+  """
+  def RemoveFromList(paths):
+    for i in xrange(len(paths), 0, -1):
+      if paths[i - 1] == path:
+        paths.pop(i - 1)
+
+  if path in sys.path:
+    RemoveFromList(sys.path)
+
+  paths = os.environ.get('PYTHONPATH', '').split(os.pathsep)
+  if path in paths:
+    RemoveFromList(paths)
+    os.environ['PYTHONPATH'] = os.pathsep.join(paths)
+
+
+def AddSiteDirectory(path):
+  """Adds the provided path to the runtime as a site directory.
+
+  Any modules that are in the site directory will be available for importing
+  after this returns. If modules are added or deleted this must be called
+  again for the changes to be reflected in the runtime.
+
+  This calls both AddToPythonPath and site.addsitedir. Both are needed to
+  convince easy_install to treat |path| as a site directory.
+
+  Args:
+    path: the path of the site directory to add.
+  """
+  AddToPythonPath(path)
+  site.addsitedir(path)
+
+
+def CreateOrAddSiteDirectory(path):

[Sigurður Ásgeirsson, 2011/12/17 15:26:03]

Maybe name EnsureSiteDirectory?

[chrisha, 2012/01/06 21:21:28]

Done.

+  """Creates and/or adds the provided path to the runtime as a site directory.
+
+  This works like AddSiteDirectory but it will create the directory if it
+  does not yet exist.
+
+  Args:
+    path: the path of the site directory to create and/or add.
+
+  Raise:
+    Error: if the site directory is unable to be created, or if it exist and
+        is not a directory.
+  """
+  if os.path.exists(path):
+    if not os.path.isdir(path):
+      raise Error('Path is not a directory: %s' % path)
+  else:
+    try:
+      os.mkdir(path)
+    except IOError:
+      raise Error('Unable to create directory: %s' % path)
+
+  AddSiteDirectory(path)
+
+
+def ModuleIsFromPackage(module, package_path):
+  """Determines if a module has been imported from a given package.
+
+  Args:
+    module: the module to test.
+    package_path: the path to the package to test.
+
+  Returns:
+    True if |module| has been imported from |package_path|, False otherwise.
+  """
+  m = os.path.abspath(module.__file__)
+  p = os.path.abspath(package_path)
+  if len(m) <= len(p):
+    return False
+  if m[0:len(p)] != p:
+    return False
+  return m[len(p)] == os.sep
+
+
+def _CaptureStdStreams(function, *args, **kwargs):
+  """Captures stdout and stderr while running the provided function.
+
+  This only works if |function| only accesses sys.stdout and sys.stderr. If
+  we need more than this we'll have to use subprocess.Popen.
+
+  Args:
+    function: the function to be called.
+    args: the arguments to pass to |function|.
+    kwargs: the keyword arguments to pass to |function|.
+  """
+  string_stdout = cStringIO.StringIO()
+  string_stderr = cStringIO.StringIO()
+  orig_stdout = sys.stdout
+  orig_stderr = sys.stderr
+  sys.stdout = string_stdout

[M-A Ruel, 2011/12/19 21:04:23]

No need to name the variables, just
sys.stdout = cStringIO.StringIO()

[chrisha, 2012/01/06 21:21:28]

Done.

+  sys.stderr = string_stderr
+  try:
+    function(*args, **kwargs)

[M-A Ruel, 2011/12/19 21:04:23]

return function(*args, **kwargs)

[chrisha, 2012/01/06 21:21:28]

Done.

+  finally:
+    sys.stdout = orig_stdout
+    sys.stderr = orig_stderr
+  return

[M-A Ruel, 2011/12/19 21:04:23]

remove

[chrisha, 2012/01/06 21:21:28]

Done.

+
+
+def InstallPackage(url_or_req, site_dir):
+  """Installs a package to a site directory.
+
+  |site_dir| must exist and already be an active site directory. setuptools
+  must in the path. Uses easy_install which may involve a download from
+  pypi.python.org, so this also requires network access.
+
+  Args:
+    url_or_req: the package to install, expressed as an URL (may be local),
+        or a requirement string.
+    site_dir: the site directory in which to install it.
+
+  Raises:
+    InstallError: if installation fails for any reason.
+  """
+  args = ['--quiet', '--install-dir', site_dir, '--exclude-scripts',
+          '--always-unzip', '--no-deps', url_or_req]
+
+  # The easy_install script only calls SystemExit if something goes wrong.
+  # Otherwise, it falls through returning None.
+  try:
+    import setuptools.command.easy_install
+    _CaptureStdStreams(setuptools.command.easy_install.main, args)
+  except (ImportError, SystemExit), e:
+    # Re-raise the error, preserving the stack trace and message.
+    raise InstallError, sys.exc_info()[1], sys.exc_info()[2]

[M-A Ruel, 2011/12/19 21:04:23]

same everywhere

+
+
+def _RunInSubprocess(pycode):
+  """Launches a python subprocess with the provided code.
+
+  The subprocess will be launched with the same stdout and stderr. The
+  subprocess will use the same instance of python as is currently running,
+  passing |pycode| as arguments to this script. |pycode| will be interpreted
+  as python code in the context of this module.
+
+  Args:
+    pycode: the statement to execute.
+
+  Returns:
+    True if the subprocess returned 0, False if it returned an error.
+  """
+  result = subprocess.call([sys.executable, __file__, pycode])

[M-A Ruel, 2011/12/19 21:04:23]

return not subprocess.call(...

[chrisha, 2012/01/06 21:21:28]

Done.

+  return result == 0
+
+
+def _LoadSetupToolsFromEggAndInstall(egg_path):
+  """Loads setuptools from the provided egg, and installs it to SITE_DIR.
+
+  Args:
+    egg_path: the path to the downloaded egg.
+
+  Returns:
+    True on success, False on failure.
+  """
+  AddToPythonPath(egg_path)
+
+  try:
+    # Import setuptools and ensure it comes from the EGG.
+    import setuptools
+    if not ModuleIsFromPackage(setuptools, egg_path):
+      raise ImportError()
+  except ImportError:
+    print '  Unable to import downloaded package!'
+    return False
+
+  try:
+    print '  Using setuptools to install itself ...'
+    InstallPackage(egg_path, SITE_DIR)
+  except InstallError:
+    print '  Unable to install setuptools!'
+    return False
+
+  return True
+
+
+def BootstrapSetupTools():
+  """Bootstraps the runtime with setuptools.

[M-A Ruel, 2011/12/19 21:04:23]

It'd be nice if it was documented that it's meant to be run from a child
process.

[chrisha, 2012/01/06 21:21:28]

Done.

+
+  Will try to import setuptools directly. If not found it will attempt to
+  download it and load it from there. If the download is successful it will
+  then use setuptools to install itself in the site directory.
+
+  Returns:
+    Returns True if 'import setuptools' will succeed, False otherwise.
+  """
+  AddSiteDirectory(SITE_DIR)
+
+  # Check if setuptools is already available. If so, we're done.
+  try:
+    import setuptools
+    return True
+  except ImportError:
+    pass
+
+  print 'Bootstrapping setuptools ...'
+
+  CreateOrAddSiteDirectory(SITE_DIR)
+
+  # Download the egg to a temp directory.
+  dest_dir = tempfile.mkdtemp('depot_tools')
+  path = None
+  try:
+    package = Package(*SETUPTOOLS)
+    print '  Downloading %s ...' % package.GetFilename()
+    path = package.DownloadEgg(dest_dir)
+  except Error:
+    print '  Download failed!'
+    return False
+
+  try:
+    # Load the downloaded egg, and install it to the site directory. Do this
+    # in a subprocess so as not to pollute this runtime.
+    pycode = '_LoadSetupToolsFromEggAndInstall(%s)' % repr(path)
+    if not _RunInSubprocess(pycode):
+      raise Error
+
+    # Reload our site directory, which should now contain setuptools.
+    AddSiteDirectory(SITE_DIR)
+
+    # Try and import setupttols
+    import setuptools
+  except ImportError:
+    print '  Unable to import setuptools!'
+    return False
+  except Error:
+    # This happens if RunInSubProcess fails, and the appropriate error has
+    # already been written.
+    return False
+  finally:
+    # Delete the temp directory.
+    shutil.rmtree(dest_dir)
+
+  return True
+
+
+def _GetModTime(path):
+  """Gets the file modification time associated with the given file.
+
+  If the file does not exist, returns 0.
+
+  Args:
+    path: the file to stat.
+
+  Returns:
+    The last modification time of |path| in seconds since epoch, or 0 if
+    |path| does not exist.
+  """
+  try:
+    stat = os.stat(path)

[M-A Ruel, 2011/12/19 21:04:23]

return os.stat(path).st_mtime

+    return stat.st_mtime
+  except:
+    # This error is different depending on the OS.
+    return 0
+
+
+def _SiteDirectoryIsUpToDate():
+  return _GetModTime(LAST_ROLLED) > _GetModTime(__file__)
+
+
+def UpdateSiteDirectory():
+  """Installs the packages from PACKAGES if they are not already installed.
+
+  At this point we must have setuptools in the site directory.
+
+  Returns:
+    True on success, False otherwise.
+  """
+  if _SiteDirectoryIsUpToDate():
+    return True
+
+  try:
+    AddSiteDirectory(SITE_DIR)
+    import pkg_resources
+
+    # Determine if any packages actually need installing.
+    missing_packages = []
+    for package in [SETUPTOOLS] + list(PACKAGES):
+      pkg = Package(*package)
+      req = pkg.GetAsRequirementString()
+
+      # It may be that this package is already available in the site
+      # directory. If so, we can skip past it without trying to install it.
+      dist = pkg_resources.working_set.find(
+          pkg_resources.Requirement.parse(req))
+      if dist:
+        continue
+
+      missing_packages.append(pkg)
+
+    # Install the missing packages.
+    if missing_packages:
+      print 'Updating python packages ...'
+      for pkg in missing_packages:
+        print '  Installing %s ...' % pkg.GetFilename()
+        InstallPackage(pkg.GetAsRequirementString(), SITE_DIR)
+
+    # Touch the status file so we know that we're up to date next time.
+    open(LAST_ROLLED, 'wb')
+  except InstallError, e:
+    print '  Installation failed: %s' % str(e)
+    return False
+
+  return True
+
+
+def SetupSiteDirectory():
+  """Sets up the site directory, bootstrapping setuptools if necessary.
+
+  If this finishes successfully then SITE_DIR will exist and will contain
+  the appropriate version of setuptools and all of the packages listed in
+  PACKAGES.
+
+  This is the main workhorse of this module. Calling this will do everything
+  necessary to ensure that you have the desired packages installed in the
+  site directory, and the site directory enabled in this process.
+
+  Returns:
+    True on success, False on failure.
+  """
+  if _SiteDirectoryIsUpToDate():
+    AddSiteDirectory(SITE_DIR)
+    return True
+
+  if not _RunInSubprocess('BootstrapSetupTools()'):
+    return False
+
+  if not _RunInSubprocess('UpdateSiteDirectory()'):
+    return False
+
+  # Process the site directory so that the packages within it are available
+  # for import.
+  AddSiteDirectory(SITE_DIR)
+
+  return True
+
+
+def CanImportFromSiteDirectory(package_name):
+  """Determines if the given package can be imported from the site directory.
+
+  Args:
+    package_name: the name of the package to import.
+
+  Returns:
+    True if 'import package_name' will succeed and return a module from the
+    site directory, False otherwise.
+  """
+  try:
+    exec('import %s' % package_name)

[M-A Ruel, 2011/12/19 21:04:23]

__import__(package_name)

[chrisha, 2012/01/06 21:21:28]

Well, you learn something new everyday!

Done.

+  except ImportError:
+    return False
+
+  result = False
+  exec('result = ModuleIsFromPackage(%s, SITE_DIR)' % package_name)

[M-A Ruel, 2011/12/19 21:04:23]

??
result = ModuleIsFromPackage(package_name, SITE_DIR)

because you want a pointer to package_name as a module? Note that the exec call
above redefines package_name from a string into a module. That's confusing.

[chrisha, 2012/01/06 21:21:28]

That's the intended result. Made more clear by using __import__, and keeping a
pointer to the actual module.

+  return result
+
+
+def Main():
+  """The main entry for the package management script.
+
+  If no arguments are provided simply runs SetupSiteDirectory. If arguments
+  have been passed we execute the first argument as python code in the
+  context of this module. This mechanism is used to during the bootstrap
+  process so that the main instance of Python does not have its runtime
+  polluted by various intermediate packages and imports.
+
+  Returns:
+    0 on success, 1 otherwise.
+  """
+  if len(sys.argv) == 2:
+    result = False

[M-A Ruel, 2011/12/19 21:04:23]

result = 0

+    exec('result = %s' % sys.argv[1])
+    return 0 if result else 1

[M-A Ruel, 2011/12/19 21:04:23]

return result and have the function return relevant return code?

Note that I don't really mind either way so you can ignore that.

[chrisha, 2012/01/06 21:21:28]

For consistency, I made all functions return True/False success status, and have
the 'translation' happen in Main(). I'll keep it this way with a little comment
added here.

+  else:
+    return 0 if SetupSiteDirectory() else 1
+
+
+if __name__ == '__main__':
+  sys.exit(Main())