Cross-repo recipe package system.

third_party/recipe_engine/package.py

+import ast
+import collections
+import contextlib
+import copy
+import itertools
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+
+sys.path.append(
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'third_party'))
+
+import dateutil.parser
+
+class UncleanFilesystemError(Exception):
+  pass
+
+
+class InconsistentDependencyGraphError(Exception):
+  pass
+
+
+class PackageContext(object):
+  """Contains information about where the root package and its dependency
+  checkouts live."""
+
+  def __init__(self, root_dir, package_dir):
+    self.root_dir = root_dir
+    self.package_dir = package_dir
+
+  @classmethod
+  def from_pyl_path(cls, pyl_path):
+    root_dir = os.path.dirname(pyl_path)
+    return cls(root_dir, os.path.join(root_dir, '.recipe_deps'))
+
+
+class RepoSpec(object):
+  def checkout(self, context):
+    """Fetches the specified package and returns the path of the package root
+    (the directory that contains recipe_package.pyl).
+    """
+    raise NotImplementedError()
+
+
+class GitRepoSpec(RepoSpec):
+  def __init__(self, id, repo, branch, revision, path):
+    self.id = id
+    self.repo = repo
+    self.branch = branch
+    self.revision = revision
+    self.path = path
+
+  def checkout(self, context):
+    package_dir = context.package_dir
+    dep_dir = os.path.join(package_dir, self.id)
+    logging.info('Freshening repository %s' % dep_dir)
+
+    if os.path.exists(os.path.join(package_dir, '.dont_mess_with_this')):
+      logging.warn('Skipping checkout of %s because of .dont_mess_with_this'
+                   % self.id)
+      return os.path.join(dep_dir, self.path)
+
+    if not os.path.isdir(dep_dir):
+      _run_cmd(['git', 'clone', self.repo, dep_dir])
+    elif not os.path.isdir(os.path.join(dep_dir, '.git')):
+      raise UncleanFilesystemError('%s exists but is not a git repo' % dep_dir)
+
+    with in_directory(dep_dir):
+      _run_cmd(['git', 'fetch'])
+      _run_cmd(['git', 'checkout', '-q', self.revision])
+
+    return os.path.join(dep_dir, self.path)
+
+  def dump(self):
+    return {
+        'repo': self.repo,
+        'branch': self.branch,
+        'revision': self.revision,
+        'path': self.path,
+    }
+
+  def updates(self, context):
+    """Returns a list of all updates to the branch since the revision this
+    repo spec refers to, paired with their commit timestamps; i.e.
+    (timestamp, GitRepoSpec).
+
+    Although timestamps are not completely reliable, they are the best tool we
+    have to approximate global coherence.
+    """
+    lines = filter(bool, self._raw_updates(context).strip().split('\n'))
+    return [ (_parse_date(date),
+              GitRepoSpec(self.id, self.repo, self.branch, rev, self.path))
+             for date,rev in map(str.split, lines) ]
+
+  def root_dir(self, context):
+    return os.path.join(context.package_dir, self.id, self.path)
+
+  def _raw_updates(self, context):
+    self.checkout(context)
+    git = subprocess.Popen(['git', 'log',
+                            '%s..origin/%s' % (self.revision, self.branch),
+                            '--pretty=%aI %H',
+                            '--reverse'],
+                           stdout=subprocess.PIPE,
+                           cwd=os.path.join(context.package_dir, self.id))
+    (stdout, _) = git.communicate()
+    return stdout
+
+  def _components(self):
+    return (self.id, self.repo, self.revision, self.path)
+
+  def __eq__(self, other):
+    return self._components() == other._components()
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+
+class RootRepoSpec(RepoSpec):
+  def __init__(self, pyl_path):
+    self.pyl_path = pyl_path
+
+  def checkout(self, context):
+    # We assume this is already checked out.
+    return context.root_dir
+
+  def root_dir(self, context):
+    return context.root_dir
+
+
+class Package(object):
+  def __init__(self, repo, deps, root_dir):
+    self.repo = repo
+    self.deps = deps
+    self.root_dir = root_dir
+
+  @property
+  def recipe_dirs(self):
+    return [os.path.join(self.root_dir, 'recipes')]
+
+  @property
+  def module_dirs(self):
+    return [os.path.join(self.root_dir, 'recipe_modules')]
+
+  def find_dep(self, dep_name):
+    return self.deps[dep_name]
+
+  def module_path(self, module_name):
+    return os.path.join(self.root_dir, 'recipe_modules', module_name)
+
+
+class PackageSpec(object):
+  def __init__(self, id, deps):
+    self.id = id
+    self.deps = deps
+
+  @classmethod
+  def load_pyl(cls, pyl_path):
+    with open(pyl_path, 'r') as fh:
+      pyl_spec = ast.literal_eval(fh.read())
+    return cls.load(pyl_spec)
+
+  @classmethod
+  def load(cls, spec):
+    assert spec['api_version'] == 0
+    deps = { dep: GitRepoSpec(dep,
+                              dep_dict['repo'],
+                              dep_dict['branch'],
+                              dep_dict['revision'],
+                              dep_dict['path'])
+             for dep, dep_dict in spec['deps'].iteritems() }
+
+    return cls(spec['id'], deps)
+
+  def dump(self):
+    return {
+      'api_version': 0,
+      'id': self.id,
+      'deps': { dep_id: dep.dump() for dep_id, dep in self.deps.iteritems() },
+    }
+
+  def updates(self, context):
+    dep_updates = _merge([ [ (date, dep, update)
+        for date, update in repo.updates(context) ]
+      for dep, repo in self.deps.iteritems() ])
+
+    deps_so_far = self.deps
+    ret_updates = []
+    for (date, dep_id, dep) in dep_updates:
+      deps_so_far = _updated(deps_so_far, { dep_id: dep })
+      ret_updates.append((date, PackageSpec(self.id, deps_so_far)))
+    return ret_updates
+
+  def iterate_consistent_updates(self, context):
+    root_spec = RootRepoSpec(
+        os.path.join(context.root_dir, 'recipe_package.pyl'))
+    for date, spec in self.updates(context):
+      consistent_spec = True
+      try:
+        package_deps = PackageDeps(context)
+        package_deps._create_from_spec(root_spec, spec)
+      except InconsistentDependencyGraphError:
+        # Skip inconsistent graphs, which are blocked on dependency rolls
+        consistent_spec = False
+      if consistent_spec:
+        yield date, spec
+
+  def __eq__(self, other):
+    return self.id == other.id and self.deps == other.deps
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+
+class PackageDeps(object):
+  """An object containing all the transitive dependencies of the root package.
+  """
+  def __init__(self, context):
+    self._context = context
+    self._repos = {}
+
+  @classmethod
+  def create(cls, pyl_path):
+    context = PackageContext.from_pyl_path(pyl_path)
+
+    package_deps = cls(context)
+    root_package = package_deps._create_package(RootRepoSpec(pyl_path))
+    return package_deps
+
+  def _create_package(self, repo_spec):
+    package_root = repo_spec.checkout(self._context)
+    pyl_path = os.path.join(package_root, 'recipe_package.pyl')
+
+    package_spec = PackageSpec.load_pyl(pyl_path)
+    self._create_from_spec(repo_spec, package_spec)
+
+  def _create_from_spec(self, repo_spec, package_spec):
+    deps = {}
+    for dep, dep_repo in sorted(package_spec.deps.items()):
+      deps[dep] = self._create_package(dep_repo)
+
+    if (package_spec.id in self._repos and
+        not repo_spec == self._repos[package_spec.id].repo):
+      raise InconsistentDependencyGraphError(
+          'Package specs do not match: %s vs %s' %
+          (repo_spec, self._repos[package_spec.id].repo))
+
+    package = Package(repo_spec, deps, repo_spec.root_dir(self._context))
+    self._repos[package_spec.id] = package
+    return package
+
+  # TODO(luqui): Remove this, so all accesses to packages are done
+  # via other packages with properly scoped deps.
+  def get_package(self, package_id):
+    return self._repos[package_id]
+
+  @property
+  def all_recipe_dirs(self):
+    for repo in self._repos.values():
+      for subdir in repo.recipe_dirs:
+        yield subdir
+
+  @property
+  def all_module_dirs(self):
+    for repo in self._repos.values():
+      for subdir in repo.module_dirs:
+        yield subdir
+
+
+def _run_cmd(cmd):
+  logging.info('%s', cmd)
+  subprocess.check_call(cmd)
+
+
+def _parse_date(datestr):
+  """Parses an ISO-8601 date string into a datetime object.
+
+  >>> (  _parse_date('2015-06-30T10:15:20-00:00')
+  ... <= _parse_date('2015-06-30T11:20:31-00:00'))
+  True
+  >>> (  _parse_date('2015-06-30T11:33:52-07:00')
+  ... <= _parse_date('2015-06-30T11:33:52-08:00'))
+  True
+  >>> (  _parse_date('2015-06-30T11:33:52-07:00')
+  ... <= _parse_date('2015-06-30T11:33:52-06:00'))
+  False
+  """
+  return dateutil.parser.parse(datestr)
+
+
+def _merge2(xs, ys, compare=lambda x, y: x <= y):
+  """Merges two sorted iterables, preserving sort order.
+
+  >>> list(_merge2([1, 3, 6], [2, 4, 5]))
+  [1, 2, 3, 4, 5, 6]
+  >>> list(_merge2([1, 2, 3], []))
+  [1, 2, 3]
+  >>> list(_merge2([], [4, 5, 6]))
+  [4, 5, 6]
+  >>> list(_merge2([], []))
+  []
+  >>> list(_merge2([4, 2], [3, 1], compare=lambda x, y: x >= y))
+  [4, 3, 2, 1]
+
+  The merge is left-biased and preserves order within each argument.
+
+  >>> list(_merge2([1, 4], [3, 2], compare=lambda x, y: True))
+  [1, 4, 3, 2]
+  """
+
+  xs = iter(xs)
+  ys = iter(ys)
+  x = None
+  y = None
+  try:
+    x = (xs.next(),)
+    y = (ys.next(),)
+
+    while True:
+      if compare(x[0], y[0]):
+        yield x[0]
+        x = None
+        x = (xs.next(),)
+      else:
+        yield y[0]
+        y = None
+        y = (ys.next(),)
+  except StopIteration:
+    if x: yield x[0]
+    for x in xs: yield x
+    if y: yield y[0]
+    for y in ys: yield y
+
+
+def _merge(xss, compare=lambda x, y: x <= y):
+  """Merges a sequence of sorted iterables in sorted order.
+
+  >>> list(_merge([ [1,5], [2,5,6], [], [0,7] ]))
+  [0, 1, 2, 5, 5, 6, 7]
+  >>> list(_merge([ [1,2,3] ]))
+  [1, 2, 3]
+  >>> list(_merge([]))
+  []
+  """
+  return reduce(lambda xs, ys: _merge2(xs, ys, compare=compare), xss, [])
+
+
+def _updated(d, updates):
+  """Updates a dictionary without mutation.
+
+  >>> d = { 'x': 1, 'y': 2 }
+  >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items())
+  [('x', 1), ('y', 3), ('z', 4)]
+  >>> sorted(d.items())
+  [('x', 1), ('y', 2)]
+  """
+
+  d = copy.copy(d)
+  d.update(updates)
+  return d
+
+
+@contextlib.contextmanager
+def in_directory(target_dir):
+  cwd = os.getcwd()
+  os.chdir(target_dir)
+  try:
+    yield
+  finally:
+    os.chdir(cwd)