Cross-repo recipe package system.

third_party/recipe_engine/package.py

Index: third_party/recipe_engine/package.py
diff --git a/third_party/recipe_engine/package.py b/third_party/recipe_engine/package.py
new file mode 100644
index 0000000000000000000000000000000000000000..a23503a3af8806b77af26ae24ee5dffdd9cc900c
--- /dev/null
+++ b/third_party/recipe_engine/package.py
@@ -0,0 +1,491 @@
+import ast
+import collections
+import contextlib
+import copy
+import functools
+import itertools
+import logging
+import os
+import subprocess
+import sys
+import tempfile
+
+sys.path.append(

[luqui, 2015/08/07 19:30:36]

import third_party.google instead

[luqui, 2015/08/20 22:45:24]

Done

+    os.path.join(os.path.dirname(os.path.abspath(__file__)), 'third_party'))
+
+import dateutil.parser
+
+from google import protobuf
+from recipe_engine import package_pb2
+
+class UncleanFilesystemError(Exception):
+  pass
+
+
+class InconsistentDependencyGraphError(Exception):
+  pass
+
+
+class ProtoFile(object):

[iannucci, 2015/08/06 23:57:12]

document why an object and not just a collection of functions?

[luqui, 2015/08/20 22:45:24]

Done

+  def __init__(self, path):
+    self._path = path
+
+  @property
+  def path(self):
+    return os.path.realpath(self._path)
+
+  def read_text(self):
+    with open(self._path, 'r') as fh:
+      return fh.read()
+
+  def read(self):
+    text = self.read_text()
+    buf = package_pb2.Package()
+    protobuf.text_format.Merge(text, buf)
+    return buf
+
+  def to_text(self, buf):
+    return protobuf.text_format.MessageToString(buf)
+
+  def write(self, buf):
+    with open(self._path, 'w') as fh:
+      fh.write(self.to_text(buf))
+
+
+class PackageContext(object):
+  """Contains information about where the root package and its dependency
+  checkouts live.
+
+  - recipes_dir is the location of recipes/ and recipe_modules/ which contain
+    the actual recipes of the root package.
+  - package_dir is where dependency checkouts live, e.g.
+    package_dir/recipe_engine/recipes/...
+  - repo_root is the root of the repository containing the root package.
+  """
+
+  def __init__(self, recipes_dir, package_dir, repo_root):
+    self.recipes_dir = recipes_dir
+    self.package_dir = package_dir
+    self.repo_root = repo_root
+
+  @classmethod
+  def from_proto_file(cls, proto_file):
+    if isinstance(proto_file, basestring):
+      proto_file = ProtoFile(proto_file)

[iannucci, 2015/08/06 23:57:12]

nuke

[luqui, 2015/08/20 22:45:24]

Done

+    proto_path = proto_file.path
+
+    repo_root = os.path.dirname(os.path.dirname(os.path.dirname(proto_path)))
+    expected_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg')
+    assert proto_path == expected_path, (
+        'Recipes config must be in infra/config/recipes.cfg from root of repo\n'

[iannucci, 2015/08/06 23:57:12]

maybe need to talk to luci-cfg to see if this is the right path... maybe we
don't need to. There debate about making this flexible.

[luqui, 2015/08/20 22:45:24]

Ugh, it is configurable.

Since we don't need this for initial rollout, I'm going to abstract it a little
in preparation but not go all the way.

This is a bigger problem for the bootstrap. The alternatives are:

1) have the bootstrap parse refs.cfg (a protobuf) in order to find the correct
location.
2) add a special variable in the bootstrap where you can say "use this as the
config file instead"

I'm thinking (2) right now.  I know it's ugly, but the extra complexity of doing
it automatically in a bootstrap script which cannot be updated is substantial. 
Doing (2) means you can't copypasta an updated bootstrap script, mostly.  Ughhhh

+        '  Expected location: %s\n'
+        '  Actual location: %s\n'
+        % (expected_path, proto_path))

[iannucci, 2015/08/06 23:57:13]

can we validate this higher in the stack? like main()? or arg parsing

[luqui, 2015/08/20 22:45:24]

Done.

+
+    buf = proto_file.read()
+
+    recipes_path = buf.recipes_path
+    if sys.platform.startswith('win'):
+      recipes_path.replace('/', '\\')

[iannucci, 2015/08/06 23:57:12]

os.sep instead of '\\'

  recipes_path = buf.recipes_path.replace('/', os.sep)

[luqui, 2015/08/20 22:45:24]

Done

+
+    return cls(os.path.join(repo_root, recipes_path),
+               os.path.join(repo_root, recipes_path, '.recipe_deps'),
+               repo_root)
+
+
+@functools.total_ordering
+class RepoUpdate(object):
+  def __init__(self, spec):
+    self.spec = spec
+
+  @property
+  def id(self):
+    return self.spec.id
+
+  def __eq__(self, other):
+    return (self.id, self.spec.revision) == (other.id, other.spec.revision)
+
+  def __le__(self, other):
+    return (self.id, self.spec.revision) <= (other.id, other.spec.revision)
+
+
+class RepoSpec(object):
+  def checkout(self, context):

[iannucci, 2015/08/06 23:57:12]

need check_checkout (and whatever other methods there are)

maybe a docstring for RepoSpec?

[luqui, 2015/08/20 22:45:24]

Done

+    """Fetches the specified package and returns the path of the package root
+    (the directory that contains recipes and recipe_modules).
+    """
+    raise NotImplementedError()
+
+
+class GitRepoSpec(RepoSpec):
+  def __init__(self, id, repo, branch, revision, path):
+    self.id = id
+    self.repo = repo
+    self.branch = branch
+    self.revision = revision
+    self.path = path
+
+  def checkout(self, context):
+    package_dir = context.package_dir
+    dep_dir = os.path.join(package_dir, self.id)
+    logging.info('Freshening repository %s' % dep_dir)
+
+    if not os.path.isdir(dep_dir):
+      _run_cmd(['git', 'clone', self.repo, dep_dir])
+    elif not os.path.isdir(os.path.join(dep_dir, '.git')):
+      raise UncleanFilesystemError('%s exists but is not a git repo' % dep_dir)
+
+    with _in_directory(dep_dir):
+      _run_cmd(['git', 'fetch'])

[iannucci, 2015/08/06 23:57:13]

mebbeh use cwd=

[luqui, 2015/08/07 19:30:36]

check & fetch, auto-fetch. 

[luqui, 2015/08/20 22:45:24]

Done
Done

+      _run_cmd(['git', 'reset', '--hard', self.revision])
+
+  def check_checkout(self, context):
+    dep_dir = os.path.join(context.package_dir, self.id)
+    if not os.path.isdir(dep_dir):
+      raise UncleanFilesystemError('Dependency %s does not exist' %
+                                   dep_dir)
+    elif not os.path.isdir(os.path.join(dep_dir, '.git')):
+      raise UncleanFilesystemError('Dependency %s is not a git repo' %
+                                   dep_dir)
+
+    with _in_directory(dep_dir):
+      git_status_command = ['git', 'status', '--porcelain']
+      logging.info('%s', git_status_command)
+      output = subprocess.check_output(git_status_command)
+      if output:
+        raise UncleanFilesystemError('Dependency %s is unclean:\n%s' %
+                                     (dep_dir, output))
+
+
+  def repo_root(self, context):
+    return os.path.join(context.package_dir, self.id, self.path)
+
+  def dump(self):
+    buf = package_pb2.DepSpec(
+        project_id=self.id,
+        url=self.repo,
+        branch=self.branch,
+        revision=self.revision)
+    if self.path:
+      buf.path_override = self.path
+    return buf
+
+  def updates(self, context):
+    """Returns a list of all updates to the branch since the revision this
+    repo spec refers to, paired with their commit timestamps; i.e.
+    (timestamp, GitRepoSpec).
+
+    Although timestamps are not completely reliable, they are the best tool we
+    have to approximate global coherence.
+    """
+    lines = filter(bool, self._raw_updates(context).strip().split('\n'))
+    return [ RepoUpdate(
+                 GitRepoSpec(self.id, self.repo, self.branch, rev, self.path))
+             for rev in lines ]
+
+  def _raw_updates(self, context):
+    self.checkout(context)
+    # XXX(luqui): Should this just focus on the recipes subtree rather than
+    # the whole repo?
+    git = subprocess.Popen(['git', 'log',
+                            '%s..origin/%s' % (self.revision, self.branch),

[iannucci, 2015/08/06 23:57:12]

'origin' is an assumption? or is this a repo managed precisely by us.

[luqui, 2015/08/20 22:45:24]

This repo is managed by us.

+                            '--pretty=%H',
+                            '--reverse'],
+                           stdout=subprocess.PIPE,
+                           cwd=os.path.join(context.package_dir, self.id))
+    (stdout, _) = git.communicate()
+    return stdout
+
+  def _components(self):
+    return (self.id, self.repo, self.revision, self.path)
+
+  def __eq__(self, other):
+    return self._components() == other._components()
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+
+class RootRepoSpec(RepoSpec):
+  def __init__(self):
+    pass
+
+  def checkout(self, context):
+    # We assume this is already checked out.
+    pass
+
+  def check_checkout(self, context):
+    pass
+
+  def repo_root(self, context):
+    return context.repo_root
+
+
+class Package(object):
+  def __init__(self, repo, deps, recipes_dir):
+    self.repo = repo

[iannucci, 2015/08/06 23:57:12]

s/repo/repo_spec

doc type of deps

doc that repo_spec is used externally

[luqui, 2015/08/20 22:45:24]

Done

+    self.deps = deps
+    self.recipes_dir = recipes_dir
+
+  @property
+  def recipe_dirs(self):
+    return [os.path.join(self.recipes_dir, 'recipes')]
+
+  @property
+  def module_dirs(self):
+    return [os.path.join(self.recipes_dir, 'recipe_modules')]
+
+  def find_dep(self, dep_name):
+    return self.deps[dep_name]
+
+  def module_path(self, module_name):
+    return os.path.join(self.recipes_dir, 'recipe_modules', module_name)
+
+
+class PackageSpec(object):
+  API_VERSION = 1
+
+  def __init__(self, project_id, recipes_path, deps):
+    self._project_id = project_id
+    self._recipes_path = recipes_path
+    self._deps = deps
+
+  @classmethod
+  def load_proto(cls, proto_file):
+    buf = proto_file.read()
+    assert buf.api_version == cls.API_VERSION
+
+    deps = { dep.project_id: GitRepoSpec(dep.project_id,
+                                         dep.url,
+                                         dep.branch,
+                                         dep.revision,
+                                         dep.path_override)
+             for dep in buf.deps }
+    return cls(buf.project_id, buf.recipes_path, deps)
+
+  @property
+  def project_id(self):
+    return self._project_id
+
+  @property
+  def recipes_path(self):
+    return self._recipes_path
+
+  @property
+  def deps(self):
+    return self._deps
+
+  def dump(self):
+    return package_pb2.Package(
+        api_version=self.API_VERSION,
+        project_id=self._project_id,
+        recipes_path=self._recipes_path,
+        deps=[ self._deps[dep].dump() for dep in sorted(self._deps.keys()) ])
+
+  def updates(self, context):

[iannucci, 2015/08/06 23:57:12]

need docstrings

[luqui, 2015/08/20 22:45:24]

Done

+    dep_updates = _merge([
+        self._deps[dep].updates(context) for dep in sorted(self._deps.keys()) ])
+
+    deps_so_far = self._deps
+    ret_updates = []
+    for update in dep_updates:
+      deps_so_far = _updated(deps_so_far, { update.id: update.spec })
+      ret_updates.append(RepoUpdate(PackageSpec(

[iannucci, 2015/08/06 23:57:12]

maybe don't need RepoUpdate because it's only used for comparison and we don't
end up comparing them?

[luqui, 2015/08/07 19:30:36]

Document how rolling work.
Document global coherence and why we don't do it.

[luqui, 2015/08/20 22:45:24]

Done

+          self._project_id, self._recipes_path, deps_so_far)))
+    return ret_updates
+
+  def iterate_consistent_updates(self, context):
+    root_spec = RootRepoSpec()
+    for update in self.updates(context):
+      consistent_spec = True
+      try:
+        package_deps = PackageDeps(context)
+        package_deps._create_from_spec(root_spec, update.spec, fetch=True)
+      except InconsistentDependencyGraphError:
+        # Skip inconsistent graphs, which are blocked on dependency rolls
+        consistent_spec = False
+      if consistent_spec:
+        yield update

[iannucci, 2015/08/06 23:57:12]

remove consistent_spec

try:
  ...
  yield update
except ...:
  ...

[luqui, 2015/08/20 22:45:24]

Done

+
+  def __eq__(self, other):
+    return (
+        self._project_id == other._project_id and
+        self._recipes_path == other._recipes_path and
+        self._deps == other._deps)
+
+  def __ne__(self, other):
+    return not self.__eq__(other)
+
+
+class PackageDeps(object):
+  """An object containing all the transitive dependencies of the root package.
+  """
+  def __init__(self, context):
+    self._context = context
+    self._repos = {}
+
+  @classmethod
+  def create(cls, proto_file, fetch=False):

[iannucci, 2015/08/06 23:57:13]

doc: proto_file is the root-repo's `infra/config/recipes.cfg` file

[luqui, 2015/08/20 22:45:24]

Done

+    if isinstance(proto_file, basestring):
+      proto_file = ProtoFile(proto_file)

[iannucci, 2015/08/06 23:57:12]

nuke2faic (always require ProtoFile)

[luqui, 2015/08/20 22:45:24]

Done

+    context = PackageContext.from_proto_file(proto_file)
+    package_deps = cls(context)
+
+    root_package = package_deps._create_package(RootRepoSpec(), fetch)

[iannucci, 2015/08/06 23:57:12]

maybe name 'allowFetch'?

just call _create_from_spec directly here?

[luqui, 2015/08/20 22:45:24]

allow_fetch, Done.
I think this way makes more logical sense actually.

+    return package_deps
+
+  def _create_package(self, repo_spec, fetch):
+    if fetch:
+      repo_spec.checkout(self._context)
+    else:
+      try:
+        repo_spec.check_checkout(self._context)
+      except UncleanFilesystemError as e:
+        logging.warn(
+            'Unclean environment. You probably need to run "recipes.py fetch"\n'
+            '%s' % e.message)
+
+    proto_path = os.path.join(repo_spec.repo_root(self._context),
+                              'infra', 'config', 'recipes.cfg')

[iannucci, 2015/08/06 23:57:12]

can this 'infra' 'config' join concept be abstracted everywhere? too many
hardcoded strings?

[luqui, 2015/08/20 22:45:24]

Done mostly

+    package_spec = PackageSpec.load_proto(ProtoFile(proto_path))
+
+    return self._create_from_spec(repo_spec, package_spec, fetch)
+
+  def _create_from_spec(self, repo_spec, package_spec, fetch):

[iannucci, 2015/08/06 23:57:12]

I have sneaking suspicion that repo_spec is only really used for repo_root from
this point on.... looks like it maybe used for revision too. I'm a tad worried
that someone might call a state-affecting method (like 'checkout') on the object
elsewhere in the code, even though I would expect that the only call to checkout
happens at PackageDeps._create_package time.

+    deps = {}
+    for dep, dep_repo in sorted(package_spec.deps.items()):
+      deps[dep] = self._create_package(dep_repo, fetch)
+
+    if (package_spec.project_id in self._repos and
+        not repo_spec == self._repos[package_spec.project_id].repo):

[iannucci, 2015/08/06 23:57:12]

!=




possibly?

[luqui, 2015/08/20 22:45:24]

Done







.

+      raise InconsistentDependencyGraphError(
+          'Package specs do not match: %s vs %s' %
+          (repo_spec, self._repos[package_spec.project_id].repo))
+
+    package = Package(
+        repo_spec, deps,
+        os.path.join(repo_spec.repo_root(self._context),
+                     package_spec.recipes_path))
+
+    self._repos[package_spec.project_id] = package

[iannucci, 2015/08/06 23:57:12]

need a cycle breaker "currently loading" object per project_id 

[luqui, 2015/08/20 22:45:24]

Done

+    return package
+
+  # TODO(luqui): Remove this, so all accesses to packages are done
+  # via other packages with properly scoped deps.
+  def get_package(self, package_id):
+    return self._repos[package_id]
+
+  @property
+  def all_recipe_dirs(self):
+    for repo in self._repos.values():
+      for subdir in repo.recipe_dirs:
+        yield str(subdir)
+
+  @property
+  def all_module_dirs(self):
+    for repo in self._repos.values():
+      for subdir in repo.module_dirs:
+        yield str(subdir)
+
+
+def _run_cmd(cmd):
+  logging.info('%s', cmd)
+  subprocess.check_call(cmd)
+
+
+def _parse_date(datestr):

[iannucci, 2015/08/06 23:57:13]

don't need this :P

[luqui, 2015/08/20 22:45:24]

Done

+  """Parses an ISO-8601 date string into a datetime object.
+
+  >>> (  _parse_date('2015-06-30T10:15:20-00:00')
+  ... <= _parse_date('2015-06-30T11:20:31-00:00'))
+  True
+  >>> (  _parse_date('2015-06-30T11:33:52-07:00')
+  ... <= _parse_date('2015-06-30T11:33:52-08:00'))
+  True
+  >>> (  _parse_date('2015-06-30T11:33:52-07:00')
+  ... <= _parse_date('2015-06-30T11:33:52-06:00'))
+  False
+  """
+  return dateutil.parser.parse(datestr)
+
+
+def _merge2(xs, ys, compare=lambda x, y: x <= y):

[iannucci, 2015/08/06 23:57:13]

maybe just

return sorted(xs + ys)

[luqui, 2015/08/20 22:45:24]

Will not work, since we need the order in xs and ys to be preserved

+  """Merges two sorted iterables, preserving sort order.
+
+  >>> list(_merge2([1, 3, 6], [2, 4, 5]))
+  [1, 2, 3, 4, 5, 6]
+  >>> list(_merge2([1, 2, 3], []))
+  [1, 2, 3]
+  >>> list(_merge2([], [4, 5, 6]))
+  [4, 5, 6]
+  >>> list(_merge2([], []))
+  []
+  >>> list(_merge2([4, 2], [3, 1], compare=lambda x, y: x >= y))
+  [4, 3, 2, 1]
+
+  The merge is left-biased and preserves order within each argument.
+
+  >>> list(_merge2([1, 4], [3, 2], compare=lambda x, y: True))
+  [1, 4, 3, 2]
+  """
+

[iannucci, 2015/08/06 23:57:12]

not_a_thing = object()

[luqui, 2015/08/20 22:45:24]

Done

+  xs = iter(xs)
+  ys = iter(ys)
+  x = None
+  y = None
+  try:
+    x = (xs.next(),)
+    y = (ys.next(),)
+
+    while True:
+      if compare(x[0], y[0]):
+        yield x[0]
+        x = None
+        x = (xs.next(),)
+      else:
+        yield y[0]
+        y = None
+        y = (ys.next(),)
+  except StopIteration:
+    if x: yield x[0]
+    for x in xs: yield x
+    if y: yield y[0]
+    for y in ys: yield y
+
+
+def _merge(xss, compare=lambda x, y: x <= y):
+  """Merges a sequence of sorted iterables in sorted order.
+
+  >>> list(_merge([ [1,5], [2,5,6], [], [0,7] ]))
+  [0, 1, 2, 5, 5, 6, 7]
+  >>> list(_merge([ [1,2,3] ]))
+  [1, 2, 3]
+  >>> list(_merge([]))
+  []
+  """
+  return reduce(lambda xs, ys: _merge2(xs, ys, compare=compare), xss, [])
+
+
+def _updated(d, updates):
+  """Updates a dictionary without mutation.
+
+  >>> d = { 'x': 1, 'y': 2 }
+  >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items())
+  [('x', 1), ('y', 3), ('z', 4)]
+  >>> sorted(d.items())
+  [('x', 1), ('y', 2)]
+  """
+
+  d = copy.copy(d)
+  d.update(updates)
+  return d
+
+
+@contextlib.contextmanager
+def _in_directory(target_dir):
+  cwd = os.getcwd()
+  os.chdir(target_dir)
+  try:
+    yield
+  finally:
+    os.chdir(cwd)