Cross-repo recipe package system.

scripts/slave/recipes.py

Index: scripts/slave/recipes.py
diff --git a/scripts/slave/recipes.py b/scripts/slave/recipes.py
new file mode 100755
index 0000000000000000000000000000000000000000..f3fd19ef7a7d459ea9778688f48544c263dbb428
--- /dev/null
+++ b/scripts/slave/recipes.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Bootstrap script to clone and forward to the recipe engine tool."""
+
+import ast
+import os
+import random
+import re
+import subprocess
+import sys
+import time
+
+BOOTSTRAP_VERSION = 1
+RECIPES_CFG = os.path.join(
+    os.pardir, os.pardir, 'infra', 'config', 'recipes.cfg')
+
+
+def parse_protobuf(fh):
+  """Parse the protobuf text format just well enough to understand recipes.cfg.
+
+  We don't use the protobuf library because we want to be as self-contained
+  as possible in this bootstrap, so it can be simply vendored into a client
+  repo.
+
+  We assume all fields are repeated since we don't have a proto spec to work
+  with.
+
+  Args:
+    fh: a filehandle containing the text format protobuf.
+  Returns:
+    A recursive dictionary of lists.
+  """
+  def parse_atom(text):
+    if text == 'true': return True
+    if text == 'false': return False
+    return ast.literal_eval(text)
+
+  ret = {}
+  for line in fh:
+    line = line.strip()
+    m = re.match(r'(\w+)\s*:\s*(.*)', line)

[iannucci, 2015/09/15 00:10:48]

can this file have comments?

[luqui, 2015/09/15 19:17:42]

It cannot.  For once this makes me happy.

+    if m:
+      ret.setdefault(m.group(1), []).append(parse_atom(m.group(2)))

[iannucci, 2015/09/15 00:10:48]

why not inline parse_atom? It's super small and only used once I think

[luqui, 2015/09/15 19:17:41]

Because it involves an intermediate variable, and mixes together some cleanly
separable functionality with a name.  "Why not split this out into a function?"

+      continue
+
+    m = re.match(r'(\w+)\s*{', line)
+    if m:
+      subparse = parse_protobuf(fh)
+      ret.setdefault(m.group(1), []).append(subparse)
+      continue
+
+    if line == '}': return ret
+    if line == '': continue
+
+    raise Exception('Could not understand line: <%s>' % line)
+
+  return ret
+
+
+
+def main():
+  # Find the repository and config file to operate on.
+  git_dir = os.path.dirname(
+    subprocess.check_output(['git', 'rev-parse', '--git-dir'],
+    cwd=os.path.dirname(__file__)).strip())
+  recipes_cfg_path = os.path.join(os.path.dirname(__file__), RECIPES_CFG)
+
+  with open(recipes_cfg_path, 'rU') as fh:
+    protobuf = parse_protobuf(fh)
+
+  [engine_buf] = [

[iannucci, 2015/09/15 00:10:48]

whoa... how did I not know about this particular assignment syntax.... seems
like it might be safer to just call next() on the genexp tho, in case there are
somehow multiple matching things.

  foo = next(x for x in thingee)

is the efficient version of

  foo = [x for x in thingee][0]

Though you're actually using a list comp anyway (don't do that in general... use
genexps), so you're already paying the allocation overhead.

[luqui, 2015/09/15 19:17:42]

Seems like it is safer to die here instead of choose arbitrarily in the case of
multiple matching things.  I'm going to pick the 0th element and warn if there
is more than one.

+      b for b in protobuf['deps'] if b.get('project_id') == ['recipe_engine'] ]
+  [engine_url] = engine_buf['url']
+  [engine_revision] = engine_buf['revision']
+  [engine_subpath] = engine_buf.get('path_override', [''])
+  engine_subpath = engine_subpath.replace('/', os.path.sep)
+
+  [recipes_path] = protobuf['recipes_path']
+  recipes_path = os.path.join(git_dir, recipes_path.replace('/', os.path.sep))
+  deps_path = os.path.join(recipes_path, '.recipe_deps')
+  engine_path = os.path.join(deps_path, 'recipe_engine')
+
+  # Ensure that we have the recipe engine cloned.
+  def ensure_engine():
+    if not os.path.exists(deps_path):
+      os.makedirs(deps_path)
+    if not os.path.exists(engine_path):
+      subprocess.check_call(['git', 'clone', engine_url, engine_path])
+
+    needs_fetch = subprocess.call(
+        ['git', 'rev-parse', '--verify', '%s^{commit}' % engine_revision],
+        cwd=engine_path, stdout=open(os.devnull, 'w'))
+    if needs_fetch:
+      subprocess.check_call(['git', 'fetch'], cwd=engine_path)
+    subprocess.check_call(
+        ['git', 'checkout', '--quiet', engine_revision], cwd=engine_path)
+
+  try:
+    ensure_engine()
+  except subprocess.CalledProcessError as e:
+    if e.returncode == 128:  # Thrown when git gets a lock error.
+      time.sleep(random.uniform(2,5))
+      ensure_engine()
+    else:
+      raise
+
+  args = ([sys.argv[0]] + ['--package', recipes_cfg_path,
+                           '--bootstrap-script', __file__] + sys.argv[1:])
+  os.execvp(
+      os.path.join(engine_path, engine_subpath, 'recipes.py'),
+      args)
+
+if __name__ == '__main__':
+  main()