Move Skia recipes from build repo

infra/bots/recipes.py

+#!/usr/bin/env python
+# Copyright 2016 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+"""Bootstrap script to clone and forward to the recipe engine tool.
+***********************************************************************
+** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. **
+***********************************************************************
+This is a copy of https://github.com/luci/recipes-py/blob/master/doc/recipes.py.
+To fix bugs, fix in the github repo then copy it back to here and fix the
+PER-REPO CONFIGURATION section to look like this one.
+"""
+import os
+#### PER-REPO CONFIGURATION (editable) ####

[rmistry, 2016/07/26 18:00:38]

This file is very compact which is fine because it is a copy of the file on line
9. But can we make lines 14 to 19 have newlines and be slightly more readable?

[borenet, 2016/07/26 18:08:45]

Sure, done.

+# The root of the repository relative to the directory of this file.
+REPO_ROOT = os.path.join(os.pardir, os.pardir)
+# The path of the recipes.cfg file relative to the root of the repository.
+RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg')
+#### END PER-REPO CONFIGURATION ####
+BOOTSTRAP_VERSION = 1
+import ast
+import logging
+import random
+import re
+import subprocess
+import sys
+import time
+import traceback
+def parse_protobuf(fh):
+  """Parse the protobuf text format just well enough to understand recipes.cfg.
+  We don't use the protobuf library because we want to be as self-contained
+  as possible in this bootstrap, so it can be simply vendored into a client
+  repo.
+  We assume all fields are repeated since we don't have a proto spec to work
+  with.
+  Args:
+    fh: a filehandle containing the text format protobuf.
+  Returns:
+    A recursive dictionary of lists.
+  """
+  def parse_atom(text):
+    if text == 'true':
+      return True
+    if text == 'false':
+      return False
+    return ast.literal_eval(text)
+  ret = {}
+  for line in fh:
+    line = line.strip()
+    m = re.match(r'(\w+)\s*:\s*(.*)', line)
+    if m:
+      ret.setdefault(m.group(1), []).append(parse_atom(m.group(2)))
+      continue
+    m = re.match(r'(\w+)\s*{', line)
+    if m:
+      subparse = parse_protobuf(fh)
+      ret.setdefault(m.group(1), []).append(subparse)
+      continue
+    if line == '}':
+      return ret
+    if line == '':
+      continue
+    raise ValueError('Could not understand line: <%s>' % line)
+  return ret
+def get_unique(things):
+  if len(things) == 1:
+    return things[0]
+  elif len(things) == 0:
+    raise ValueError("Expected to get one thing, but dinna get none.")
+  else:
+    logging.warn('Expected to get one thing, but got a bunch: %s\n%s' %
+                 (things, traceback.format_stack()))
+    return things[0]
+def _subprocess_call(argv, **kwargs):
+  logging.info('Running %r', argv)
+  return subprocess.call(argv, **kwargs)
+def _subprocess_check_call(argv, **kwargs):
+  logging.info('Running %r', argv)
+  subprocess.check_call(argv, **kwargs)
+def main():
+  if '--verbose' in sys.argv:
+    logging.getLogger().setLevel(logging.INFO)
+  if sys.platform.startswith(('win', 'cygwin')):
+    git = 'git.bat'
+  else:
+    git = 'git'
+  # Find the repository and config file to operate on.
+  repo_root = os.path.abspath(
+      os.path.join(os.path.dirname(__file__), REPO_ROOT))
+  recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG)
+  with open(recipes_cfg_path, 'rU') as fh:
+    protobuf = parse_protobuf(fh)
+  engine_buf = get_unique([
+      b for b in protobuf['deps'] if b.get('project_id') == ['recipe_engine'] ])
+  engine_url = get_unique(engine_buf['url'])
+  engine_revision = get_unique(engine_buf['revision'])
+  engine_subpath = (get_unique(engine_buf.get('path_override', ['']))
+                    .replace('/', os.path.sep))
+  recipes_path = os.path.join(repo_root,
+      get_unique(protobuf['recipes_path']).replace('/', os.path.sep))
+  deps_path = os.path.join(recipes_path, '.recipe_deps')
+  engine_path = os.path.join(deps_path, 'recipe_engine')
+  # Ensure that we have the recipe engine cloned.
+  def ensure_engine():
+    if not os.path.exists(deps_path):
+      os.makedirs(deps_path)
+    if not os.path.exists(engine_path):
+      _subprocess_check_call([git, 'clone', engine_url, engine_path])
+    needs_fetch = _subprocess_call(
+        [git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision],
+        cwd=engine_path, stdout=open(os.devnull, 'w'))
+    if needs_fetch:
+      _subprocess_check_call([git, 'fetch'], cwd=engine_path)
+    _subprocess_check_call(
+        [git, 'checkout', '--quiet', engine_revision], cwd=engine_path)
+  try:
+    ensure_engine()
+  except subprocess.CalledProcessError:
+    logging.exception('ensure_engine failed')
+    # Retry errors.
+    time.sleep(random.uniform(2,5))
+    ensure_engine()
+  args = ['--package', recipes_cfg_path,
+          '--bootstrap-script', __file__] + sys.argv[1:]
+  return _subprocess_call([
+      sys.executable, '-u',
+      os.path.join(engine_path, engine_subpath, 'recipes.py')] + args)
+if __name__ == '__main__':
+  sys.exit(main())