Cross-repo recipe package system.

third_party/recipe_engine/recipes.py

Index: third_party/recipe_engine/recipes.py
diff --git a/third_party/recipe_engine/recipes.py b/third_party/recipe_engine/recipes.py
new file mode 100755
index 0000000000000000000000000000000000000000..654347ef9caee984b91ff84b292e494022ea1669
--- /dev/null
+++ b/third_party/recipe_engine/recipes.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import logging
+import os
+import subprocess
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def get_package_pyl(args):
+  if args.package:
+    cand = os.path.realpath(os.path.join(os.getcwd(), args.package,
+                                         'recipe_package.pyl'))
+    if os.path.exists(cand):
+      return cand
+    else:
+      raise EnvironmentError(
+          'Specified package %s does not exist' % cand)
+
+  cwd = os.path.realpath(os.getcwd())
+  while os.path.dirname(cwd) != cwd:
+    cand = os.path.join(cwd, 'recipe_package.pyl')
+    if os.path.exists(cand):
+      return cand
+
+    if not os.path.isdir(os.path.join(cwd, '.git')):
+      cwd = os.path.dirname(cwd)
+    else:
+      raise EnvironmentError(
+          'Will not cross git repository boundary %s when looking for '
+          'recipe_package.pyl' % cwd)
+  else:
+    raise EnvironmentError(
+        'Reached root of filesystem when looking for recipe_package.pyl')
+
+def simulation_test(package, args):
+  from recipe_engine import simulation_test
+  simulation_test.main(package, args=args.args)
+
+
+def lint(package, args):
+  from recipe_engine import lint_test
+  lint_test.main(package)
+
+
+def run(package, args):
+  from recipe_engine import main as recipe_main
+  from recipe_engine import loader
+  from recipe_engine import package
+  from recipe_engine.third_party import annotator
+
+  def get_properties_from_args(args):
+    properties = dict(x.split('=', 1) for x in args)
+    for key, val in properties.iteritems():
+      try:
+        properties[key] = ast.literal_eval(val)
+      except (ValueError, SyntaxError):
+        pass  # If a value couldn't be evaluated, keep the string version
+    return properties
+
+  def get_properties_from_file(filename):
+    properties_file = sys.stdin if filename == '-' else open(filename)
+    properties =  ast.literal_eval(properties_file.read())
+    assert isinstance(properties, dict)
+
+  arg_properties = get_properties_from_args(args.properties)
+  if args.properties_file and arg_properties:
+    raise Exception(
+        'Cannot specify both properties file and command-line properties')
+  elif args.properties_file:
+    properties = get_properties_from_file(args.properties_file)
+  else:
+    properties = arg_properties
+
+  properties['recipe'] = args.recipe
+
+  # TODO(luqui): Environment whitelist. Cf crbug.com/392992
+  os.environ['PYTHONUNBUFFERED'] = '1'
+  os.environ['PYTHONIOENCODING'] = 'UTF-8'
+
+  universe = loader.RecipeUniverse(package)
+
+  workdir = (args.workdir or
+      os.path.join(os.path.dirname(os.path.realpath(__file__)), 'workdir'))
+  logging.info('Using %s as work directory' % workdir)
+  if not os.path.exists(workdir):
+    os.makedirs(workdir)
+
+  old_cwd = os.getcwd()
+  os.chdir(workdir)
+  try:
+    ret = recipe_main.run_steps(
+        properties, annotator.StructuredAnnotationStream(), universe=universe)
+    return ret.status_code
+  finally:
+    os.chdir(old_cwd)
+
+
+def roll(args):
+  from recipe_engine import package
+  pyl_path = get_package_pyl(args)
+  context = package.PackageContext.from_pyl_path(pyl_path)
+  package_spec = package.PackageSpec.load_pyl(pyl_path)
+  package_spec_dump = package_spec.dump()
+
+  for date, spec in package_spec.iterate_consistent_updates(context):
+    with open(pyl_path, 'w') as fh:
+      json.dump(spec.dump(), fh, indent=2, separators=(',', ': '))
+    print 'Wrote %s' % pyl_path
+
+    updated_deps = {
+        dep_id: dep['revision']
+        for dep_id, dep in spec.dump()['deps'].iteritems()
+        if dep['revision'] != package_spec_dump['deps'][dep_id]['revision']
+    }
+    print 'To commit this roll, run:'
+    print ' '.join([
+        'git commit -a --date "%s" -m "Roll dependencies"' % date.isoformat(),
+        ' '.join([ '-m "Roll %s to %s"' % (dep_id, rev)
+                   for dep_id, rev in sorted(updated_deps.iteritems())]),
+    ])
+
+    break
+  else:
+    print 'No consistent rolls found'
+
+
+def main():
+  from recipe_engine import package
+
+  parser = argparse.ArgumentParser(description='Do things with recipes.')
+
+  parser.add_argument(
+      '--package',
+      help='Package to operate on (directory containing recipe_package.pyl)')
+  parser.add_argument(
+      '--verbose', '-v', action='store_true',
+      help='Increase logging verboisty')
+  parser.add_argument(
+      '--bootstrap', action='store_true',
+      help='Use current rather than pinned version of recipe engine')
+
+  subp = parser.add_subparsers()
+
+  simulation_test_p = subp.add_parser('simulation_test',
+      help='Generate or check expectations by simulating with mock actions')
+  simulation_test_p.set_defaults(command='simulation_test')
+  simulation_test_p.add_argument('args', nargs=argparse.REMAINDER)
+
+  lint_p = subp.add_parser(
+      'lint',
+      help='Check recipes for stylistic and hygenic issues')
+  lint_p.set_defaults(command='lint')
+
+  run_p = subp.add_parser(
+      'run',
+      help='Run a recipe locally')
+  run_p.set_defaults(command='run')
+  run_p.add_argument(
+      '--properties-file',
+      help='A file containing a json blob of properties')
+  run_p.add_argument(
+      '--workdir',
+      help='The working directory of recipe execution')
+  run_p.add_argument(
+      'recipe',
+      help='The recipe to execute')
+  run_p.add_argument(
+      'properties', nargs=argparse.REMAINDER,
+      help='A list of property pairs; e.g. mastername=chromium.linux '
+           'issue=12345')
+
+  roll_p = subp.add_parser(
+      'roll',
+      help='Roll dependencies of a recipe package forward')
+  roll_p.set_defaults(command='roll')
+
+  args = parser.parse_args()
+  if args.verbose:
+    logging.getLogger().setLevel(logging.INFO)
+
+  package_deps = package.PackageDeps.create(get_package_pyl(args))
+  # We don't forward to pinned tool on rolling, because there might be new
+  # things to know about since then, who knows?  Easier to look backward than
+  # forward.
+  if not args.bootstrap and args.command != 'roll':
+    tool = os.path.join(
+        package_deps.get_package('recipe_engine').root_dir, 'recipes.py')
+    cmd = [ sys.executable, '-u', tool, '--bootstrap' ] + sys.argv[1:]
+    logging.info(cmd)
+    return subprocess.call(cmd)
+
+  if args.command == 'simulation_test':
+    return simulation_test(package_deps, args)
+  elif args.command == 'lint':
+    return lint(package_deps, args)
+  elif args.command == 'run':
+    return run(package_deps, args)
+  elif args.command == 'roll':
+    return roll(args)
+  else:
+    print """Dear sir or madam,
+        It has come to my attention that a quite impossible condition has come
+    to pass in the specification you have issued a request for us to fulfill.
+    It is with a heavy heart that I inform you that, at the present juncture,
+    there is no conceivable next action to be taken upon your request, and as
+    such, we have decided to abort the request with a nonzero status code.  We
+    hope that your larger goals have not been put at risk due to this
+    unfortunate circumstance, and wish you the best in deciding the next action
+    in your venture and larger life.
+
+    Warmly,
+    recipes.py
+    """
+    return 1
+
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())