Cross-repo recipe package system.

third_party/recipe_engine/recipes.py

Index: third_party/recipe_engine/recipes.py
diff --git a/third_party/recipe_engine/recipes.py b/third_party/recipe_engine/recipes.py
new file mode 100755
index 0000000000000000000000000000000000000000..aa71a98f480da770e54f8277cc48ee6acab0f96c
--- /dev/null
+++ b/third_party/recipe_engine/recipes.py
@@ -0,0 +1,265 @@
+#!/usr/bin/env python

[iannucci, 2015/08/06 23:57:13]

maybe call this main.py and rename main.py to run.py

[luqui, 2015/08/20 22:45:25]

Done.

+
+"""Tool to interact with recipe repositories.
+
+This tool operates on the nearest ancestor directory containing a
+recipe_package.pyl file.

[luqui, 2015/08/07 19:30:36]

Update docstring

[luqui, 2015/08/20 22:45:24]

Done.

+"""
+
+import argparse
+import json
+import logging
+import os
+import subprocess
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

[iannucci, 2015/08/06 23:57:13]

!!!!!!!

[luqui, 2015/08/20 22:45:25]

!!!!!!!

+
+
+def get_package_config(args):
+  from recipe_engine import package
+
+  def try_path(path):
+    cand = os.path.realpath(
+        os.path.join(os.getcwd(), path, 'infra', 'config', 'recipes.cfg'))
+    if os.path.exists(cand):
+      return package.ProtoFile(cand)
+    else:
+      raise EnvironmentError(
+          'Recipes config file %s does not exist' % cand)
+
+  if args.package:
+    return try_path(args.package)
+
+  cwd = os.path.realpath(os.getcwd())
+  while os.path.dirname(cwd) != cwd:
+    try:
+      return try_path(cwd)
+    except EnvironmentError:
+      if not os.path.isdir(os.path.join(cwd, '.git')):
+        cwd = os.path.dirname(cwd)
+      else:
+        raise EnvironmentError(
+            'Will not cross git repository boundary %s when looking for '
+            'recipe package spec' % cwd)
+  else:
+    raise EnvironmentError(
+        'Reached root of filesystem when looking for recipe package spec')
+
+
+def simulation_test(package, args):
+  from recipe_engine import simulation_test
+  simulation_test.main(package, args=json.loads(args.args))
+
+
+def lint(package, args):
+  from recipe_engine import lint_test
+  lint_test.main(package)
+
+
+def run(package, args):
+  from recipe_engine import main as recipe_main
+  from recipe_engine import loader
+  from recipe_engine import package
+  from recipe_engine.third_party import annotator
+
+  def get_properties_from_args(args):
+    properties = dict(x.split('=', 1) for x in args)
+    for key, val in properties.iteritems():
+      try:
+        properties[key] = ast.literal_eval(val)
+      except (ValueError, SyntaxError):
+        pass  # If a value couldn't be evaluated, keep the string version
+    return properties
+
+  def get_properties_from_file(filename):
+    properties_file = sys.stdin if filename == '-' else open(filename)
+    properties =  ast.literal_eval(properties_file.read())
+    assert isinstance(properties, dict)
+
+  arg_properties = get_properties_from_args(args.properties)
+  if args.properties_file and arg_properties:
+    raise Exception(
+        'Cannot specify both properties file and command-line properties')
+  elif args.properties_file:
+    properties = get_properties_from_file(args.properties_file)
+  else:
+    properties = arg_properties
+
+  properties['recipe'] = args.recipe
+
+  # TODO(luqui): Environment whitelist. Cf crbug.com/392992
+  os.environ['PYTHONUNBUFFERED'] = '1'
+  os.environ['PYTHONIOENCODING'] = 'UTF-8'
+
+  universe = loader.RecipeUniverse(package)
+
+  workdir = (args.workdir or
+      os.path.join(os.path.dirname(os.path.realpath(__file__)), 'workdir'))
+  logging.info('Using %s as work directory' % workdir)
+  if not os.path.exists(workdir):
+    os.makedirs(workdir)
+
+  old_cwd = os.getcwd()
+  os.chdir(workdir)
+  try:
+    ret = recipe_main.run_steps(
+        properties, annotator.StructuredAnnotationStream(), universe=universe)
+    return ret.status_code
+  finally:
+    os.chdir(old_cwd)
+
+
+def roll(args):

[iannucci, 2015/08/06 23:57:13]

maybe have option to do this in a loop until it can't find any consistent rolls?

[luqui, 2015/08/20 22:45:25]

On second thought, no.  The reason for one at a time is so that we can run
simulation tests and junk.  We would want to roll as far forward as those
passed, but no farther, so this is not the right place for this logic.

+  from recipe_engine import package
+  config_file = get_package_config(args)
+  context = package.PackageContext.from_proto_file(config_file)
+  package_spec = package.PackageSpec.load_proto(config_file)
+
+  for update in package_spec.iterate_consistent_updates(context):
+    config_file.write(update.spec.dump())
+    print 'Wrote %s' % config_file.path
+
+    updated_deps = {
+        dep_id: dep.revision
+        for dep_id, dep in update.spec.deps.iteritems()
+        if dep.revision != package_spec.deps[dep_id].revision
+    }
+    print 'To commit this roll, run:'
+    print ' '.join([
+        'git commit -a -m "Roll dependencies"',
+        ' '.join([ '-m "Roll %s to %s"' % (dep_id, rev)
+                   for dep_id, rev in sorted(updated_deps.iteritems())]),
+    ])
+
+    break
+  else:
+    print 'No consistent rolls found'
+
+
+def show_me_the_modules(package):
+  from recipe_engine import show_me_the_modules
+  show_me_the_modules.main(package)
+
+
+def main():
+  from recipe_engine import package
+
+  # Super-annoyingly, we need to manually parse for simulation_test since
+  # argparse is bonkers and doesn't allow us to forward --help to subcommands.
+  if 'simulation_test' in sys.argv:
+    index = sys.argv.index('simulation_test')
+    sys.argv = sys.argv[:index+1] + [json.dumps(sys.argv[index+1:])]
+
+  parser = argparse.ArgumentParser(description='Do things with recipes.')
+
+  parser.add_argument(
+      '--package',
+      help='Package to operate on (directory containing '
+           'infra/config/recipes.cfg)')
+  parser.add_argument(
+      '--verbose', '-v', action='store_true',
+      help='Increase logging verboisty')
+  parser.add_argument(
+      '--bootstrap', action='store_true',

[luqui, 2015/08/07 19:30:36]

--no-fetch

Don't checkout recipe engine at HEAD.  Instead have copypasta bootstrap check
out pinned recipe engine by parsing the protobuf stupidly and then just forward
to the pinned recipe engine right off.

[luqui, 2015/08/20 22:45:24]

Done.

+      help='Use current rather than pinned version of recipe engine')
+
+  subp = parser.add_subparsers()
+
+  fetch_p = subp.add_parser(
+      'fetch',
+      help='Fetch and update dependencies.')
+  fetch_p.set_defaults(command='fetch')
+
+  simulation_test_p = subp.add_parser('simulation_test',
+      help='Generate or check expectations by simulating with mock actions')
+  simulation_test_p.set_defaults(command='simulation_test')
+  simulation_test_p.add_argument('args')
+
+  lint_p = subp.add_parser(
+      'lint',
+      help='Check recipes for stylistic and hygenic issues')
+  lint_p.set_defaults(command='lint')
+
+  run_p = subp.add_parser(
+      'run',
+      help='Run a recipe locally')
+  run_p.set_defaults(command='run')
+  run_p.add_argument(
+      '--properties-file',
+      help='A file containing a json blob of properties')
+  run_p.add_argument(
+      '--workdir',
+      help='The working directory of recipe execution')
+  run_p.add_argument(
+      'recipe',
+      help='The recipe to execute')
+  run_p.add_argument(
+      'properties', nargs=argparse.REMAINDER,
+      help='A list of property pairs; e.g. mastername=chromium.linux '
+           'issue=12345')
+
+  roll_p = subp.add_parser(
+      'roll',
+      help='Roll dependencies of a recipe package forward')
+  roll_p.set_defaults(command='roll')
+
+  show_me_the_modules_p = subp.add_parser(
+      'show_me_the_modules',

[iannucci, 2015/08/06 23:57:13]

maybe 'doc'? 

doc <module name>
doc <recipe name>
doc --all-modules

?

[luqui, 2015/08/20 22:45:25]

Done

+      help='List all known modules reachable from the current package with '
+           'various info about each')
+  show_me_the_modules_p.set_defaults(command='show_me_the_modules')
+
+  args = parser.parse_args()
+  if args.verbose:
+    logging.getLogger().setLevel(logging.INFO)
+
+  if args.command == 'fetch' or args.command == 'roll':
+    package_deps = package.PackageDeps.create(
+        get_package_config(args), fetch=True)
+  else:
+    package_deps = package.PackageDeps.create(get_package_config(args))

[iannucci, 2015/08/06 23:57:13]

let's have a `fetch = <condition>` line, and then `package_deps = ....(...,
fetch=fetch)` maybe a bit clearer.

[luqui, 2015/08/20 22:45:24]

Nope, because of error message stuff.

+
+  # We don't forward to pinned tool on rolling, because there might be new
+  # things to know about since then, who knows?  Easier to look backward than
+  # forward.
+  if not args.bootstrap and args.command != 'roll':
+    tool = os.path.join(
+        package_deps.get_package('recipe_engine').recipes_dir, 'recipes.py')
+    cmd = [ sys.executable, '-u', tool, '--bootstrap' ] + sys.argv[1:]
+    logging.info(cmd)
+    return subprocess.call(cmd)
+
+  if args.command == 'fetch':
+    # We already did everything in the create() call above.
+    return 0
+  if args.command == 'simulation_test':
+    return simulation_test(package_deps, args)
+  elif args.command == 'lint':
+    return lint(package_deps, args)
+  elif args.command == 'run':
+    return run(package_deps, args)
+  elif args.command == 'roll':
+    return roll(args)
+  elif args.command == 'show_me_the_modules':
+    return show_me_the_modules(package_deps)
+  else:
+    print """Dear sir or madam,
+        It has come to my attention that a quite impossible condition has come
+    to pass in the specification you have issued a request for us to fulfill.
+    It is with a heavy heart that I inform you that, at the present juncture,
+    there is no conceivable next action to be taken upon your request, and as
+    such, we have decided to abort the request with a nonzero status code.  We
+    hope that your larger goals have not been put at risk due to this
+    unfortunate circumstance, and wish you the best in deciding the next action
+    in your venture and larger life.
+
+    Warmly,
+    recipes.py
+    """
+    return 1
+
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main())