Add support for operational arguments protobuf.

recipes.py

 #!/usr/bin/env python
 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Tool to interact with recipe repositories.
 
 This tool operates on the nearest ancestor directory containing an
 infra/config/recipes.cfg.
 """
 
 import argparse
 import json
 import logging
 import os
 import subprocess
 import sys
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, ROOT_DIR)
 
 from recipe_engine import env
+from recipe_engine import arguments_pb2
+from google.protobuf import json_format as jsonpb
 
 def get_package_config(args):
   from recipe_engine import package
 
   assert args.package, 'No recipe config (--package) given.'
   assert os.path.exists(args.package), (
       'Given recipes config file %s does not exist.' % args.package)
   return (
       package.InfraRepoConfig().from_recipes_cfg(args.package),
       package.ProtoFile(args.package)
@@ skipping 44 matching lines @@
       with s.new_log_stream('reason') as l:
         for line in recipe_result.result['reason'].splitlines():
           l.write_line(line)
 
   if 'status_code' in recipe_result.result:
     return recipe_result.result['status_code']
   else:
     return 0
 
 
-def run(package_deps, args):
+def run(package_deps, args, op_args):
   from recipe_engine import run as recipe_run
   from recipe_engine import loader
   from recipe_engine import step_runner
   from recipe_engine import stream
 
   def get_properties_from_args(args):
     properties = dict(x.split('=', 1) for x in args)
     for key, val in properties.iteritems():
       try:
         properties[key] = json.loads(val)
       except (ValueError, SyntaxError):
         pass  # If a value couldn't be evaluated, keep the string version
     return properties
 
   def get_properties_from_file(filename):
     properties_file = sys.stdin if filename == '-' else open(filename)
     properties = json.load(properties_file)
     assert isinstance(properties, dict)
     return properties
 
   def get_properties_from_json(props):
     return json.loads(props)
 
+  def get_properties_from_operational_args(op_args):
+    if not op_args.properties.property:
+      return None
+    return _op_properties_to_dict(op_args.properties.property)
+
+
   arg_properties = get_properties_from_args(args.props)
+  op_properties = get_properties_from_operational_args(op_args)
   assert len(filter(bool,
-      [arg_properties, args.properties_file, args.properties])) <= 1, (
+      (arg_properties, args.properties_file, args.properties,
+       op_properties))) <= 1, (
           'Only one source of properties is allowed')
   if args.properties:
     properties = get_properties_from_json(args.properties)
   elif args.properties_file:
     properties = get_properties_from_file(args.properties_file)
+  elif op_properties is not None:
+    properties = op_properties
   else:
     properties = arg_properties
 
   properties['recipe'] = args.recipe
 
   os.environ['PYTHONUNBUFFERED'] = '1'
   os.environ['PYTHONIOENCODING'] = 'UTF-8'
 
   _, config_file = get_package_config(args)
   universe = loader.UniverseView(
       loader.RecipeUniverse(
           package_deps, config_file), package_deps.root_package)
 
   workdir = (args.workdir or
       os.path.join(os.path.dirname(os.path.realpath(__file__)), 'workdir'))
   logging.info('Using %s as work directory' % workdir)
   if not os.path.exists(workdir):
     os.makedirs(workdir)
 
   old_cwd = os.getcwd()
   os.chdir(workdir)
   stream_engine = stream.ProductStreamEngine(
       stream.StreamEngineInvariants(),
-      stream.AnnotatorStreamEngine(sys.stdout, emit_timestamps=args.timestamps))
+      stream.AnnotatorStreamEngine(
+        sys.stdout, emit_timestamps=(args.timestamps or
+                                     op_args.annotation_flags.emit_timestamp)))
   with stream_engine:
     try:
       ret = recipe_run.run_steps(
           properties, stream_engine,
           step_runner.SubprocessStepRunner(stream_engine),
           universe=universe)
     finally:
       os.chdir(old_cwd)
 
     return handle_recipe_return(ret, args.output_result_json, stream_engine)
@@ skipping 68 matching lines @@
 
 def info(args):
   from recipe_engine import package
   repo_root, config_file = get_package_config(args)
   package_spec = package.PackageSpec.load_proto(config_file)
 
   if args.recipes_dir:
     print package_spec.recipes_path
 
 
+# Map of arguments_pb2.Property "value" oneof conversion functions.

[martiniss, 2016/08/10 22:54:45]

Add comment saying "Keep in sync with recipe_engine/arguments.proto"?

[dnj, 2016/08/11 01:21:31]

Done.

+_OP_PROPERTY_CONV = {
+    's': lambda prop: prop.s,
+    'int': lambda prop: prop.int,
+    'uint': lambda prop: prop.uint,
+    'd': lambda prop: prop.d,
+    'b': lambda prop: prop.b,
+    'data': lambda prop: prop.data,
+    'map': lambda prop: _op_properties_to_dict(prop.map.property),
+    'list': lambda prop: [_op_property_value(v) for v in prop.list.property],
+}
+
+def _op_property_value(prop):
+  """Returns the Python-converted value of an arguments_pb2.Property.
+
+  Args:
+    prop (arguments_pb2.Property): property to convert.
+  Returns: The converted value.
+  Raises:
+    ValueError: If "prop" is incomplete or invalid.
+  """
+  typ = prop.WhichOneof('value')
+  conv = _OP_PROPERTY_CONV.get(typ)
+  if not conv:
+    raise ValueError('Unknown property field [%s]' % (typ,))
+  return conv(prop)
+
+
+def _op_properties_to_dict(pmap):
+  """Creates a properties dictionary from an arguments_pb2.PropertyMap entry.
+
+  Args:
+    pmap (arguments_pb2.PropertyMap): Map to convert to dictionary form.
+  Returns (dict): A dictionary derived from the properties in "pmap".
+  """
+  return dict((k, _op_property_value(pmap[k])) for k in pmap)
+
+
 def main():
   from recipe_engine import package
 
   # Super-annoyingly, we need to manually parse for simulation_test since
   # argparse is bonkers and doesn't allow us to forward --help to subcommands.
   # Save old_args for if we're using bootstrap
   original_sys_argv = sys.argv[:]
   if 'simulation_test' in sys.argv:
     index = sys.argv.index('simulation_test')
     sys.argv = sys.argv[:index+1] + [json.dumps(sys.argv[index+1:])]
@@ skipping 18 matching lines @@
   parser.add_argument(
       '--bootstrap-script',
       help='Path to the script used to bootstrap this tool (internal use only)')
   parser.add_argument('-O', '--project-override', metavar='ID=PATH',
       action=ProjectOverrideAction,
       help='Override a project repository path with a local one.')
   parser.add_argument(
       '--use-bootstrap', action='store_true',
       help='Use bootstrap/bootstrap.py to create a isolated python virtualenv'
            ' with required python dependencies.')
+  parser.add_argument(
+      '--operational-args-path', action='store',
+      help='The path to an operational Arguments file. If provided, this file '
+           'must contain a JSONPB-encoded Arguments protobuf message, and will '
+           'be integrated into the runtime parameters.')
 
   subp = parser.add_subparsers()
 
   fetch_p = subp.add_parser(
       'fetch',
       description='Fetch and update dependencies.')
   fetch_p.set_defaults(command='fetch')
 
   simulation_test_p = subp.add_parser(
     'simulation_test',
@@ skipping 121 matching lines @@
   info_p = subp.add_parser(
       'info',
       description='Query information about the current recipe package')
   info_p.set_defaults(command='info')
   info_p.add_argument(
       '--recipes-dir', action='store_true',
       help='Get the subpath where the recipes live relative to repository root')
 
   args = parser.parse_args()
 
+  # Load/parse operational arguments.
+  op_args = arguments_pb2.Arguments()
+  if args.operational_args_path is not None:
+    with open(args.operational_args_path) as fd:
+      data = fd.read()
+    jsonpb.Parse(data, op_args)
+
   if args.use_bootstrap and not os.environ.pop('RECIPES_RUN_BOOTSTRAP', None):
     subprocess.check_call(
         [sys.executable, 'bootstrap/bootstrap.py', '--deps-file',
          'bootstrap/deps.pyl', 'ENV'],
         cwd=os.path.dirname(os.path.realpath(__file__)))
 
     os.environ['RECIPES_RUN_BOOTSTRAP'] = '1'
     args = sys.argv
     return subprocess.call(
         [os.path.join(
@@ skipping 28 matching lines @@
 
   if args.command == 'fetch':
     # We already did everything in the create() call above.
     assert not args.no_fetch, 'Fetch? No-fetch? Make up your mind!'
     return 0
   if args.command == 'simulation_test':
     return simulation_test(package_deps, args)
   elif args.command == 'lint':
     return lint(package_deps, args)
   elif args.command == 'run':
-    return run(package_deps, args)
+    return run(package_deps, args, op_args)
   elif args.command == 'autoroll':
     return autoroll(args)
   elif args.command == 'depgraph':
     return depgraph(package_deps, args)
   elif args.command == 'refs':
     return refs(package_deps, args)
   elif args.command == 'doc':
     return doc(package_deps, args)
   elif args.command == 'info':
     return info(args)
@@ skipping 22 matching lines @@
   ret = main()
   if not isinstance(ret, int):
     if ret is None:
       ret = 0
     else:
       print >> sys.stderr, ret
       ret = 1
   sys.stdout.flush()
   sys.stderr.flush()
   os._exit(ret)