Add support for operational arguments protobuf.

recipes.py

 #!/usr/bin/env python
 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Tool to interact with recipe repositories.
 
 This tool operates on the nearest ancestor directory containing an
 infra/config/recipes.cfg.
 """
 
 import argparse
 import json
 import logging
 import os
 import subprocess
 import sys
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, ROOT_DIR)
 
 from recipe_engine import env
+from recipe_engine import arguments_pb2
+from google.protobuf import json_format as jsonpb
 
 def get_package_config(args):
   from recipe_engine import package
 
   assert args.package, 'No recipe config (--package) given.'
   assert os.path.exists(args.package), (
       'Given recipes config file %s does not exist.' % args.package)
   return (
       package.InfraRepoConfig().from_recipes_cfg(args.package),
       package.ProtoFile(args.package)
@@ skipping 44 matching lines @@
       with s.new_log_stream('reason') as l:
         for line in recipe_result.result['reason'].splitlines():
           l.write_line(line)
 
   if 'status_code' in recipe_result.result:
     return recipe_result.result['status_code']
   else:
     return 0
 
 
-def run(package_deps, args):
+def run(package_deps, args, op_args):
   from recipe_engine import run as recipe_run
   from recipe_engine import loader
   from recipe_engine import step_runner
   from recipe_engine import stream
 
   def get_properties_from_args(args):
     properties = dict(x.split('=', 1) for x in args)
     for key, val in properties.iteritems():
       try:
         properties[key] = json.loads(val)
       except (ValueError, SyntaxError):
         pass  # If a value couldn't be evaluated, keep the string version
     return properties
 
   def get_properties_from_file(filename):
     properties_file = sys.stdin if filename == '-' else open(filename)
     properties = json.load(properties_file)
     assert isinstance(properties, dict)
     return properties
 
   def get_properties_from_json(props):
     return json.loads(props)
 
+  def get_properties_from_operational_args(op_args):
+    if not op_args.properties.property:
+      return None
+    return _op_properties_to_dict(op_args.properties.property)
+
+
   arg_properties = get_properties_from_args(args.props)
+  op_properties = get_properties_from_operational_args(op_args)
   assert len(filter(bool,
-      [arg_properties, args.properties_file, args.properties])) <= 1, (
+      (arg_properties, args.properties_file, args.properties,
+       op_properties))) <= 1, (
           'Only one source of properties is allowed')
   if args.properties:
     properties = get_properties_from_json(args.properties)
   elif args.properties_file:
     properties = get_properties_from_file(args.properties_file)
+  elif op_properties is not None:
+    properties = op_properties
   else:
     properties = arg_properties
 
   properties['recipe'] = args.recipe
 
   os.environ['PYTHONUNBUFFERED'] = '1'
   os.environ['PYTHONIOENCODING'] = 'UTF-8'
 
   _, config_file = get_package_config(args)
   universe = loader.UniverseView(
       loader.RecipeUniverse(
           package_deps, config_file), package_deps.root_package)
 
   workdir = (args.workdir or
       os.path.join(os.path.dirname(os.path.realpath(__file__)), 'workdir'))
   logging.info('Using %s as work directory' % workdir)
   if not os.path.exists(workdir):
     os.makedirs(workdir)
 
   old_cwd = os.getcwd()
   os.chdir(workdir)
   stream_engine = stream.ProductStreamEngine(
       stream.StreamEngineInvariants(),
-      stream.AnnotatorStreamEngine(sys.stdout, emit_timestamps=args.timestamps))
+      stream.AnnotatorStreamEngine(
+        sys.stdout, emit_timestamps=(args.timestamps or
+                                     op_args.annotation_flags.emit_timestamp)))
   with stream_engine:
     try:
       ret = recipe_run.run_steps(
           properties, stream_engine,
           step_runner.SubprocessStepRunner(stream_engine),
           universe=universe)
     finally:
       os.chdir(old_cwd)
 
     return handle_recipe_return(ret, args.output_result_json, stream_engine)
@@ skipping 68 matching lines @@
 
 def info(args):
   from recipe_engine import package
   repo_root, config_file = get_package_config(args)
   package_spec = package.PackageSpec.load_proto(config_file)
 
   if args.recipes_dir:
     print package_spec.recipes_path
 
 
+def _op_properties_to_dict(pmap):
+  """Adds an arguments_pb2.PropertyList entry into a dictionary.

[martiniss, 2016/08/10 21:38:25]

"into a dictionary"? This returns a dictionary... 

[dnj, 2016/08/10 22:04:41]

Acknowledged.

+
+  Args:
+    pmap (Map): Protobuf map of property name (str) to arguments_pb2.Property.
+  """
+  d = {}
+
+  for k in pmap:
+    prop = pmap[k]
+    typ = prop.WhichOneof('value')
+    if typ == 's':
+      v = prop.s
+    elif typ == 'int':
+      v = prop.int
+    elif typ == 'uint':
+      v = prop.uint
+    elif typ == 'd':
+      v = prop.d
+    elif typ == 'b':
+      v = prop.b
+    elif typ == 'data':
+      v = prop.data
+    elif typ == 'properties':
+      v = _op_properties_to_dict(prop.properties.property)
+    elif typ is None:
+      v = None
+    else:
+      raise ValueError('Unknown property field [%s]' % (typ,))
+    d[k] = v
+  return d
+
+
 def main():
   from recipe_engine import package
 
   # Super-annoyingly, we need to manually parse for simulation_test since
   # argparse is bonkers and doesn't allow us to forward --help to subcommands.
   # Save old_args for if we're using bootstrap
   original_sys_argv = sys.argv[:]
   if 'simulation_test' in sys.argv:
     index = sys.argv.index('simulation_test')
     sys.argv = sys.argv[:index+1] + [json.dumps(sys.argv[index+1:])]
@@ skipping 18 matching lines @@
   parser.add_argument(
       '--bootstrap-script',
       help='Path to the script used to bootstrap this tool (internal use only)')
   parser.add_argument('-O', '--project-override', metavar='ID=PATH',
       action=ProjectOverrideAction,
       help='Override a project repository path with a local one.')
   parser.add_argument(
       '--use-bootstrap', action='store_true',
       help='Use bootstrap/bootstrap.py to create a isolated python virtualenv'
            ' with required python dependencies.')
+  parser.add_argument(
+      '--operational-args-path', action='store',
+      help='The path to an operational Arguments file. If provided, this file '
+           'must contain a JSONPB-encoded Arguments protobuf message, and will '
+           'be integrated into the runtime parameters.')
 
   subp = parser.add_subparsers()
 
   fetch_p = subp.add_parser(
       'fetch',
       description='Fetch and update dependencies.')
   fetch_p.set_defaults(command='fetch')
 
   simulation_test_p = subp.add_parser(
     'simulation_test',
@@ skipping 121 matching lines @@
   info_p = subp.add_parser(
       'info',
       description='Query information about the current recipe package')
   info_p.set_defaults(command='info')
   info_p.add_argument(
       '--recipes-dir', action='store_true',
       help='Get the subpath where the recipes live relative to repository root')
 
   args = parser.parse_args()
 
+  # Load/parse operational arguments.

[martiniss, 2016/08/10 21:38:25]

Why is this done down here, vs in the run function?

[dnj, 2016/08/10 22:04:41]

The idea is that the operational parameters may sometime be used by something
other than the run function. If you don't think this is going to be the case,
I'll move it to a run parameter and evaluate it there.

+  op_args = arguments_pb2.Arguments()
+  if args.operational_args_path is not None:
+    with open(args.operational_args_path) as fd:
+      data = fd.read()
+    jsonpb.Parse(data, op_args)
+
   if args.use_bootstrap and not os.environ.pop('RECIPES_RUN_BOOTSTRAP', None):
     subprocess.check_call(
         [sys.executable, 'bootstrap/bootstrap.py', '--deps-file',
          'bootstrap/deps.pyl', 'ENV'],
         cwd=os.path.dirname(os.path.realpath(__file__)))
 
     os.environ['RECIPES_RUN_BOOTSTRAP'] = '1'
     args = sys.argv
     return subprocess.call(
         [os.path.join(
@@ skipping 28 matching lines @@
 
   if args.command == 'fetch':
     # We already did everything in the create() call above.
     assert not args.no_fetch, 'Fetch? No-fetch? Make up your mind!'
     return 0
   if args.command == 'simulation_test':
     return simulation_test(package_deps, args)
   elif args.command == 'lint':
     return lint(package_deps, args)
   elif args.command == 'run':
-    return run(package_deps, args)
+    return run(package_deps, args, op_args)
   elif args.command == 'autoroll':
     return autoroll(args)
   elif args.command == 'depgraph':
     return depgraph(package_deps, args)
   elif args.command == 'refs':
     return refs(package_deps, args)
   elif args.command == 'doc':
     return doc(package_deps, args)
   elif args.command == 'info':
     return info(args)
@@ skipping 22 matching lines @@
   ret = main()
   if not isinstance(ret, int):
     if ret is None:
       ret = 0
     else:
       print >> sys.stderr, ret
       ret = 1
   sys.stdout.flush()
   sys.stderr.flush()
   os._exit(ret)