Add support for operational arguments protobuf.

recipes.py

Index: recipes.py
diff --git a/recipes.py b/recipes.py
index d78d4abf9bb7d119c6bccdadce671b049b6d95a1..7f835e5d19f60811636a81b59a584a21d272e00b 100755
--- a/recipes.py
+++ b/recipes.py
@@ -20,6 +20,8 @@ ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, ROOT_DIR)
 
 from recipe_engine import env
+from recipe_engine import arguments_pb2
+from google.protobuf import json_format as jsonpb
 
 def get_package_config(args):
   from recipe_engine import package
@@ -84,7 +86,7 @@ def handle_recipe_return(recipe_result, result_filename, stream_engine):
     return 0
 
 
-def run(package_deps, args):
+def run(package_deps, args, op_args):
   from recipe_engine import run as recipe_run
   from recipe_engine import loader
   from recipe_engine import step_runner
@@ -108,14 +110,24 @@ def run(package_deps, args):
   def get_properties_from_json(props):
     return json.loads(props)
 
+  def get_properties_from_operational_args(op_args):
+    if not op_args.properties.property:
+      return None
+    return _op_properties_to_dict(op_args.properties.property)
+
+
   arg_properties = get_properties_from_args(args.props)
+  op_properties = get_properties_from_operational_args(op_args)
   assert len(filter(bool,
-      [arg_properties, args.properties_file, args.properties])) <= 1, (
+      (arg_properties, args.properties_file, args.properties,
+       op_properties))) <= 1, (
           'Only one source of properties is allowed')
   if args.properties:
     properties = get_properties_from_json(args.properties)
   elif args.properties_file:
     properties = get_properties_from_file(args.properties_file)
+  elif op_properties is not None:
+    properties = op_properties
   else:
     properties = arg_properties
 
@@ -139,7 +151,9 @@ def run(package_deps, args):
   os.chdir(workdir)
   stream_engine = stream.ProductStreamEngine(
       stream.StreamEngineInvariants(),
-      stream.AnnotatorStreamEngine(sys.stdout, emit_timestamps=args.timestamps))
+      stream.AnnotatorStreamEngine(
+        sys.stdout, emit_timestamps=(args.timestamps or
+                                     op_args.annotation_flags.emit_timestamp)))
   with stream_engine:
     try:
       ret = recipe_run.run_steps(
@@ -228,6 +242,39 @@ def info(args):
     print package_spec.recipes_path
 
 
+def _op_properties_to_dict(pmap):
+  """Adds an arguments_pb2.PropertyList entry into a dictionary.

[martiniss, 2016/08/10 21:38:25]

"into a dictionary"? This returns a dictionary... 

[dnj, 2016/08/10 22:04:41]

Acknowledged.

+
+  Args:
+    pmap (Map): Protobuf map of property name (str) to arguments_pb2.Property.
+  """
+  d = {}
+
+  for k in pmap:
+    prop = pmap[k]
+    typ = prop.WhichOneof('value')
+    if typ == 's':
+      v = prop.s
+    elif typ == 'int':
+      v = prop.int
+    elif typ == 'uint':
+      v = prop.uint
+    elif typ == 'd':
+      v = prop.d
+    elif typ == 'b':
+      v = prop.b
+    elif typ == 'data':
+      v = prop.data
+    elif typ == 'properties':
+      v = _op_properties_to_dict(prop.properties.property)
+    elif typ is None:
+      v = None
+    else:
+      raise ValueError('Unknown property field [%s]' % (typ,))
+    d[k] = v
+  return d
+
+
 def main():
   from recipe_engine import package
 
@@ -266,6 +313,11 @@ def main():
       '--use-bootstrap', action='store_true',
       help='Use bootstrap/bootstrap.py to create a isolated python virtualenv'
            ' with required python dependencies.')
+  parser.add_argument(
+      '--operational-args-path', action='store',
+      help='The path to an operational Arguments file. If provided, this file '
+           'must contain a JSONPB-encoded Arguments protobuf message, and will '
+           'be integrated into the runtime parameters.')
 
   subp = parser.add_subparsers()
 
@@ -407,6 +459,13 @@ def main():
 
   args = parser.parse_args()
 
+  # Load/parse operational arguments.

[martiniss, 2016/08/10 21:38:25]

Why is this done down here, vs in the run function?

[dnj, 2016/08/10 22:04:41]

The idea is that the operational parameters may sometime be used by something
other than the run function. If you don't think this is going to be the case,
I'll move it to a run parameter and evaluate it there.

+  op_args = arguments_pb2.Arguments()
+  if args.operational_args_path is not None:
+    with open(args.operational_args_path) as fd:
+      data = fd.read()
+    jsonpb.Parse(data, op_args)
+
   if args.use_bootstrap and not os.environ.pop('RECIPES_RUN_BOOTSTRAP', None):
     subprocess.check_call(
         [sys.executable, 'bootstrap/bootstrap.py', '--deps-file',
@@ -455,7 +514,7 @@ def main():
   elif args.command == 'lint':
     return lint(package_deps, args)
   elif args.command == 'run':
-    return run(package_deps, args)
+    return run(package_deps, args, op_args)
   elif args.command == 'autoroll':
     return autoroll(args)
   elif args.command == 'depgraph':