Setup basic Runtime with properties and platform.

recipe_engine/run.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Entry point for fully-annotated builds.
 
 This script is part of the effort to move all builds to annotator-based
 systems. Any builder configured to use the AnnotatorFactory.BaseFactory()
 found in scripts/master/factory/annotator_factory.py executes a single
 AddAnnotatedScript step. That step (found in annotator_commands.py) calls
@@ skipping 48 matching lines @@
     convenience functions defined:
       * last_step   - Returns the last step that ran or None
       * nth_step(n) - Returns the N'th step that ran or None
 
 'failed' is a boolean representing if the build is in a 'failed' state.
 """
 
 import collections
 import json
 import os
-import sys
 import traceback
 
 from . import env
 
 from . import loader
 from . import recipe_api
 from . import recipe_test_api
 from . import step_runner as step_runner_module
 from . import types
 from . import util
@@ skipping 101 matching lines @@
 
 
 # Return value of run_steps and RecipeEngine.run.  Just a container for the
 # literal return value of the recipe.
 RecipeResult = collections.namedtuple('RecipeResult', 'result')
 
 
 # TODO(dnj): Replace "properties" with a generic runtime instance. This instance
 # will be used to seed recipe clients and expanded to include managed runtime
 # entities.
-def run_steps(properties, stream_engine, step_runner, universe_view):
+def run_steps(rt, stream_engine, step_runner, universe_view):
   """Runs a recipe (given by the 'recipe' property).
 
   Args:
-    properties: a dictionary of properties to pass to the recipe.  The
-      'recipe' property defines which recipe to actually run.
+    rt (Runtime): The runtime instance.
     stream_engine: the StreamEngine to use to create individual step streams.
     step_runner: The StepRunner to use to 'actually run' the steps.
     universe_view: The RecipeUniverse to use to load the recipes & modules.
 
   Returns: RecipeResult
   """
-  # NOTE(iannucci): 'root' was a terribly bad idea and has been replaced by

[dnj, 2016/10/13 01:08:23]

Talked with iannucci@, probably don't need this.

-  # 'patch_project'. 'root' had Rietveld knowing about the implementation of
-  # the builders. 'patch_project' lets the builder (recipe) decide its own
-  # destiny.
-  properties.pop('root', None)
-
-  # TODO(iannucci): A much better way to do this would be to dynamically

[dnj, 2016/10/13 01:08:23]

Transplanted into "recipes.py", closer to other properties setup.

-  #   detect if the mirrors are actually available during the execution of the
-  #   recipe.
-  if ('use_mirror' not in properties and (
-    'TESTING_MASTERNAME' in os.environ or
-    'TESTING_SLAVENAME' in os.environ)):
-    properties['use_mirror'] = False
-
   with stream_engine.make_step_stream('setup_build') as s:
-    engine = RecipeEngine(step_runner, properties, universe_view)
+    engine = RecipeEngine(step_runner, rt, universe_view)
 
     # Create all API modules and top level RunSteps function.  It doesn't launch
     # any recipe code yet; RunSteps needs to be called.
     api = None
 
-    assert 'recipe' in properties
-    recipe = properties['recipe']
+    assert 'recipe' in rt.properties
+    recipe = rt.properties['recipe']
 
-    properties_to_print = properties.copy()
-    if 'use_mirror' in properties:
-      del properties_to_print['use_mirror']
+    # Remove domain-specific properties.
+    properties_to_print = rt.properties.copy()
+    properties_to_print.pop('use_mirror', None)

[iannucci, 2016/10/15 00:39:45]

I would ditch this

[dnj, 2016/10/15 00:59:27]

altogether, or this specific change?

[iannucci, 2016/10/15 01:04:09]

Well you're moving the other property stuff in this change... I would actually
feel a bit more comfortable if the deletion of the property-massaging code was a
separate CL (before or after this one)

[dnj, 2016/10/15 15:42:05]

Done.

 
     root_package = universe_view.universe.package_deps.root_package
     run_recipe_help_lines = [
         'To repro this locally, run the following line from the root of a %r'
           ' checkout:' % (root_package.name),
         '',
         '%s run --properties-file - %s <<EOF' % (
             os.path.join( '.', root_package.relative_recipes_dir, 'recipes.py'),
             recipe),
         '%s' % json.dumps(properties_to_print),
         'EOF',
         '',
         'To run on Windows, you can put the JSON in a file and redirect the',
         'contents of the file into run_recipe.py, with the < operator.',
     ]
 
     with s.new_log_stream('run_recipe') as l:
       for line in run_recipe_help_lines:
         l.write_line(line)
 
-    _isolate_environment()
+    os.environ = _isolate_environment(rt, os.environ)
 
     # Find and load the recipe to run.
     try:
       recipe_script = universe_view.load_recipe(recipe, engine=engine)
-      s.write_line('Running recipe with %s' % (properties,))
+      s.write_line('Running recipe with %s' % (rt.properties,))
 
       api = loader.create_recipe_api(recipe_script.LOADED_DEPS,
                                      engine,
                                      recipe_test_api.DisabledTestData())
 
       s.add_step_text('running recipe: "%s"' % recipe)
     except (loader.LoaderError, ImportError, AssertionError) as e:
       for line in str(e).splitlines():
         s.add_step_text(line)
       s.set_step_status('EXCEPTION')
       return RecipeResult({
           'status_code': 2,
           'reason': str(e),
       })
 
   # Run the steps emitted by a recipe via the engine, emitting annotations
   # into |stream| along the way.
-  return engine.run(recipe_script, api, properties)
+  return engine.run(recipe_script, api, rt.properties)
 
 
-def _isolate_environment():
+def _isolate_environment(rt, env):
   """Isolate the environment to a known subset set."""
-  if sys.platform.startswith('win'):
+  if rt.platform.is_win():
     whitelist = ENV_WHITELIST_WIN
-  elif sys.platform in ('darwin', 'posix', 'linux2'):
+  elif rt.platform.is_posix():
     whitelist = ENV_WHITELIST_POSIX
   else:
-    print ('WARNING: unknown platform %s, not isolating environment.' %
-           sys.platform)
-    return
+    print 'WARNING: unknown platform %s, not isolating environment.' % (
+        rt.platform,)
+    return env
+  return {k: v for k, v in env.iteritems() if k in whitelist}
 
-  for k in os.environ.keys():
-    if k not in whitelist:
-      del os.environ[k]
+
+class Runtime(object):
+  """Container for instance-global state."""
+
+  def __init__(self, properties, platform=None):
+    self._properties = properties
+    self._platform = platform if platform else util.Platform.probe()
+
+  @property
+  def properties(self):
+    """Returns (dict): The input properties."""
+    return self._properties
+
+  @property
+  def platform(self):
+    """Returns (util.Platform): The current running Platform."""
+    return self._platform
 
 
 class RecipeEngine(object):
   """
   Knows how to execute steps emitted by a recipe, holds global state such as
   step history and build properties. Each recipe module API has a reference to
   this object.
-
-  Recipe modules that are aware of the engine:
-    * properties - uses engine.properties.
-    * step - uses engine.create_step(...), and previous_step_result.
   """
 
   ActiveStep = collections.namedtuple('ActiveStep', (
       'config', 'step_result', 'open_step'))
 
-  def __init__(self, step_runner, properties, universe_view):
+  def __init__(self, step_runner, rt, universe_view):
     """See run_steps() for parameter meanings."""
     self._step_runner = step_runner
-    self._properties = properties
+    self._rt = rt
     self._universe_view = universe_view
     self._clients = {client.IDENT: client for client in (
         recipe_api.StepClient(self),
-        recipe_api.PropertiesClient(self),
+        recipe_api.PropertiesClient(self._rt.properties),
+        recipe_api.PlatformClient(self._rt.platform),
         recipe_api.DependencyManagerClient(self),
     )}
 
     # A stack of ActiveStep objects, holding the most recently executed step at
     # each nest level (objects deeper in the stack have lower nest levels).
     # When we pop from this stack, we close the corresponding step stream.
     self._step_stack = []
 
     # TODO(iannucci): come up with a more structured way to advertise/set mode

[iannucci, 2016/10/15 00:39:45]

I think this isn't used any more. you can delete this whole section. I think the
proto arguments completely supercede this.

[dnj, 2016/10/15 00:59:27]

Done.

     # flags/options for the engine.
-    if '$recipe_engine' in properties:
-      options = properties['$recipe_engine']
+    options = self._rt.properties.get('$recipe_engine')
+    if options is not None:
       try:
         mode_flags = options.get('mode_flags')
         if mode_flags:
           if mode_flags.get('use_subprocess42'):
             print "IGNORING MODE_SUBPROCESS42"
       except Exception as e:
         print "Failed to set recipe_engine options, got: %r: %s" % (options, e)
 
   @property
   def properties(self):
-    return self._properties
+    return self._rt.properties
 
   @property
   def universe(self):
     return self._universe_view.universe
 
   def _close_through_level(self, level):
     """Close all open steps whose nest level is >= the supplied level.
 
     Args:
       level (int): the nest level to close through.
@@ skipping 143 matching lines @@
         results.append(
           loader._invoke_with_properties(
             run_recipe, properties, recipe_script.PROPERTIES,
             properties.keys()))
       except TypeError as e:
         raise TypeError(
             "Got %r while trying to call recipe %s with properties %r" % (
               e, recipe, properties))
 
     return results