Add simple depends_on API.

recipe_engine/run.py

 # Copyright (c) 2013-2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Entry point for fully-annotated builds.
 
 This script is part of the effort to move all builds to annotator-based
 systems. Any builder configured to use the AnnotatorFactory.BaseFactory()
 found in scripts/master/factory/annotator_factory.py executes a single
 AddAnnotatedScript step. That step (found in annotator_commands.py) calls
@@ skipping 54 matching lines @@
 
 import collections
 import contextlib
 import copy
 import functools
 import json
 import os
 import re
 import subprocess
 import sys
+import tempfile
 import threading
 import traceback
 
 import cStringIO
 
 
 from . import loader
 from . import recipe_api
 from . import recipe_test_api
+from . import types
 from . import util
 
 
 SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
 
 BUILDBOT_MAGIC_ENV = set([
     'BUILDBOT_BLAMELIST',
     'BUILDBOT_BRANCH',
     'BUILDBOT_BUILDBOTURL',
     'BUILDBOT_BUILDERNAME',
@@ skipping 332 matching lines @@
   properties.pop('root', None)
 
   # TODO(iannucci): A much better way to do this would be to dynamically
   #   detect if the mirrors are actually available during the execution of the
   #   recipe.
   if ('use_mirror' not in properties and (
     'TESTING_MASTERNAME' in os.environ or
     'TESTING_SLAVENAME' in os.environ)):
     properties['use_mirror'] = False
 
-  engine = RecipeEngine(stream, properties, test_data)
+  engine = RecipeEngine(stream, properties, test_data, universe)
 
   # Create all API modules and top level RunSteps function.  It doesn't launch
   # any recipe code yet; RunSteps needs to be called.
   api = None
   with stream.step('setup_build') as s:
     assert 'recipe' in properties
     recipe = properties['recipe']
 
     properties_to_print = properties.copy()
     if 'use_mirror' in properties:
@@ skipping 296 matching lines @@
   Knows how to execute steps emitted by a recipe, holds global state such as
   step history and build properties. Each recipe module API has a reference to
   this object.
 
   Recipe modules that are aware of the engine:
     * properties - uses engine.properties.
     * step_history - uses engine.step_history.
     * step - uses engine.create_step(...).
 
   """
-  def __init__(self, stream, properties, test_data):
+  def __init__(self, stream, properties, test_data, universe):
     self._stream = stream
     self._properties = properties
     self._test_data = test_data
     self._step_history = collections.OrderedDict()
+    self._universe = universe
 
     self._previous_step_annotation = None
     self._previous_step_result = None
     self._api = None
 
   @property
   def properties(self):
     return self._properties
 
   @property
@@ skipping 86 matching lines @@
         exc = recipe_api.InfraFailure
 
       step_result.presentation.status = state
       if step_test.enabled:
         # To avoid cluttering the expectations, don't emit this in testmode.
         self._previous_step_annotation.emit(
             'step returned non-zero exit code: %d' % step_result.retcode)
 
       raise exc(step['name'], step_result)
 
-
   def run(self, recipe_script, api):
     """Run a recipe represented by a recipe_script object.
 
     This function blocks until recipe finishes.
+    It mainly executes the recipe, and has some exception handling logic, and
+    adds the step history to the result.
 
     Args:
       recipe_script: The recipe to run, as represented by a RecipeScript object.
       api: The api, with loaded module dependencies.
            Used by the some special modules.
 
     Returns:
       RecipeExecutionResult which has status code, list of steps ran,
         and return value.
     """
@@ skipping 54 matching lines @@
 
     Args:
       step: ConfigGroup object with information about the step, see
         recipe_modules/step/config.py.
 
     Returns:
       Opaque engine specific object that is understood by 'run_steps' method.
     """
     return step.as_jsonish()
 
+  def depend_on(self, recipe, properties, distributor=None):
+    return self.depend_on_multi(
+        ((recipe, properties),), distributor=distributor)[0]
 
+  def depend_on_multi(self, dependencies, distributor=None):
+    results = []
+    for recipe, properties in dependencies:
+      recipe_script = self._universe.load_recipe(recipe)

[iannucci, 2015/11/20 02:00:19]

probably also something for the 'advanced' API, but not all dependencies will
happen within the same version of the same toolset. For example, when bisecting,
we'll want to chose the toolset that corresponds with the current version of the
code that we're operating on.

I think in this case we'd require the depender to supply the swarming dimensions
instead of calculating them automatically, too.

[martiniss, 2015/11/23 22:05:07]

True. Will require additional thinking. But we can just change the universe we
load from. Might also stick this inside of the distributor code, wherever that
lives.

+      # run_recipe is a function which will be called once the properties have
+      # been validated by the recipe engine. The arguments being passed in are
+      # simply the values being passed to the recipe, which we already know, so
+      # we ignore them. We're only using this for its properties validation
+      # functionality.
+      run_recipe = lambda *args, **kwargs: None
+
+      if self._test_data.enabled:
+        run_recipe = lambda *args, **kwargs: (
+            self._test_data.depend_on_data[types.freeze((recipe, properties),)])
+      else:
+        # TODO(martiniss) Add DM integration
+        # For now, we just run the recipe locally.

[iannucci, 2015/11/20 02:00:19]

assert distributor == None?

[martiniss, 2015/11/23 22:05:07]

Good idea.

+        def run_recipe(*args, **kwargs):
+          with tempfile.NamedTemporaryFile() as f:
+            cmd = [sys.executable,
+                   self._universe.package_deps.engine_recipes_py,
+                   '--package=%s' % self._universe.config_file, 'run',
+                   '--output-result-json=%s' % f.name, recipe]
+            cmd.extend(['%s=%s' % (k, v) for k, v in properties.iteritems()])
+
+            retcode = subprocess.call(cmd)
+            result = json.load(f)
+            if retcode != 0:
+              raise recipe_api.StepFailure(
+                'depend on %s with properties %r failed with %d.'
+                'Recipe result: %r' % (
+                    recipe, properties, retcode, result))
+            return result
+
+      try:
+        # This does type checking for properties

[iannucci, 2015/11/20 02:00:19]

presumably distributor calculation would happen here?

[martiniss, 2015/11/23 22:05:07]

What do you mean calculation? 

I believe the distributor would just define a recipe_run essentially. So maybe
return a function or something. Which I would then pass in here, which would get
called after the properties being passed to the recipe are validated by the
engine.

+        results.append(loader._invoke_with_properties(
+            run_recipe, properties, recipe_script.PROPERTIES, properties.keys()))
+      except TypeError as e:
+        raise TypeError(
+            "Got %r while trying to call recipe %s with properties %r" % (
+              e, recipe, properties))
+
+    return results
+
+
+