Add simple depends_on API.

recipe_engine/run.py

 # Copyright (c) 2013-2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Entry point for fully-annotated builds.
 
 This script is part of the effort to move all builds to annotator-based
 systems. Any builder configured to use the AnnotatorFactory.BaseFactory()
 found in scripts/master/factory/annotator_factory.py executes a single
 AddAnnotatedScript step. That step (found in annotator_commands.py) calls
@@ skipping 54 matching lines @@
 
 import collections
 import contextlib
 import copy
 import functools
 import json
 import os
 import re
 import subprocess
 import sys
+import tempfile
 import threading
 import traceback
 
 import cStringIO
 
 
 from . import loader
 from . import recipe_api
 from . import recipe_test_api
+from . import types
 from . import util
 
 
 SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
 
 BUILDBOT_MAGIC_ENV = set([
     'BUILDBOT_BLAMELIST',
     'BUILDBOT_BRANCH',
     'BUILDBOT_BUILDBOTURL',
     'BUILDBOT_BUILDERNAME',
@@ skipping 332 matching lines @@
   properties.pop('root', None)
 
   # TODO(iannucci): A much better way to do this would be to dynamically
   #   detect if the mirrors are actually available during the execution of the
   #   recipe.
   if ('use_mirror' not in properties and (
     'TESTING_MASTERNAME' in os.environ or
     'TESTING_SLAVENAME' in os.environ)):
     properties['use_mirror'] = False
 
-  engine = RecipeEngine(stream, properties, test_data)
+  engine = RecipeEngine(stream, properties, test_data, universe)
 
   # Create all API modules and top level RunSteps function.  It doesn't launch
   # any recipe code yet; RunSteps needs to be called.
   api = None
   with stream.step('setup_build') as s:
     assert 'recipe' in properties
     recipe = properties['recipe']
 
     properties_to_print = properties.copy()
     if 'use_mirror' in properties:
@@ skipping 296 matching lines @@
   Knows how to execute steps emitted by a recipe, holds global state such as
   step history and build properties. Each recipe module API has a reference to
   this object.
 
   Recipe modules that are aware of the engine:
     * properties - uses engine.properties.
     * step_history - uses engine.step_history.
     * step - uses engine.create_step(...).
 
   """
-  def __init__(self, stream, properties, test_data):
+  def __init__(self, stream, properties, test_data, universe):
     self._stream = stream
     self._properties = properties
     self._test_data = test_data
     self._step_history = collections.OrderedDict()
+    self._universe = universe
 
     self._previous_step_annotation = None
     self._previous_step_result = None
     self._api = None
 
+  def clone(self, stream=None, properties=None, universe=None):
+    return RecipeEngine(stream or self._stream, properties or self._properties,
+                        self._test_data, universe or self._universe)

[luqui, 2015/11/16 20:48:00]

Hmm, isn't test_data mutable, in order for example for the 'unconsumed test
data' check?

[martiniss, 2015/11/18 01:00:00]

Deleted this method. Not needed anymore.

+
   @property
   def properties(self):
     return self._properties
 
   @property
   def previous_step_result(self):
     """Allows api.step to get the active result from any context."""
     return self._previous_step_result
 
   def _emit_results(self):
@@ skipping 81 matching lines @@
         exc = recipe_api.InfraFailure
 
       step_result.presentation.status = state
       if step_test.enabled:
         # To avoid cluttering the expectations, don't emit this in testmode.
         self._previous_step_annotation.emit(
             'step returned non-zero exit code: %d' % step_result.retcode)
 
       raise exc(step['name'], step_result)
 
-
   def run(self, recipe_script, api):
     """Run a recipe represented by a recipe_script object.
 
     This function blocks until recipe finishes.
+    It mainly executes the recipe, and has some exception handling logic, and
+    adds the step history to the result.
 
     Args:
       recipe_script: The recipe to run, as represented by a RecipeScript object.
       api: The api, with loaded module dependencies.
            Used by the some special modules.
 
     Returns:
       RecipeExecutionResult which has status code, list of steps ran,
         and return value.
     """
@@ skipping 54 matching lines @@
 
     Args:
       step: ConfigGroup object with information about the step, see
         recipe_modules/step/config.py.
 
     Returns:
       Opaque engine specific object that is understood by 'run_steps' method.
     """
     return step.as_jsonish()
 
+  def depend_on(self, recipe, properties, distributor=None):
+    return self.depend_on_multi(
+        ((recipe, properties),), distributor=distributor)
 
+  def depend_on_multi(self, dependencies, distributor=None):
+    if self._test_data.enabled:
+      # TODO(martiniss) test properties against the recipe
+      for recipe, properties in dependencies:
+        recipe_script = self._universe.load_recipe(recipe)

[luqui, 2015/11/16 20:48:00]

This precludes one use case, which is that the recipe we depend on was checked
out by a previous step in the recipe, which will requite a more lenient test
mechanism.  This is well designed though, so we can cross that bridge when we
come to it.

+        loader._invoke_with_properties(
+            lambda *args, **kwargs: None, properties, recipe_script.PROPERTIES,
+            properties.keys())
+
+        return self._test_data.depend_on_data[types.freeze((recipe, properties),)]

[luqui, 2015/11/16 20:48:00]

line wrap

+
+    # TODO(martiniss) Add DM integration
+    for recipe, properties in dependencies:
+      recipes_py_loc = os.path.join(
+          self._universe.package_deps._context.repo_root, 'recipes.py')
+
+      with tempfile.NamedTemporaryFile() as f:
+        cmd = [sys.executable, recipes_py_loc,
+               '--package=%s' % self._universe._config_file, 'run',
+               '--result-file=%s' % f.name, recipe]

[luqui, 2015/11/16 20:48:00]

The convention for this argument has been --output-json AFAICT.   I did like
"result" there, though.  Maybe --output-result-json

[martiniss, 2015/11/18 01:00:00]

Fixed.

+        cmd.extend(['%s=%s' % (k, v) for k, v in properties.iteritems()])
+
+        rval = subprocess.call(cmd)

[estaab, 2015/11/17 23:07:16]

We should probably throw an exception when rval isn't 0 since we control the
process running the recipe and should always expect it to be 0, right?

[martiniss, 2015/11/18 01:00:00]

Good idea. Raising an InfraFailure.

+        return json.load(f)
+
+
+