OLD | NEW |
1 # Copyright 2016 The LUCI Authors. All rights reserved. | 1 # Copyright 2016 The LUCI Authors. All rights reserved. |
2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
4 | 4 |
5 """Entry point for fully-annotated builds. | 5 """Entry point for fully-annotated builds. |
6 | 6 |
7 This script is part of the effort to move all builds to annotator-based | 7 This script is part of the effort to move all builds to annotator-based |
8 systems. Any builder configured to use the AnnotatorFactory.BaseFactory() | 8 systems. Any builder configured to use the AnnotatorFactory.BaseFactory() |
9 found in scripts/master/factory/annotator_factory.py executes a single | 9 found in scripts/master/factory/annotator_factory.py executes a single |
10 AddAnnotatedScript step. That step (found in annotator_commands.py) calls | 10 AddAnnotatedScript step. That step (found in annotator_commands.py) calls |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
57 the current history of what steps have run, what they returned, and any | 57 the current history of what steps have run, what they returned, and any |
58 json data they emitted. Additionally, the OrderedDict has the following | 58 json data they emitted. Additionally, the OrderedDict has the following |
59 convenience functions defined: | 59 convenience functions defined: |
60 * last_step - Returns the last step that ran or None | 60 * last_step - Returns the last step that ran or None |
61 * nth_step(n) - Returns the N'th step that ran or None | 61 * nth_step(n) - Returns the N'th step that ran or None |
62 | 62 |
63 'failed' is a boolean representing if the build is in a 'failed' state. | 63 'failed' is a boolean representing if the build is in a 'failed' state. |
64 """ | 64 """ |
65 | 65 |
66 import collections | 66 import collections |
| 67 import json |
67 import os | 68 import os |
68 import sys | 69 import sys |
69 import traceback | 70 import traceback |
70 | 71 |
71 from . import env | 72 from . import env |
72 | 73 |
73 from . import loader | 74 from . import loader |
74 from . import recipe_api | 75 from . import recipe_api |
75 from . import recipe_test_api | 76 from . import recipe_test_api |
76 from . import step_runner as step_runner_module | 77 from . import step_runner as step_runner_module |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
176 'USER', | 177 'USER', |
177 'USERNAME', | 178 'USERNAME', |
178 ]) | 179 ]) |
179 | 180 |
180 | 181 |
181 # Return value of run_steps and RecipeEngine.run. Just a container for the | 182 # Return value of run_steps and RecipeEngine.run. Just a container for the |
182 # literal return value of the recipe. | 183 # literal return value of the recipe. |
183 RecipeResult = collections.namedtuple('RecipeResult', 'result') | 184 RecipeResult = collections.namedtuple('RecipeResult', 'result') |
184 | 185 |
185 | 186 |
186 def run_steps(properties, stream_engine, step_runner, universe): | 187 def run_steps(properties, stream_engine, step_runner, universe_view): |
187 """Runs a recipe (given by the 'recipe' property). | 188 """Runs a recipe (given by the 'recipe' property). |
188 | 189 |
189 Args: | 190 Args: |
190 properties: a dictionary of properties to pass to the recipe. The | 191 properties: a dictionary of properties to pass to the recipe. The |
191 'recipe' property defines which recipe to actually run. | 192 'recipe' property defines which recipe to actually run. |
192 stream_engine: the StreamEngine to use to create individual step streams. | 193 stream_engine: the StreamEngine to use to create individual step streams. |
193 step_runner: The StepRunner to use to 'actually run' the steps. | 194 step_runner: The StepRunner to use to 'actually run' the steps. |
194 universe: The RecipeUniverse to use to load the recipes & modules. | 195 universe_view: The RecipeUniverse to use to load the recipes & modules. |
195 | 196 |
196 Returns: RecipeResult | 197 Returns: RecipeResult |
197 """ | 198 """ |
198 # NOTE(iannucci): 'root' was a terribly bad idea and has been replaced by | 199 # NOTE(iannucci): 'root' was a terribly bad idea and has been replaced by |
199 # 'patch_project'. 'root' had Rietveld knowing about the implementation of | 200 # 'patch_project'. 'root' had Rietveld knowing about the implementation of |
200 # the builders. 'patch_project' lets the builder (recipe) decide its own | 201 # the builders. 'patch_project' lets the builder (recipe) decide its own |
201 # destiny. | 202 # destiny. |
202 properties.pop('root', None) | 203 properties.pop('root', None) |
203 | 204 |
204 # TODO(iannucci): A much better way to do this would be to dynamically | 205 # TODO(iannucci): A much better way to do this would be to dynamically |
205 # detect if the mirrors are actually available during the execution of the | 206 # detect if the mirrors are actually available during the execution of the |
206 # recipe. | 207 # recipe. |
207 if ('use_mirror' not in properties and ( | 208 if ('use_mirror' not in properties and ( |
208 'TESTING_MASTERNAME' in os.environ or | 209 'TESTING_MASTERNAME' in os.environ or |
209 'TESTING_SLAVENAME' in os.environ)): | 210 'TESTING_SLAVENAME' in os.environ)): |
210 properties['use_mirror'] = False | 211 properties['use_mirror'] = False |
211 | 212 |
212 with stream_engine.new_step_stream('setup_build') as s: | 213 with stream_engine.new_step_stream('setup_build') as s: |
213 engine = RecipeEngine(step_runner, properties, universe) | 214 engine = RecipeEngine(step_runner, properties, universe_view) |
214 | 215 |
215 # Create all API modules and top level RunSteps function. It doesn't launch | 216 # Create all API modules and top level RunSteps function. It doesn't launch |
216 # any recipe code yet; RunSteps needs to be called. | 217 # any recipe code yet; RunSteps needs to be called. |
217 api = None | 218 api = None |
218 | 219 |
219 assert 'recipe' in properties | 220 assert 'recipe' in properties |
220 recipe = properties['recipe'] | 221 recipe = properties['recipe'] |
221 | 222 |
222 properties_to_print = properties.copy() | 223 properties_to_print = properties.copy() |
223 if 'use_mirror' in properties: | 224 if 'use_mirror' in properties: |
224 del properties_to_print['use_mirror'] | 225 del properties_to_print['use_mirror'] |
225 | 226 |
| 227 root_package = universe_view.universe.package_deps.root_package |
226 run_recipe_help_lines = [ | 228 run_recipe_help_lines = [ |
227 'To repro this locally, run the following line from a build checkout:', | 229 'To repro this locally, run the following line from the root of a %r' |
| 230 ' checkout:' % (root_package.name), |
228 '', | 231 '', |
229 './scripts/tools/run_recipe.py %s --properties-file - <<EOF' % recipe, | 232 '%s run --properties-file - %s <<EOF' % ( |
230 repr(properties_to_print), | 233 os.path.join( '.', root_package.relative_recipes_dir, 'recipes.py'), |
| 234 recipe), |
| 235 '%s' % json.dumps(properties_to_print), |
231 'EOF', | 236 'EOF', |
232 '', | 237 '', |
233 'To run on Windows, you can put the JSON in a file and redirect the', | 238 'To run on Windows, you can put the JSON in a file and redirect the', |
234 'contents of the file into run_recipe.py, with the < operator.', | 239 'contents of the file into run_recipe.py, with the < operator.', |
235 ] | 240 ] |
236 | 241 |
237 with s.new_log_stream('run_recipe') as l: | 242 with s.new_log_stream('run_recipe') as l: |
238 for line in run_recipe_help_lines: | 243 for line in run_recipe_help_lines: |
239 l.write_line(line) | 244 l.write_line(line) |
240 | 245 |
241 _isolate_environment() | 246 _isolate_environment() |
242 | 247 |
243 # Find and load the recipe to run. | 248 # Find and load the recipe to run. |
244 try: | 249 try: |
245 recipe_script = universe.load_recipe(recipe) | 250 recipe_script = universe_view.load_recipe(recipe) |
246 s.write_line('Running recipe with %s' % (properties,)) | 251 s.write_line('Running recipe with %s' % (properties,)) |
247 | 252 |
248 api = loader.create_recipe_api(recipe_script.LOADED_DEPS, | 253 api = loader.create_recipe_api(recipe_script.LOADED_DEPS, |
249 engine, | 254 engine, |
250 recipe_test_api.DisabledTestData()) | 255 recipe_test_api.DisabledTestData()) |
251 | 256 |
252 s.add_step_text('running recipe: "%s"' % recipe) | 257 s.add_step_text('running recipe: "%s"' % recipe) |
253 except (loader.LoaderError, ImportError, AssertionError) as e: | 258 except (loader.LoaderError, ImportError, AssertionError) as e: |
254 for line in str(e).splitlines(): | 259 for line in str(e).splitlines(): |
255 s.add_step_text(line) | 260 s.add_step_text(line) |
(...skipping 240 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
496 results.append( | 501 results.append( |
497 loader._invoke_with_properties( | 502 loader._invoke_with_properties( |
498 run_recipe, properties, recipe_script.PROPERTIES, | 503 run_recipe, properties, recipe_script.PROPERTIES, |
499 properties.keys())) | 504 properties.keys())) |
500 except TypeError as e: | 505 except TypeError as e: |
501 raise TypeError( | 506 raise TypeError( |
502 "Got %r while trying to call recipe %s with properties %r" % ( | 507 "Got %r while trying to call recipe %s with properties %r" % ( |
503 e, recipe, properties)) | 508 e, recipe, properties)) |
504 | 509 |
505 return results | 510 return results |
OLD | NEW |