| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright 2013-2015 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 import contextlib | |
| 6 import keyword | |
| 7 import types | |
| 8 | |
| 9 from functools import wraps | |
| 10 | |
| 11 from .recipe_test_api import DisabledTestData, ModuleTestData | |
| 12 from .config import Single | |
| 13 | |
| 14 from .util import ModuleInjectionSite | |
| 15 | |
| 16 from . import field_composer | |
| 17 | |
| 18 | |
| 19 class StepFailure(Exception): | |
| 20 """ | |
| 21 This is the base class for all step failures. | |
| 22 | |
| 23 Raising a StepFailure counts as 'running a step' for the purpose of | |
| 24 infer_composite_step's logic. | |
| 25 """ | |
| 26 def __init__(self, name_or_reason, result=None): | |
| 27 _STEP_CONTEXT['ran_step'][0] = True | |
| 28 if result: | |
| 29 self.name = name_or_reason | |
| 30 self.result = result | |
| 31 self.reason = self.reason_message() | |
| 32 else: | |
| 33 self.name = None | |
| 34 self.result = None | |
| 35 self.reason = name_or_reason | |
| 36 | |
| 37 super(StepFailure, self).__init__(self.reason) | |
| 38 | |
| 39 def reason_message(self): | |
| 40 return "Step({!r}) failed with return_code {}".format( | |
| 41 self.name, self.result.retcode) | |
| 42 | |
| 43 def __str__(self): # pragma: no cover | |
| 44 return "Step Failure in %s" % self.name | |
| 45 | |
| 46 @property | |
| 47 def retcode(self): | |
| 48 """ | |
| 49 Returns the retcode of the step which failed. If this was a manual | |
| 50 failure, returns None | |
| 51 """ | |
| 52 if not self.result: | |
| 53 return None | |
| 54 return self.result.retcode | |
| 55 | |
| 56 | |
| 57 class StepWarning(StepFailure): | |
| 58 """ | |
| 59 A subclass of StepFailure, which still fails the build, but which is | |
| 60 a warning. Need to figure out how exactly this will be useful. | |
| 61 """ | |
| 62 def reason_message(self): # pragma: no cover | |
| 63 return "Warning: Step({!r}) returned {}".format( | |
| 64 self.name, self.result.retcode) | |
| 65 | |
| 66 def __str__(self): # pragma: no cover | |
| 67 return "Step Warning in %s" % self.name | |
| 68 | |
| 69 | |
| 70 class InfraFailure(StepFailure): | |
| 71 """ | |
| 72 A subclass of StepFailure, which fails the build due to problems with the | |
| 73 infrastructure. | |
| 74 """ | |
| 75 def reason_message(self): | |
| 76 return "Infra Failure: Step({!r}) returned {}".format( | |
| 77 self.name, self.result.retcode) | |
| 78 | |
| 79 def __str__(self): | |
| 80 return "Infra Failure in %s" % self.name | |
| 81 | |
| 82 | |
| 83 class AggregatedStepFailure(StepFailure): | |
| 84 def __init__(self, result): | |
| 85 super(AggregatedStepFailure, self).__init__( | |
| 86 "Aggregate step failure.", result=result) | |
| 87 | |
| 88 def reason_message(self): | |
| 89 msg = "{!r} out of {!r} aggregated steps failed. Failures: ".format( | |
| 90 len(self.result.failures), len(self.result.all_results)) | |
| 91 msg += ', '.join((f.reason or f.name) for f in self.result.failures) | |
| 92 return msg | |
| 93 | |
| 94 def __str__(self): # pragma: no cover | |
| 95 return "Aggregate Step Failure" | |
| 96 | |
| 97 | |
| 98 _FUNCTION_REGISTRY = { | |
| 99 'aggregated_result': {'combine': lambda a, b: b}, | |
| 100 'env': {'combine': lambda a, b: dict(a, **b)}, | |
| 101 'name': {'combine': lambda a, b: '%s.%s' % (a, b)}, | |
| 102 'nest_level': {'combine': lambda a, b: a + b}, | |
| 103 'ran_step': {'combine': lambda a, b: b}, | |
| 104 } | |
| 105 | |
| 106 | |
| 107 class AggregatedResult(object): | |
| 108 """Holds the result of an aggregated run of steps. | |
| 109 | |
| 110 Currently this is only used internally by defer_results, but it may be exposed | |
| 111 to the consumer of defer_results at some point in the future. For now it's | |
| 112 expected to be easier for defer_results consumers to do their own result | |
| 113 aggregation, as they may need to pick and chose (or label) which results they | |
| 114 really care about. | |
| 115 """ | |
| 116 def __init__(self): | |
| 117 self.successes = [] | |
| 118 self.failures = [] | |
| 119 | |
| 120 # Needs to be here to be able to treat this as a step result | |
| 121 self.retcode = None | |
| 122 | |
| 123 @property | |
| 124 def all_results(self): | |
| 125 """ | |
| 126 Return a list of two item tuples (x, y), where | |
| 127 x is whether or not the step succeeded, and | |
| 128 y is the result of the run | |
| 129 """ | |
| 130 res = [(True, result) for result in self.successes] | |
| 131 res.extend([(False, result) for result in self.failures]) | |
| 132 return res | |
| 133 | |
| 134 def add_success(self, result): | |
| 135 self.successes.append(result) | |
| 136 | |
| 137 def add_failure(self, exception): | |
| 138 self.failures.append(exception) | |
| 139 | |
| 140 | |
| 141 class DeferredResult(object): | |
| 142 def __init__(self, result, failure): | |
| 143 self._result = result | |
| 144 self._failure = failure | |
| 145 | |
| 146 @property | |
| 147 def is_ok(self): | |
| 148 return self._failure is None | |
| 149 | |
| 150 def get_result(self): | |
| 151 if not self.is_ok: | |
| 152 raise self.get_error() | |
| 153 return self._result | |
| 154 | |
| 155 def get_error(self): | |
| 156 assert self._failure, "WHAT IS IT ARE YOU DOING???!?!?!? SHTAP NAO" | |
| 157 return self._failure | |
| 158 | |
| 159 | |
| 160 _STEP_CONTEXT = field_composer.FieldComposer( | |
| 161 {'ran_step': [False]}, _FUNCTION_REGISTRY) | |
| 162 | |
| 163 | |
| 164 def non_step(func): | |
| 165 """A decorator which prevents a method from automatically being wrapped as | |
| 166 a infer_composite_step by RecipeApiMeta. | |
| 167 | |
| 168 This is needed for utility methods which don't run any steps, but which are | |
| 169 invoked within the context of a defer_results(). | |
| 170 | |
| 171 @see infer_composite_step, defer_results, RecipeApiMeta | |
| 172 """ | |
| 173 assert not hasattr(func, "_skip_inference"), \ | |
| 174 "Double-wrapped method %r?" % func | |
| 175 func._skip_inference = True # pylint: disable=protected-access | |
| 176 return func | |
| 177 | |
| 178 _skip_inference = non_step | |
| 179 | |
| 180 | |
| 181 @contextlib.contextmanager | |
| 182 def context(fields): | |
| 183 global _STEP_CONTEXT | |
| 184 old = _STEP_CONTEXT | |
| 185 try: | |
| 186 _STEP_CONTEXT = old.compose(fields) | |
| 187 yield | |
| 188 finally: | |
| 189 _STEP_CONTEXT = old | |
| 190 | |
| 191 | |
| 192 def infer_composite_step(func): | |
| 193 """A decorator which possibly makes this step act as a single step, for the | |
| 194 purposes of the defer_results function. | |
| 195 | |
| 196 Behaves as if this function were wrapped by composite_step, unless this | |
| 197 function: | |
| 198 * is already wrapped by non_step | |
| 199 * returns a result without calling api.step | |
| 200 * raises an exception which is not derived from StepFailure | |
| 201 | |
| 202 In any of these cases, this function will behave like a normal function. | |
| 203 | |
| 204 This decorator is automatically applied by RecipeApiMeta (or by inheriting | |
| 205 from RecipeApi). If you want to decalare a method's behavior explicitly, you | |
| 206 may decorate it with either composite_step or with non_step. | |
| 207 """ | |
| 208 if getattr(func, "_skip_inference", False): | |
| 209 return func | |
| 210 | |
| 211 @_skip_inference # to prevent double-wraps | |
| 212 @wraps(func) | |
| 213 def _inner(*a, **kw): | |
| 214 # We're not in a defer_results context, so just run the function normally. | |
| 215 if _STEP_CONTEXT.get('aggregated_result') is None: | |
| 216 return func(*a, **kw) | |
| 217 | |
| 218 agg = _STEP_CONTEXT['aggregated_result'] | |
| 219 | |
| 220 # Setting the aggregated_result to None allows the contents of func to be | |
| 221 # written in the same style (e.g. with exceptions) no matter how func is | |
| 222 # being called. | |
| 223 with context({'aggregated_result': None, 'ran_step': [False]}): | |
| 224 try: | |
| 225 ret = func(*a, **kw) | |
| 226 if not _STEP_CONTEXT.get('ran_step', [False])[0]: | |
| 227 return ret | |
| 228 agg.add_success(ret) | |
| 229 return DeferredResult(ret, None) | |
| 230 except StepFailure as ex: | |
| 231 agg.add_failure(ex) | |
| 232 return DeferredResult(None, ex) | |
| 233 return _inner | |
| 234 | |
| 235 | |
| 236 def composite_step(func): | |
| 237 """A decorator which makes this step act as a single step, for the purposes of | |
| 238 the defer_results function. | |
| 239 | |
| 240 This means that this function will not quit during the middle of its execution | |
| 241 because of a StepFailure, if there is an aggregator active. | |
| 242 | |
| 243 You may use this decorator explicitly if infer_composite_step is detecting | |
| 244 the behavior of your method incorrectly to force it to behave as a step. You | |
| 245 may also need to use this if your Api class inherits from RecipeApiPlain and | |
| 246 so doesn't have its methods automatically wrapped by infer_composite_step. | |
| 247 """ | |
| 248 @_skip_inference # to avoid double-wraps | |
| 249 @wraps(func) | |
| 250 def _inner(*a, **kw): | |
| 251 # always counts as running a step | |
| 252 _STEP_CONTEXT['ran_step'][0] = True | |
| 253 | |
| 254 if _STEP_CONTEXT.get('aggregated_result') is None: | |
| 255 return func(*a, **kw) | |
| 256 | |
| 257 agg = _STEP_CONTEXT['aggregated_result'] | |
| 258 | |
| 259 # Setting the aggregated_result to None allows the contents of func to be | |
| 260 # written in the same style (e.g. with exceptions) no matter how func is | |
| 261 # being called. | |
| 262 with context({'aggregated_result': None}): | |
| 263 try: | |
| 264 ret = func(*a, **kw) | |
| 265 agg.add_success(ret) | |
| 266 return DeferredResult(ret, None) | |
| 267 except StepFailure as ex: | |
| 268 agg.add_failure(ex) | |
| 269 return DeferredResult(None, ex) | |
| 270 return _inner | |
| 271 | |
| 272 | |
| 273 @contextlib.contextmanager | |
| 274 def defer_results(): | |
| 275 """ | |
| 276 Use this to defer step results in your code. All steps which would previously | |
| 277 return a result or throw an exception will instead return a DeferredResult. | |
| 278 | |
| 279 Any exceptions which were thrown during execution will be thrown when either: | |
| 280 a. You call get_result() on the step's result. | |
| 281 b. You exit the suite inside of the with statement | |
| 282 | |
| 283 Example: | |
| 284 with defer_results(): | |
| 285 api.step('a', ..) | |
| 286 api.step('b', ..) | |
| 287 result = api.m.module.im_a_composite_step(...) | |
| 288 api.m.echo('the data is', result.get_result()) | |
| 289 | |
| 290 If 'a' fails, 'b' and 'im a composite step' will still run. | |
| 291 If 'im a composite step' fails, then the get_result() call will raise | |
| 292 an exception. | |
| 293 If you don't try to use the result (don't call get_result()), an aggregate | |
| 294 failure will still be raised once you exit the suite inside | |
| 295 the with statement. | |
| 296 """ | |
| 297 assert _STEP_CONTEXT.get('aggregated_result') is None, ( | |
| 298 "may not call defer_results in an active defer_results context") | |
| 299 agg = AggregatedResult() | |
| 300 with context({'aggregated_result': agg}): | |
| 301 yield | |
| 302 if agg.failures: | |
| 303 raise AggregatedStepFailure(agg) | |
| 304 | |
| 305 | |
| 306 class RecipeApiMeta(type): | |
| 307 WHITELIST = ('__init__',) | |
| 308 def __new__(mcs, name, bases, attrs): | |
| 309 """Automatically wraps all methods of subclasses of RecipeApi with | |
| 310 @infer_composite_step. This allows defer_results to work as intended without | |
| 311 manually decorating every method. | |
| 312 """ | |
| 313 wrap = lambda f: infer_composite_step(f) if f else f | |
| 314 for attr in attrs: | |
| 315 if attr in RecipeApiMeta.WHITELIST: | |
| 316 continue | |
| 317 val = attrs[attr] | |
| 318 if isinstance(val, types.FunctionType): | |
| 319 attrs[attr] = wrap(val) | |
| 320 elif isinstance(val, property): | |
| 321 attrs[attr] = property( | |
| 322 wrap(val.fget), | |
| 323 wrap(val.fset), | |
| 324 wrap(val.fdel), | |
| 325 val.__doc__) | |
| 326 return super(RecipeApiMeta, mcs).__new__(mcs, name, bases, attrs) | |
| 327 | |
| 328 | |
| 329 class RecipeApiPlain(ModuleInjectionSite): | |
| 330 """ | |
| 331 Framework class for handling recipe_modules. | |
| 332 | |
| 333 Inherit from this in your recipe_modules/<name>/api.py . This class provides | |
| 334 wiring for your config context (in self.c and methods, and for dependency | |
| 335 injection (in self.m). | |
| 336 | |
| 337 Dependency injection takes place in load_recipe_modules() in recipe_loader.py. | |
| 338 | |
| 339 USE RecipeApi INSTEAD, UNLESS your RecipeApi subclass derives from something | |
| 340 which defines its own __metaclass__. Deriving from RecipeApi instead of | |
| 341 RecipeApiPlain allows your RecipeApi subclass to automatically work with | |
| 342 defer_results without needing to decorate every methods with | |
| 343 @infer_composite_step. | |
| 344 """ | |
| 345 | |
| 346 def __init__(self, module=None, engine=None, | |
| 347 test_data=DisabledTestData(), **_kwargs): | |
| 348 """Note: Injected dependencies are NOT available in __init__().""" | |
| 349 super(RecipeApiPlain, self).__init__() | |
| 350 | |
| 351 # |engine| is an instance of annotated_run.RecipeEngine. Modules should not | |
| 352 # generally use it unless they're low-level framework level modules. | |
| 353 self._engine = engine | |
| 354 self._module = module | |
| 355 | |
| 356 assert isinstance(test_data, (ModuleTestData, DisabledTestData)) | |
| 357 self._test_data = test_data | |
| 358 | |
| 359 # If we're the 'root' api, inject directly into 'self'. | |
| 360 # Otherwise inject into 'self.m' | |
| 361 self.m = self if module is None else ModuleInjectionSite(self) | |
| 362 | |
| 363 # If our module has a test api, it gets injected here. | |
| 364 self.test_api = None | |
| 365 | |
| 366 # Config goes here. | |
| 367 self.c = None | |
| 368 | |
| 369 def get_config_defaults(self): # pylint: disable=R0201 | |
| 370 """ | |
| 371 Allows your api to dynamically determine static default values for configs. | |
| 372 """ | |
| 373 return {} | |
| 374 | |
| 375 def make_config(self, config_name=None, optional=False, **CONFIG_VARS): | |
| 376 """Returns a 'config blob' for the current API.""" | |
| 377 return self.make_config_params(config_name, optional, **CONFIG_VARS)[0] | |
| 378 | |
| 379 def make_config_params(self, config_name, optional=False, **CONFIG_VARS): | |
| 380 """Returns a 'config blob' for the current API, and the computed params | |
| 381 for all dependent configurations. | |
| 382 | |
| 383 The params have the following order of precendence. Each subsequent param | |
| 384 is dict.update'd into the final parameters, so the order is from lowest to | |
| 385 higest precedence on a per-key basis: | |
| 386 * if config_name in CONFIG_CTX | |
| 387 * get_config_defaults() | |
| 388 * CONFIG_CTX[config_name].DEFAULT_CONFIG_VARS() | |
| 389 * CONFIG_VARS | |
| 390 * else | |
| 391 * get_config_defaults() | |
| 392 * CONFIG_VARS | |
| 393 """ | |
| 394 generic_params = self.get_config_defaults() # generic defaults | |
| 395 generic_params.update(CONFIG_VARS) # per-invocation values | |
| 396 | |
| 397 ctx = self._module.CONFIG_CTX | |
| 398 if optional and not ctx: | |
| 399 return None, generic_params | |
| 400 | |
| 401 assert ctx, '%s has no config context' % self | |
| 402 try: | |
| 403 params = self.get_config_defaults() # generic defaults | |
| 404 itm = ctx.CONFIG_ITEMS[config_name] if config_name else None | |
| 405 if itm: | |
| 406 params.update(itm.DEFAULT_CONFIG_VARS()) # per-item defaults | |
| 407 params.update(CONFIG_VARS) # per-invocation values | |
| 408 | |
| 409 base = ctx.CONFIG_SCHEMA(**params) | |
| 410 if config_name is None: | |
| 411 return base, params | |
| 412 else: | |
| 413 return itm(base), params | |
| 414 except KeyError: | |
| 415 if optional: | |
| 416 return None, generic_params | |
| 417 else: # pragma: no cover | |
| 418 raise # TODO(iannucci): raise a better exception. | |
| 419 | |
| 420 def set_config(self, config_name=None, optional=False, **CONFIG_VARS): | |
| 421 """Sets the modules and its dependencies to the named configuration.""" | |
| 422 assert self._module | |
| 423 config, params = self.make_config_params(config_name, optional, | |
| 424 **CONFIG_VARS) | |
| 425 if config: | |
| 426 self.c = config | |
| 427 | |
| 428 def apply_config(self, config_name, config_object=None, optional=False): | |
| 429 """Apply a named configuration to the provided config object or self.""" | |
| 430 self._module.CONFIG_CTX.CONFIG_ITEMS[config_name]( | |
| 431 config_object or self.c, optional=optional) | |
| 432 | |
| 433 def resource(self, *path): | |
| 434 """Returns path to a file under <recipe module>/resources/ directory. | |
| 435 | |
| 436 Args: | |
| 437 path: path relative to module's resources/ directory. | |
| 438 """ | |
| 439 # TODO(vadimsh): Verify that file exists. Including a case like: | |
| 440 # module.resource('dir').join('subdir', 'file.py') | |
| 441 return self._module.MODULE_DIRECTORY.join('resources', *path) | |
| 442 | |
| 443 @property | |
| 444 def name(self): | |
| 445 return self._module.NAME | |
| 446 | |
| 447 | |
| 448 class RecipeApi(RecipeApiPlain): | |
| 449 __metaclass__ = RecipeApiMeta | |
| 450 | |
| 451 | |
| 452 class Property(object): | |
| 453 sentinel = object() | |
| 454 | |
| 455 @staticmethod | |
| 456 def legal_name(name): | |
| 457 if name.startswith('_'): | |
| 458 return False | |
| 459 | |
| 460 if name in ('self',): | |
| 461 return False | |
| 462 | |
| 463 if keyword.iskeyword(name): | |
| 464 return False | |
| 465 | |
| 466 return True | |
| 467 | |
| 468 @property | |
| 469 def name(self): | |
| 470 return self._name | |
| 471 | |
| 472 @name.setter | |
| 473 def name(self, name): | |
| 474 if not Property.legal_name(name): | |
| 475 raise ValueError("Illegal name '{}'".format(name)) | |
| 476 | |
| 477 self._name = name | |
| 478 | |
| 479 def __init__(self, default=sentinel, help="", kind=None): | |
| 480 """ | |
| 481 Constructor for Property. | |
| 482 | |
| 483 Args: | |
| 484 default: The default value for this Property. Note: A default | |
| 485 value of None is allowed. To have no default value, omit | |
| 486 this argument. | |
| 487 help: The help text for this Property. | |
| 488 type: The type of this Property. You can either pass in a raw python | |
| 489 type, or a Config Type, using the recipe engine config system. | |
| 490 """ | |
| 491 self._default = default | |
| 492 self.help = help | |
| 493 self._name = None | |
| 494 | |
| 495 if isinstance(kind, type): | |
| 496 kind = Single(kind) | |
| 497 self.kind = kind | |
| 498 | |
| 499 def interpret(self, value): | |
| 500 """ | |
| 501 Interprets the value for this Property. | |
| 502 | |
| 503 Args: | |
| 504 value: The value to interpret. May be None, which | |
| 505 means no value provided. | |
| 506 | |
| 507 Returns: | |
| 508 The value to use for this property. Raises an error if | |
| 509 this property has no valid interpretation. | |
| 510 """ | |
| 511 if value is not Property.sentinel: | |
| 512 if self.kind is not None: | |
| 513 # The config system handles type checking for us here. | |
| 514 self.kind.set_val(value) | |
| 515 return value | |
| 516 | |
| 517 if self._default is not Property.sentinel: | |
| 518 return self._default | |
| 519 | |
| 520 raise ValueError( | |
| 521 "No default specified and no value provided for '{}'".format( | |
| 522 self.name)) | |
| 523 | |
| 524 class UndefinedPropertyException(TypeError): | |
| 525 pass | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1347273002:
RELAND of Cross-repo recipe package system. (Closed)
Base URL: https://chromium.googlesource.com/chromium/tools/build@master