| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright 2013-2015 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 import contextlib | |
| 6 import imp | |
| 7 import inspect | |
| 8 import os | |
| 9 import sys | |
| 10 | |
| 11 from .config import ConfigContext | |
| 12 from .config_types import Path, ModuleBasePath, RECIPE_MODULE_PREFIX | |
| 13 from .recipe_api import RecipeApi, RecipeApiPlain, Property, UndefinedPropertyEx
ception | |
| 14 from .recipe_test_api import RecipeTestApi, DisabledTestData | |
| 15 from .util import scan_directory | |
| 16 | |
| 17 | |
| 18 class NoSuchRecipe(Exception): | |
| 19 """Raised by load_recipe is recipe is not found.""" | |
| 20 | |
| 21 | |
| 22 class RecipeScript(object): | |
| 23 """Holds dict of an evaluated recipe script.""" | |
| 24 | |
| 25 def __init__(self, recipe_dict): | |
| 26 recipe_dict.setdefault('PROPERTIES', {}) | |
| 27 # Let each property object know about the property name. | |
| 28 for name, value in recipe_dict['PROPERTIES'].items(): | |
| 29 value.name = name | |
| 30 | |
| 31 for k, v in recipe_dict.iteritems(): | |
| 32 setattr(self, k, v) | |
| 33 | |
| 34 @classmethod | |
| 35 def from_script_path(cls, script_path, universe): | |
| 36 """Evaluates a script and returns RecipeScript instance.""" | |
| 37 | |
| 38 script_vars = {} | |
| 39 script_vars['__file__'] = script_path | |
| 40 | |
| 41 with _preserve_path(): | |
| 42 execfile(script_path, script_vars) | |
| 43 | |
| 44 script_vars['LOADED_DEPS'] = universe.deps_from_mixed( | |
| 45 script_vars.get('DEPS', []), os.path.basename(script_path)) | |
| 46 return cls(script_vars) | |
| 47 | |
| 48 | |
| 49 class Dependency(object): | |
| 50 def load(self, universe): | |
| 51 raise NotImplementedError() | |
| 52 | |
| 53 @property | |
| 54 def local_name(self): | |
| 55 raise NotImplementedError() | |
| 56 | |
| 57 @property | |
| 58 def unique_name(self): | |
| 59 """A unique identifier for the module that this dependency refers to. | |
| 60 This must be generated without loading the module.""" | |
| 61 raise NotImplementedError() | |
| 62 | |
| 63 | |
| 64 class PathDependency(Dependency): | |
| 65 def __init__(self, path, local_name, universe, base_path=None): | |
| 66 self._path = _normalize_path(base_path, path) | |
| 67 self._local_name = local_name | |
| 68 | |
| 69 # We forbid modules from living outside our main paths to keep clients | |
| 70 # from going crazy before we have standardized recipe locations. | |
| 71 mod_dir = os.path.dirname(path) | |
| 72 assert mod_dir in universe.module_dirs, ( | |
| 73 'Modules living outside of approved directories are forbidden: ' | |
| 74 '%s is not in %s' % (mod_dir, universe.module_dirs)) | |
| 75 | |
| 76 def load(self, universe): | |
| 77 return _load_recipe_module_module(self._path, universe) | |
| 78 | |
| 79 @property | |
| 80 def local_name(self): | |
| 81 return self._local_name | |
| 82 | |
| 83 @property | |
| 84 def unique_name(self): | |
| 85 return self._path | |
| 86 | |
| 87 | |
| 88 class NamedDependency(PathDependency): | |
| 89 def __init__(self, name, universe): | |
| 90 for path in universe.module_dirs: | |
| 91 mod_path = os.path.join(path, name) | |
| 92 if _is_recipe_module_dir(mod_path): | |
| 93 super(NamedDependency, self).__init__(mod_path, name, universe=universe) | |
| 94 return | |
| 95 raise NoSuchRecipe('Recipe module named %s does not exist' % name) | |
| 96 | |
| 97 | |
| 98 class RecipeUniverse(object): | |
| 99 def __init__(self, module_dirs, recipe_dirs): | |
| 100 self._loaded = {} | |
| 101 self._module_dirs = module_dirs[:] | |
| 102 self._recipe_dirs = recipe_dirs[:] | |
| 103 | |
| 104 @property | |
| 105 def module_dirs(self): | |
| 106 return self._module_dirs | |
| 107 | |
| 108 @property | |
| 109 def recipe_dirs(self): | |
| 110 return self._recipe_dirs | |
| 111 | |
| 112 def load(self, dep): | |
| 113 """Load a Dependency.""" | |
| 114 name = dep.unique_name | |
| 115 if name in self._loaded: | |
| 116 mod = self._loaded[name] | |
| 117 assert mod is not None, ( | |
| 118 'Cyclic dependency when trying to load %s' % name) | |
| 119 return mod | |
| 120 else: | |
| 121 self._loaded[name] = None | |
| 122 mod = dep.load(self) | |
| 123 self._loaded[name] = mod | |
| 124 return mod | |
| 125 | |
| 126 def deps_from_names(self, deps): | |
| 127 """Load dependencies given a list simple module names (old style).""" | |
| 128 return { dep: self.load(NamedDependency(dep, universe=self)) | |
| 129 for dep in deps } | |
| 130 | |
| 131 def deps_from_paths(self, deps, base_path): | |
| 132 """Load dependencies given a dictionary of local names to module paths | |
| 133 (new style).""" | |
| 134 return { name: self.load(PathDependency(path, name, | |
| 135 universe=self, base_path=base_path)) | |
| 136 for name, path in deps.iteritems() } | |
| 137 | |
| 138 def deps_from_mixed(self, deps, base_path): | |
| 139 """Load dependencies given either a new style or old style deps spec.""" | |
| 140 if isinstance(deps, (list, tuple)): | |
| 141 return self.deps_from_names(deps) | |
| 142 elif isinstance(deps, dict): | |
| 143 return self.deps_from_paths(deps, base_path) | |
| 144 else: | |
| 145 raise ValueError('%s is not a valid or known deps structure' % deps) | |
| 146 | |
| 147 def load_recipe(self, recipe): | |
| 148 """Given name of a recipe, loads and returns it as RecipeScript instance. | |
| 149 | |
| 150 Args: | |
| 151 recipe (str): name of a recipe, can be in form '<module>:<recipe>'. | |
| 152 | |
| 153 Returns: | |
| 154 RecipeScript instance. | |
| 155 | |
| 156 Raises: | |
| 157 NoSuchRecipe: recipe is not found. | |
| 158 """ | |
| 159 # If the recipe is specified as "module:recipe", then it is an recipe | |
| 160 # contained in a recipe_module as an example. Look for it in the modules | |
| 161 # imported by load_recipe_modules instead of the normal search paths. | |
| 162 if ':' in recipe: | |
| 163 module_name, example = recipe.split(':') | |
| 164 assert example.endswith('example') | |
| 165 for module_dir in self.module_dirs: | |
| 166 for subitem in os.listdir(module_dir): | |
| 167 if module_name == subitem: | |
| 168 return RecipeScript.from_script_path( | |
| 169 os.path.join(module_dir, subitem, 'example.py'), self) | |
| 170 raise NoSuchRecipe(recipe, | |
| 171 'Recipe example %s:%s does not exist' % | |
| 172 (module_name, example)) | |
| 173 else: | |
| 174 for recipe_path in (os.path.join(p, recipe) for p in self.recipe_dirs): | |
| 175 if os.path.exists(recipe_path + '.py'): | |
| 176 return RecipeScript.from_script_path(recipe_path + '.py', self) | |
| 177 raise NoSuchRecipe(recipe) | |
| 178 | |
| 179 def loop_over_recipe_modules(self): | |
| 180 for path in self.module_dirs: | |
| 181 if os.path.isdir(path): | |
| 182 for item in os.listdir(path): | |
| 183 subpath = os.path.join(path, item) | |
| 184 if _is_recipe_module_dir(subpath): | |
| 185 yield subpath | |
| 186 | |
| 187 def loop_over_recipes(self): | |
| 188 """Yields pairs (path to recipe, recipe name). | |
| 189 | |
| 190 Enumerates real recipes in recipes/* as well as examples in recipe_modules/*
. | |
| 191 """ | |
| 192 for path in self.recipe_dirs: | |
| 193 for recipe in scan_directory( | |
| 194 path, lambda f: f.endswith('.py') and f[0] != '_'): | |
| 195 yield recipe, recipe[len(path)+1:-len('.py')] | |
| 196 for path in self.module_dirs: | |
| 197 for recipe in scan_directory( | |
| 198 path, lambda f: f.endswith('example.py')): | |
| 199 module_name = os.path.dirname(recipe)[len(path)+1:] | |
| 200 yield recipe, '%s:example' % module_name | |
| 201 | |
| 202 | |
| 203 def _is_recipe_module_dir(path): | |
| 204 return (os.path.isdir(path) and | |
| 205 os.path.isfile(os.path.join(path, '__init__.py'))) | |
| 206 | |
| 207 | |
| 208 @contextlib.contextmanager | |
| 209 def _preserve_path(): | |
| 210 old_path = sys.path[:] | |
| 211 try: | |
| 212 yield | |
| 213 finally: | |
| 214 sys.path = old_path | |
| 215 | |
| 216 | |
| 217 def _normalize_path(base_path, path): | |
| 218 if base_path is None or os.path.isabs(path): | |
| 219 return os.path.realpath(path) | |
| 220 else: | |
| 221 return os.path.realpath(os.path.join(base_path, path)) | |
| 222 | |
| 223 | |
| 224 def _find_and_load_module(fullname, modname, path): | |
| 225 imp.acquire_lock() | |
| 226 try: | |
| 227 if fullname not in sys.modules: | |
| 228 fil = None | |
| 229 try: | |
| 230 fil, pathname, descr = imp.find_module(modname, | |
| 231 [os.path.dirname(path)]) | |
| 232 imp.load_module(fullname, fil, pathname, descr) | |
| 233 finally: | |
| 234 if fil: | |
| 235 fil.close() | |
| 236 return sys.modules[fullname] | |
| 237 finally: | |
| 238 imp.release_lock() | |
| 239 | |
| 240 | |
| 241 def _load_recipe_module_module(path, universe): | |
| 242 modname = os.path.splitext(os.path.basename(path))[0] | |
| 243 fullname = '%s.%s' % (RECIPE_MODULE_PREFIX, modname) | |
| 244 mod = _find_and_load_module(fullname, modname, path) | |
| 245 | |
| 246 # This actually loads the dependencies. | |
| 247 mod.LOADED_DEPS = universe.deps_from_mixed( | |
| 248 getattr(mod, 'DEPS', []), os.path.basename(path)) | |
| 249 | |
| 250 # Prevent any modules that mess with sys.path from leaking. | |
| 251 with _preserve_path(): | |
| 252 # TODO(luqui): Remove this hack once configs are cleaned. | |
| 253 sys.modules['%s.DEPS' % fullname] = mod.LOADED_DEPS | |
| 254 _recursive_import(path, RECIPE_MODULE_PREFIX) | |
| 255 _patchup_module(modname, mod) | |
| 256 | |
| 257 return mod | |
| 258 | |
| 259 | |
| 260 def _recursive_import(path, prefix): | |
| 261 modname = os.path.splitext(os.path.basename(path))[0] | |
| 262 fullname = '%s.%s' % (prefix, modname) | |
| 263 mod = _find_and_load_module(fullname, modname, path) | |
| 264 if not os.path.isdir(path): | |
| 265 return mod | |
| 266 | |
| 267 for subitem in os.listdir(path): | |
| 268 subpath = os.path.join(path, subitem) | |
| 269 subname = os.path.splitext(subitem)[0] | |
| 270 if os.path.isdir(subpath): | |
| 271 if not os.path.exists(os.path.join(subpath, '__init__.py')): | |
| 272 continue | |
| 273 elif not subpath.endswith('.py') or subitem.startswith('__init__.py'): | |
| 274 continue | |
| 275 | |
| 276 submod = _recursive_import(subpath, fullname) | |
| 277 | |
| 278 if not hasattr(mod, subname): | |
| 279 setattr(mod, subname, submod) | |
| 280 else: | |
| 281 prev = getattr(mod, subname) | |
| 282 assert submod is prev, ( | |
| 283 'Conflicting modules: %s and %s' % (prev, mod)) | |
| 284 | |
| 285 return mod | |
| 286 | |
| 287 | |
| 288 def _patchup_module(name, submod): | |
| 289 """Finds framework related classes and functions in a |submod| and adds | |
| 290 them to |submod| as top level constants with well known names such as | |
| 291 API, CONFIG_CTX, TEST_API, and PROPERTIES. | |
| 292 | |
| 293 |submod| is a recipe module (akin to python package) with submodules such as | |
| 294 'api', 'config', 'test_api'. This function scans through dicts of that | |
| 295 submodules to find subclasses of RecipeApi, RecipeTestApi, etc. | |
| 296 """ | |
| 297 submod.NAME = name | |
| 298 submod.UNIQUE_NAME = name # TODO(luqui): use a luci-config unique name | |
| 299 submod.MODULE_DIRECTORY = Path(ModuleBasePath(submod)) | |
| 300 submod.CONFIG_CTX = getattr(submod, 'CONFIG_CTX', None) | |
| 301 | |
| 302 if hasattr(submod, 'config'): | |
| 303 for v in submod.config.__dict__.itervalues(): | |
| 304 if isinstance(v, ConfigContext): | |
| 305 assert not submod.CONFIG_CTX, ( | |
| 306 'More than one configuration context: %s, %s' % | |
| 307 (submod.config, submod.CONFIG_CTX)) | |
| 308 submod.CONFIG_CTX = v | |
| 309 assert submod.CONFIG_CTX, 'Config file, but no config context?' | |
| 310 | |
| 311 submod.API = getattr(submod, 'API', None) | |
| 312 for v in submod.api.__dict__.itervalues(): | |
| 313 if inspect.isclass(v) and issubclass(v, RecipeApiPlain): | |
| 314 assert not submod.API, ( | |
| 315 '%s has more than one Api subclass: %s, %s' % (name, v, submod.api)) | |
| 316 submod.API = v | |
| 317 assert submod.API, 'Submodule has no api? %s' % (submod) | |
| 318 | |
| 319 submod.TEST_API = getattr(submod, 'TEST_API', None) | |
| 320 if hasattr(submod, 'test_api'): | |
| 321 for v in submod.test_api.__dict__.itervalues(): | |
| 322 if inspect.isclass(v) and issubclass(v, RecipeTestApi): | |
| 323 assert not submod.TEST_API, ( | |
| 324 'More than one TestApi subclass: %s' % submod.api) | |
| 325 submod.TEST_API = v | |
| 326 assert submod.API, ( | |
| 327 'Submodule has test_api.py but no TestApi subclass? %s' | |
| 328 % (submod) | |
| 329 ) | |
| 330 | |
| 331 submod.PROPERTIES = getattr(submod, 'PROPERTIES', {}) | |
| 332 # Let each property object know about the property name. | |
| 333 for name, value in submod.PROPERTIES.items(): | |
| 334 value.name = name | |
| 335 | |
| 336 | |
| 337 class DependencyMapper(object): | |
| 338 """DependencyMapper topologically traverses the dependency DAG beginning at | |
| 339 a module, executing a callback ("instantiator") for each module. | |
| 340 | |
| 341 For example, if the dependency DAG looked like this: | |
| 342 | |
| 343 A | |
| 344 / \ | |
| 345 B C | |
| 346 \ / | |
| 347 D | |
| 348 | |
| 349 (with D depending on B and C, etc.), DependencyMapper(f).instantiate(D) would | |
| 350 construct | |
| 351 | |
| 352 f_A = f(A, {}) | |
| 353 f_B = f(B, { 'A': f_A }) | |
| 354 f_C = f(C, { 'A': f_A }) | |
| 355 f_D = f(D, { 'B': f_B, 'C': f_C }) | |
| 356 | |
| 357 finally returning f_D. instantiate can be called multiple times, which reuses | |
| 358 already-computed results. | |
| 359 """ | |
| 360 | |
| 361 def __init__(self, instantiator): | |
| 362 self._instantiator = instantiator | |
| 363 self._instances = {} | |
| 364 | |
| 365 def instantiate(self, mod): | |
| 366 if mod in self._instances: | |
| 367 return self._instances[mod] | |
| 368 deps_dict = { name: self.instantiate(dep) | |
| 369 for name, dep in mod.LOADED_DEPS.iteritems() } | |
| 370 self._instances[mod] = self._instantiator(mod, deps_dict) | |
| 371 return self._instances[mod] | |
| 372 | |
| 373 def invoke_with_properties(callable_obj, all_props, prop_defs, | |
| 374 **additional_args): | |
| 375 """ | |
| 376 Invokes callable with filtered, type-checked properties. | |
| 377 | |
| 378 Args: | |
| 379 callable_obj: The function to call, or class to instantiate. | |
| 380 This supports passing in either RunSteps, or a recipe module, | |
| 381 which is a class. | |
| 382 all_props: A dictionary containing all the properties | |
| 383 currently defined in the system. | |
| 384 prop_defs: A dictionary of name to property definitions for this callable. | |
| 385 additional_args: kwargs to pass through to the callable. | |
| 386 Note that the names of the arguments can correspond to | |
| 387 positional arguments as well. | |
| 388 | |
| 389 Returns: | |
| 390 The result of calling callable with the filtered properties | |
| 391 and additional arguments. | |
| 392 """ | |
| 393 # To detect when they didn't specify a property that they have as a | |
| 394 # function argument, list the arguments, through inspection, | |
| 395 # and then comparing this list to the provided properties. We use a list | |
| 396 # instead of a dict because getargspec returns a list which we would have to | |
| 397 # convert to a dictionary, and the benefit of the dictionary is pretty small. | |
| 398 props = [] | |
| 399 if inspect.isclass(callable_obj): | |
| 400 arg_names = inspect.getargspec(callable_obj.__init__).args | |
| 401 | |
| 402 arg_names.pop(0) | |
| 403 else: | |
| 404 arg_names = inspect.getargspec(callable_obj).args | |
| 405 | |
| 406 for arg in arg_names: | |
| 407 if arg in additional_args: | |
| 408 props.append(additional_args.pop(arg)) | |
| 409 continue | |
| 410 | |
| 411 if arg not in prop_defs: | |
| 412 raise UndefinedPropertyException( | |
| 413 "Missing property definition for '{}'.".format(arg)) | |
| 414 | |
| 415 props.append(prop_defs[arg].interpret(all_props.get( | |
| 416 arg, Property.sentinel))) | |
| 417 | |
| 418 return callable_obj(*props, **additional_args) | |
| 419 | |
| 420 def create_recipe_api(toplevel_deps, engine, test_data=DisabledTestData()): | |
| 421 def instantiator(mod, deps): | |
| 422 kwargs = { | |
| 423 'module': mod, | |
| 424 'engine': engine, | |
| 425 # TODO(luqui): test_data will need to use canonical unique names. | |
| 426 'test_data': test_data.get_module_test_data(mod.NAME) | |
| 427 } | |
| 428 prop_defs = mod.PROPERTIES | |
| 429 mod_api = invoke_with_properties( | |
| 430 mod.API, engine.properties, prop_defs, **kwargs) | |
| 431 mod_api.test_api = (getattr(mod, 'TEST_API', None) | |
| 432 or RecipeTestApi)(module=mod) | |
| 433 for k, v in deps.iteritems(): | |
| 434 setattr(mod_api.m, k, v) | |
| 435 setattr(mod_api.test_api.m, k, v.test_api) | |
| 436 return mod_api | |
| 437 | |
| 438 mapper = DependencyMapper(instantiator) | |
| 439 api = RecipeApi(module=None, engine=engine, | |
| 440 test_data=test_data.get_module_test_data(None)) | |
| 441 for k, v in toplevel_deps.iteritems(): | |
| 442 setattr(api, k, mapper.instantiate(v)) | |
| 443 return api | |
| 444 | |
| 445 | |
| 446 def create_test_api(toplevel_deps, universe): | |
| 447 def instantiator(mod, deps): | |
| 448 modapi = (getattr(mod, 'TEST_API', None) or RecipeTestApi)(module=mod) | |
| 449 for k,v in deps.iteritems(): | |
| 450 setattr(modapi.m, k, v) | |
| 451 return modapi | |
| 452 | |
| 453 mapper = DependencyMapper(instantiator) | |
| 454 api = RecipeTestApi(module=None) | |
| 455 for k,v in toplevel_deps.iteritems(): | |
| 456 setattr(api, k, mapper.instantiate(v)) | |
| 457 return api | |
| 458 | |
| 459 | |
| 460 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1241323004:
Cross-repo recipe package system. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build