Cross-repo recipe package system.

third_party/recipe_engine/loader.py

 # Copyright 2013-2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import contextlib
 import imp
 import inspect
 import os
 import sys
 
@@ skipping 18 matching lines @@
   @classmethod
   def from_script_path(cls, script_path, universe):
     """Evaluates a script and returns RecipeScript instance."""
 
     script_vars = {}
     script_vars['__file__'] = script_path
 
     with _preserve_path():
       execfile(script_path, script_vars)
 
-    script_vars['LOADED_DEPS'] = universe.deps_from_mixed(
-        script_vars.get('DEPS', []), os.path.basename(script_path))
+    script_vars['LOADED_DEPS'] = universe.deps_from_spec(
+        script_vars.get('DEPS', []))
     return cls(script_vars)
 
 
 class Dependency(object):
   def load(self, universe):
     raise NotImplementedError()
 
   @property
   def local_name(self):
     raise NotImplementedError()
 
   @property
   def unique_name(self):
     """A unique identifier for the module that this dependency refers to.
     This must be generated without loading the module."""
     raise NotImplementedError()
 
 
 class PathDependency(Dependency):
-  def __init__(self, path, local_name, universe, base_path=None):
-    self._path = _normalize_path(base_path, path)
+  def __init__(self, path, local_name, universe):
+    assert os.path.isabs(path), (
+        'Path dependencies must be absolute, but %s is not' % path)
+    self._path = path
     self._local_name = local_name
 
     # We forbid modules from living outside our main paths to keep clients
     # from going crazy before we have standardized recipe locations.
     mod_dir = os.path.dirname(path)
     assert mod_dir in universe.module_dirs, (
       'Modules living outside of approved directories are forbidden: '
       '%s is not in %s' % (mod_dir, universe.module_dirs))
 
   def load(self, universe):
@@ skipping 11 matching lines @@
 class NamedDependency(PathDependency):
   def __init__(self, name, universe):
     for path in universe.module_dirs:
       mod_path = os.path.join(path, name)
       if os.path.exists(os.path.join(mod_path, '__init__.py')):
         super(NamedDependency, self).__init__(mod_path, name, universe=universe)
         return
     raise NoSuchRecipe('Recipe module named %s does not exist' % name)
 
 
+class PackageDependency(PathDependency):
+  # TODO(luqui): Forbid depending on a module from a (locally) undeclared
+  # dependency.
+  def __init__(self, package, module, local_name, universe):
+    mod_path = (
+        universe.package_deps.get_package(package).module_path(module))
+    super(PackageDependency, self).__init__(
+        mod_path, local_name, universe=universe)
+
+
 class RecipeUniverse(object):
-  def __init__(self, module_dirs, recipe_dirs):
+  def __init__(self, package_deps):
     self._loaded = {}
-    self._module_dirs = module_dirs[:]
-    self._recipe_dirs = recipe_dirs[:]
+    self._package_deps = package_deps
 
   @property
   def module_dirs(self):
-    return self._module_dirs
+    return self._package_deps.all_module_dirs
 
   @property
   def recipe_dirs(self):
-    return self._recipe_dirs
+    return self._package_deps.all_recipe_dirs
+
+  @property
+  def package_deps(self):
+    return self._package_deps
 
   def load(self, dep):
     """Load a Dependency."""
     name = dep.unique_name
     if name in self._loaded:
       mod = self._loaded[name]
       assert mod is not None, (
           'Cyclic dependency when trying to load %s' % name)
       return mod
     else:
       self._loaded[name] = None
       mod = dep.load(self)
       self._loaded[name] = mod
       return mod
 
-  def deps_from_names(self, deps):
-    """Load dependencies given a list simple module names (old style)."""
-    return { dep: self.load(NamedDependency(dep, universe=self))
-             for dep in deps }
+  def _dep_from_name(self, name):
+    if '/' in name:
+      [package,module] = name.split('/')
+      dep = PackageDependency(package, module, module, universe=self)
+    else:
+      # Old style: bare module name, search paths to find it.
+      module = name
+      dep = NamedDependency(name, universe=self)
 
-  def deps_from_paths(self, deps, base_path):
-    """Load dependencies given a dictionary of local names to module paths
-    (new style)."""
-    return { name: self.load(PathDependency(path, name,
-                                            universe=self, base_path=base_path))
-             for name, path in deps.iteritems() }
+    return module, dep
 
-  def deps_from_mixed(self, deps, base_path):
-    """Load dependencies given either a new style or old style deps spec."""
-    if isinstance(deps, (list, tuple)):
-      return self.deps_from_names(deps)
-    elif isinstance(deps, dict):
-      return self.deps_from_paths(deps, base_path)
-    else:
-      raise ValueError('%s is not a valid or known deps structure' % deps)
+  def deps_from_spec(self, spec):
+    # Automatic local names.
+    if isinstance(spec, (list, tuple)):
+      deps = {}
+      for item in spec:
+        name, dep = self._dep_from_name(item)
+        deps[name] = self.load(dep)
+    # Explicit local names.
+    elif isinstance(spec, dict):
+      deps = {}
+      for name, item in spec.iteritems():
+        _, dep = self._dep_from_name(item)
+        deps[name] = self.load(dep)
+    return deps
 
   def load_recipe(self, recipe):
     """Given name of a recipe, loads and returns it as RecipeScript instance.
 
     Args:
       recipe (str): name of a recipe, can be in form '<module>:<recipe>'.
 
     Returns:
       RecipeScript instance.
 
     Raises:
       NoSuchRecipe: recipe is not found.
     """
     # If the recipe is specified as "module:recipe", then it is an recipe
     # contained in a recipe_module as an example. Look for it in the modules
     # imported by load_recipe_modules instead of the normal search paths.
     if ':' in recipe:
       module_name, example = recipe.split(':')
       assert example.endswith('example')
       for module_dir in self.module_dirs:
-        for subitem in os.listdir(module_dir):
-          if module_name == subitem:
-            return RecipeScript.from_script_path(
-                os.path.join(module_dir, subitem, 'example.py'), self)
+        if os.path.isdir(module_dir):
+          for subitem in os.listdir(module_dir):
+            if module_name == subitem:
+              return RecipeScript.from_script_path(
+                  os.path.join(module_dir, subitem, 'example.py'), self)
       raise NoSuchRecipe(recipe,
                          'Recipe example %s:%s does not exist' %
                          (module_name, example))
     else:
       for recipe_path in (os.path.join(p, recipe) for p in self.recipe_dirs):
         if os.path.exists(recipe_path + '.py'):
           return RecipeScript.from_script_path(recipe_path + '.py', self)
     raise NoSuchRecipe(recipe)
 
   def loop_over_recipe_modules(self):
@@ skipping 22 matching lines @@
 
 @contextlib.contextmanager
 def _preserve_path():
   old_path = sys.path[:]
   try:
     yield
   finally:
     sys.path = old_path
 
 
-def _normalize_path(base_path, path):
-  if base_path is None or os.path.isabs(path):
-    return os.path.realpath(path)
-  else:
-    return os.path.realpath(os.path.join(base_path, path))
-
-
 def _find_and_load_module(fullname, modname, path):
   imp.acquire_lock()
   try:
     if fullname not in sys.modules:
       fil = None
       try:
         fil, pathname, descr = imp.find_module(modname,
                                                [os.path.dirname(path)])
         imp.load_module(fullname, fil, pathname, descr)
       finally:
         if fil:
           fil.close()
     return sys.modules[fullname]
   finally:
     imp.release_lock()
 
 
 def _load_recipe_module_module(path, universe):
   modname = os.path.splitext(os.path.basename(path))[0]
   fullname = '%s.%s' % (RECIPE_MODULE_PREFIX, modname)
   mod = _find_and_load_module(fullname, modname, path)
 
   # This actually loads the dependencies.
-  mod.LOADED_DEPS = universe.deps_from_mixed(
-      getattr(mod, 'DEPS', []), os.path.basename(path))
+  mod.LOADED_DEPS = universe.deps_from_spec(getattr(mod, 'DEPS', []))
 
   # Prevent any modules that mess with sys.path from leaking.
   with _preserve_path():
     # TODO(luqui): Remove this hack once configs are cleaned.
     sys.modules['%s.DEPS' % fullname] = mod.LOADED_DEPS
     _recursive_import(path, RECIPE_MODULE_PREFIX)
     _patchup_module(modname, mod)
 
   return mod
 
@@ skipping 135 matching lines @@
     return modapi
 
   mapper = DependencyMapper(instantiator)
   api = RecipeTestApi(module=None)
   for k,v in toplevel_deps.iteritems():
     setattr(api, k, mapper.instantiate(v))
   return api