Add name, package_repo_resource and resource support to recipe scripts.

recipe_engine/loader.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import collections
 import contextlib
 import imp
 import inspect
 import os
 import sys
 
 from . import env
 
 from .config import ConfigContext, ConfigGroupSchema
 from .config_types import Path, ModuleBasePath, PackageRepoBasePath
+from .config_types import RecipeScriptBasePath
 from .config_types import RECIPE_MODULE_PREFIX
 from .recipe_api import RecipeApi, RecipeApiPlain, RecipeScriptApi
 from .recipe_api import _UnresolvedRequirement
 from .recipe_api import Property, BoundProperty
 from .recipe_api import UndefinedPropertyException, PROPERTY_SENTINEL
 from .recipe_test_api import RecipeTestApi, DisabledTestData
-from .util import scan_directory
 
 
 class LoaderError(Exception):
   """Raised when something goes wrong loading recipes or modules."""
 
 
 class NoSuchRecipe(LoaderError):
   """Raised by load_recipe is recipe is not found."""
   def __init__(self, recipe):
     super(NoSuchRecipe, self).__init__('No such recipe: %s' % recipe)
 
 
 class RecipeScript(object):
   """Holds dict of an evaluated recipe script."""
 
-  def __init__(self, recipe_globals, name):
-    self.name = name
+  def __init__(self, recipe_globals, path):
+    self.path = path
     self._recipe_globals = recipe_globals
 
     self.run_steps, self.gen_tests = [
         recipe_globals.get(k) for k in ('RunSteps', 'GenTests')]
 
     # Let each property object know about the property name.
     recipe_globals['PROPERTIES'] = {
         name: value.bind(name, BoundProperty.RECIPE_PROPERTY, name)
         for name, value in recipe_globals.get('PROPERTIES', {}).items()}
 
     return_schema = recipe_globals.get('RETURN_SCHEMA')
     if return_schema and not isinstance(return_schema, ConfigGroupSchema):
       raise ValueError('Invalid RETURN_SCHEMA; must be an instance of '
                        'ConfigGroupSchema')
 
   @property
+  def name(self):
+    # 'a/b/c/my_name.py' -> my_name
+    return os.path.splitext(os.path.basename(self.path))[0]
+
+  @property
   def globals(self):
     return self._recipe_globals
 
   @property
   def PROPERTIES(self):
     return self._recipe_globals['PROPERTIES']
 
   @property
   def LOADED_DEPS(self):
     return self._recipe_globals['LOADED_DEPS']
@@ skipping 23 matching lines @@
 
     recipe_globals = {}
     recipe_globals['__file__'] = script_path
 
     with env.temp_sys_path():
       execfile(script_path, recipe_globals)
 
     recipe_globals['LOADED_DEPS'] = universe_view.deps_from_spec(
         recipe_globals.get('DEPS', []))
 
-    # 'a/b/c/my_name.py' -> my_name
-    name = os.path.basename(script_path).split('.')[0]
-    return cls(recipe_globals, name)
+    return cls(recipe_globals, script_path)
 
 
 class RecipeUniverse(object):
   def __init__(self, package_deps, config_file):
     self._loaded = {}
     self._package_deps = package_deps
     self._config_file = config_file
 
   @property
   def module_dirs(self):
@@ skipping 158 matching lines @@
   def recipe_dirs(self):
     for recipe_dir in self.package.recipe_dirs:
       yield recipe_dir
 
   def loop_over_recipes(self):
     """Yields pairs (path to recipe, recipe name).
 
     Enumerates real recipes in recipes/*, as well as examples in
     recipe_modules/*.
     """
+    def scan_directory(path, predicate):

[iannucci, 2016/11/19 00:15:32]

this was only used inside loop_over_recipes, so I moved it.

+      for root, dirs, files in os.walk(path):
+        dirs[:] = [x for x in dirs
+                   if not x.endswith(('.expected', '.resources'))]
+        for file_name in (f for f in files if predicate(f)):
+          file_path = os.path.join(root, file_name)
+          yield file_path
+
     for path in self.package.recipe_dirs:
       for recipe in scan_directory(
           path, lambda f: f.endswith('.py') and f[0] != '_'):
         yield recipe, recipe[len(path)+1:-len('.py')]
     for path in self.package.module_dirs:
       for recipe in scan_directory(
           path, lambda f: f.endswith('example.py')):
         module_name = os.path.dirname(recipe)[len(path)+1:]
         yield recipe, '%s:example' % module_name
 
@@ skipping 87 matching lines @@
   API, CONFIG_CTX, TEST_API, and PROPERTIES.
 
   |submod| is a recipe module (akin to python package) with submodules such as
   'api', 'config', 'test_api'. This function scans through dicts of that
   submodules to find subclasses of RecipeApi, RecipeTestApi, etc.
   """
   fullname = '%s/%s' % (universe_view.package.name, name)
   submod.NAME = name
   submod.UNIQUE_NAME = fullname
   submod.MODULE_DIRECTORY = Path(ModuleBasePath(submod))
+  submod.RESOURCE_DIRECTORY = submod.MODULE_DIRECTORY.join('resources')
   submod.PACKAGE_REPO_ROOT = Path(PackageRepoBasePath(universe_view.package))
   submod.CONFIG_CTX = getattr(submod, 'CONFIG_CTX', None)
 
   if hasattr(submod, 'config'):
     for v in submod.config.__dict__.itervalues():
       if isinstance(v, ConfigContext):
         assert not submod.CONFIG_CTX, (
           'More than one configuration context: %s, %s' %
           (submod.config, submod.CONFIG_CTX))
         submod.CONFIG_CTX = v
@@ skipping 140 matching lines @@
   if inspect.isclass(callable_obj):
     arg_names = inspect.getargspec(callable_obj.__init__).args
 
     arg_names.pop(0)
   else:
     arg_names = inspect.getargspec(callable_obj).args
   return _invoke_with_properties(callable_obj, all_props, prop_defs, arg_names,
                                  **additional_args)
 
 
-def create_recipe_api(toplevel_deps, engine, test_data=DisabledTestData()):
+def create_recipe_api(toplevel_package, toplevel_deps, recipe_script_path,
+                      engine, test_data=DisabledTestData()):
   def instantiator(mod, deps):
     kwargs = {
       'module': mod,
       # TODO(luqui): test_data will need to use canonical unique names.
       'test_data': test_data.get_module_test_data(mod.NAME)
     }
     prop_defs = mod.PROPERTIES
     mod_api = invoke_with_properties(
         mod.API, engine.properties, prop_defs, **kwargs)
     mod_api.test_api = (getattr(mod, 'TEST_API', None)
                         or RecipeTestApi)(module=mod)
     for k, v in deps.iteritems():
       setattr(mod_api.m, k, v)
       setattr(mod_api.test_api.m, k, v.test_api)
 
     # Replace class-level Requirements placeholders in the recipe API with
     # their instance-level real values.
     map(lambda (k, v): setattr(mod_api, k, _resolve_requirement(v, engine)),
         ((k, v) for k, v in type(mod_api).__dict__.iteritems()
          if isinstance(v, _UnresolvedRequirement)))
 
     mod_api.initialize()
     return mod_api
 
   mapper = DependencyMapper(instantiator)
-  api = RecipeScriptApi(module=None, engine=engine,
+  # Provide a fake module to the ScriptApi so that recipes can use:
+  #   * .name
+  #   * .resource
+  #   * .package_repo_resource
+  # This is obviously a hack, however it homogenizes the api and removes the
+  # need for some ugly workarounds in user code. A better way to do this would
+  # be to migrate all recipes to be members of modules.
+  name = os.path.join(
+    toplevel_package.name,
+    os.path.relpath(
+      os.path.splitext(recipe_script_path)[0],
+      toplevel_package.recipe_dirs[0]))
+  fakeModule = collections.namedtuple(
+    "fakeModule", "PACKAGE_REPO_ROOT NAME RESOURCE_DIRECTORY")(
+      Path(PackageRepoBasePath(toplevel_package)),
+      name,
+      Path(RecipeScriptBasePath(
+        name, os.path.splitext(recipe_script_path)[0]+".resources")))
+  api = RecipeScriptApi(module=fakeModule, engine=engine,
                   test_data=test_data.get_module_test_data(None))
   for k, v in toplevel_deps.iteritems():
     setattr(api, k, mapper.instantiate(v))
   return api
 
 
 def create_test_api(toplevel_deps, universe):
   def instantiator(mod, deps):
     modapi = (getattr(mod, 'TEST_API', None) or RecipeTestApi)(module=mod)
     for k,v in deps.iteritems():
       setattr(modapi.m, k, v)
     return modapi
 
   mapper = DependencyMapper(instantiator)
   api = RecipeTestApi(module=None)
   for k,v in toplevel_deps.iteritems():
     setattr(api, k, mapper.instantiate(v))
   return api
 
 
 def _resolve_requirement(req, engine):
   if req._typ == 'client':
     return engine._get_client(req._name)
   else:
     raise ValueError('Unknown requirement type [%s]' % (req._typ,))