[loader] Make recipe engine support examples/ module subdir

recipe_engine/loader.py

 # Copyright 2016 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import collections
 import imp
 import inspect
 import os
 import sys
 
@@ skipping 233 matching lines @@
       'chromiuminternal': 'build_internal/chromium',
     }
     """
     return {
       local_name: self.universe.load(pkg, name)
       for local_name, (pkg, name) in self.normalize_deps_spec(spec).iteritems()
     }
 
 
   def find_recipe(self, recipe):
+    """Finds the given recipe.
+
+    Args:
+      recipe (str) - the name of the recipe as you would specify to
+      `recipes.py run`. This may have the form:
+        recipe, path/to/recipe - a recipe under the `recipes folder`
+        module:tests/recipe - a test recipe under a recipe_module.
+        module:examples/recipe - an example recipe under a recipe_module.
+        module:example - a legacy example example recipe under a recipe_module
+
+    Return (str) - The absolute path to the recipe .py file.
+
+    Raises NoSuchRecipe if this cannot find the recipe.
+    """
+    # If the recipe is specified as "module:recipe", then it is an recipe
+    # contained in a recipe_module as an example, or a test. Look for it in the
+    # modules imported by load_recipe_modules instead of the normal search
+    # paths.
+    #
+    # TODO(martiniss) change "infra/example" to ["infra", "example"],
+    # and handle appropriately, because of windows.
     if ':' in recipe:
       module_name, recipe_name = recipe.split(':')
       module_dir = os.path.join(self.package.module_dir, module_name)
       if _is_recipe_module_dir(module_dir):
-        if recipe_name.endswith('example'):
-          # TODO(martinis) change to example == 'example' ? Technically a bug...
-          recipe_path = os.path.join(module_dir, 'example.py')
-        elif recipe_name.startswith('tests/'):
-          recipe_path = os.path.join(
-              module_dir, 'tests', recipe_name[len('tests/'):] + '.py')
+        # TODO(iannucci): remove legacy example.
+        if (recipe_name == 'example' or
+            recipe_name.startswith(('tests/', 'examples/'))):
+          recipe_path = os.path.join(module_dir, recipe_name + '.py')
         else:
           raise NoSuchRecipe(recipe)
     else:
       recipe_path = os.path.join(self.package.recipe_dir, recipe)+".py"
 
     if os.path.exists(recipe_path):
       return recipe_path
 
     raise NoSuchRecipe(recipe)
 
   def load_recipe(self, recipe, engine=None):
     """Given name of a recipe, loads and returns it as RecipeScript instance.
 
     Args:
       recipe (str): name of a recipe, can be in form '<module>:<recipe>'.
       engine (RecipeEngine): If not None, resolve requirements with this recipe
           engine. Otherwise, requirements will be unresolved.
 
     Returns:
       RecipeScript instance.
 
     Raises:
       NoSuchRecipe: recipe is not found.
     """
-    # If the recipe is specified as "module:recipe", then it is an recipe
-    # contained in a recipe_module as an example. Look for it in the modules
-    # imported by load_recipe_modules instead of the normal search paths.
-    # TODO(martiniss) change "infra/example" to ["infra", "example"], and handle
-    # appropriately, because of windows.
     recipe_path = self.find_recipe(recipe)
     try:
       script = RecipeScript.from_script_path(recipe, recipe_path, self)
     except (LoaderError,AssertionError,ImportError) as e:
       _amend_exception(e, 'while loading recipe %s' % recipe)
 
     if engine is not None:
       script.globals.update({k: _resolve_requirement(v, engine)
                              for k, v in script.globals.iteritems()
                              if isinstance(v, _UnresolvedRequirement)})
@@ skipping 27 matching lines @@
           yield file_path
 
     path = self.package.recipe_dir
     for recipe in scan_directory(
         path, lambda f: f.endswith('.py') and f[0] != '_'):
       yield recipe, recipe[len(path)+1:-len('.py')]
 
     for module_name in self.loop_over_recipe_modules():
       module_dir = os.path.join(self.package.module_dir, module_name)
 
+      # TODO(iannucci): remove unscoped example.py
       example_path = os.path.join(module_dir, 'example.py')
       if os.path.exists(example_path):
         yield example_path, '%s:example' % module_name
 
-      test_dir = os.path.join(module_dir, 'tests')
-      if os.path.exists(test_dir):
-        for recipe in scan_directory(test_dir, lambda f: f.endswith('.py')):
-          yield recipe, '%s:tests/%s' % (
-              module_name, recipe[len(test_dir)+1:-len('.py')])
+      for subdir_name in ('tests', 'examples'):
+        subdir = os.path.join(module_dir, subdir_name)
+        if os.path.exists(subdir):
+          for recipe in scan_directory(subdir, lambda f: f.endswith('.py')):
+            yield recipe, '%s:%s/%s' % (
+                module_name, subdir_name, recipe[len(subdir)+1:-len('.py')])
 
   def loop_over_recipe_modules(self):
     """Yields the paths to all the modules that this view can see."""
     path = self.package.module_dir
     if os.path.isdir(path):
       for item in os.listdir(path):
         subpath = os.path.join(path, item)
         if _is_recipe_module_dir(subpath):
           yield os.path.basename(subpath)
 
@@ skipping 338 matching lines @@
   for k,v in toplevel_deps.iteritems():
     setattr(api, k, mapper.instantiate(v))
   return api
 
 
 def _resolve_requirement(req, engine):
   if req._typ == 'client':
     return engine._get_client(req._name)
   else:
     raise ValueError('Unknown requirement type [%s]' % (req._typ,))