add recipe continuous builders for each repo with recipes

scripts/slave/recipes/infra/recipe_simulation.py

Index: scripts/slave/recipes/infra/recipe_simulation.py
diff --git a/scripts/slave/recipes/infra/recipe_simulation.py b/scripts/slave/recipes/infra/recipe_simulation.py
index 7283e3a183c4398271289d88923fa4f1d2732815..97db7a0e5711208c31940c2727ce16789dd56114 100644
--- a/scripts/slave/recipes/infra/recipe_simulation.py
+++ b/scripts/slave/recipes/infra/recipe_simulation.py
@@ -3,27 +3,66 @@
 # found in the LICENSE file.
 """A continious builder for build repo which simulates recipes."""
 
+from recipe_engine.recipe_api import Property
+
 DEPS = [
   'depot_tools/bot_update',
   'depot_tools/gclient',
   'recipe_engine/path',
   'recipe_engine/properties',
   'recipe_engine/python',
+  'luci_config',
 ]
 
+def _python(self, name, script, args, **kwargs):

[martiniss, 2016/06/11 02:20:30]

ignore this

+  """Call python from infra's virtualenv.
+
+  This is needed because of the coverage module, which is not installed by
+  default, but which infra's python has installed."""
+  return self.m.step(name, [
+      self.m.path['checkout'].join('ENV', 'bin', 'python'),
+      '-u', script] + args, **kwargs)
+
+PROPERTIES = {
+  'project_under_test': Property(
+      default='build', kind=str, help='luci-config project to run tests for')
+}
+
+
+def RunSteps(api, project_under_test):
+  cache_dir = api.path['checkout'].join('_cache_dir')
+
+  # Needed to set up the infra checkout for infra python, which has coverage.
+  c = api.gclient.make_config('infra', CACHE_DIR=cache_dir)
+  c.solutions[0].revision = 'origin/master'
+  api.bot_update.ensure_checkout(
+      force=True, gclient_config=c, cwd=api.path['checkout'].join(
+          'custom_infra'))
+
+  c = api.gclient.make_config(CACHE_DIR=cache_dir)
+  soln = c.solutions.add()
+  soln.name = project_under_test
+  soln.url = api.luci_config.get_project_metadata(
+      project_under_test)['repo_url']
+
+  api.bot_update.ensure_checkout(
+      force=True, gclient_config=c,
+      cwd=api.path['checkout'])
 
-def RunSteps(api):
-  api.gclient.set_config('build')
-  api.bot_update.ensure_checkout(force=True)
-  recipes_py = api.path['checkout'].join('scripts', 'slave', 'recipes.py')
-  api.python('recipe fetch deps', recipes_py, ['fetch'])
-  # In theory, this should work too. But in practice, this fails to import
-  # coverage module (http://crbug.com/577049).
-  # api.python('recipe simulation test', recipes_py, ['simulation_test'])
-  recipe_simulation_test = api.path['checkout'].join(
-      'scripts', 'slave', 'unittests', 'recipe_simulation_test.py')
-  api.python('recipe simulation test', recipe_simulation_test, ['test'])
+  # TODO(martiniss): allow recipes.cfg patches to take affect
+  # This requires getting the refs.cfg from luci_config, reading the local
+  # patched version, etc.
+  result = api.luci_config.get_project_config(project_under_test, 'recipes.cfg')
+  recipes_cfg = parse_protobuf(result['content'].split('\n'))

[martiniss, 2016/06/11 02:20:30]

should i move this function to the luci_config recipe module? is anything other
than a pseudo proto ever stored in luci-config?

Implementation is
https://chromium.googlesource.com/chromium/tools/build.git/+/master/scripts/s...

[Vadim Sh., 2016/06/11 02:25:05]

luci-config can store any kinds of smallish files, not necessary protos. 

[martiniss, 2016/06/11 02:33:14]

Ok. WDYT of adding something like luci_config.get_project_proto_config, which
does this fake proto parsing? 

It basically turns the proto into a python dict. It should do some parsing given
the proto file, but i'm not sure how I would do that.... maybe build protos into
the recipe engine.

+  path = recipes_cfg['recipes_path'][0].split('/')
 
+  api.step(
+      'recipe simulation test', [
+          api.path['checkout'].join('custom_infra', 'ENV', 'bin', 'python'),
+          api.path['checkout'].join(
+              (project_under_test,) + path + ('recipes.py')),
+          'simulation_test'
+      ])
 
 def GenTests(api):
   yield (