First cut of testing infrastructure for recipes.

scripts/slave/unittests/recipes_test.py

Index: scripts/slave/unittests/recipes_test.py
diff --git a/scripts/slave/unittests/recipes_test.py b/scripts/slave/unittests/recipes_test.py
new file mode 100755
index 0000000000000000000000000000000000000000..0050b509839a3c97ef313c8ac8dab75f18d0dbe3
--- /dev/null
+++ b/scripts/slave/unittests/recipes_test.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import contextlib
+import json
+import os
+import sys
+import unittest
+
+from glob import glob
+
+import test_env  # pylint: disable=W0611
+
+import coverage
+
+from common import annotator
+from slave import annotated_run
+from slave import recipe_util
+
+SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
+ROOT_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, os.pardir, os.pardir,
+                                         os.pardir))
+SLAVE_DIR = os.path.join(ROOT_PATH, 'slave', 'fake_slave', 'build')
+BASE_DIRS = {
+  'Public': os.path.dirname(SCRIPT_PATH)
+}
+# TODO(iannucci): Check for duplicate recipe names when we have more than one
+# base_dir
+
+COVERAGE = coverage.coverage(
+    include=[os.path.join(x, 'recipes', '*') for x in BASE_DIRS.values()])
+@contextlib.contextmanager

[Vadim Shtayura, 2013/05/11 00:33:32]

Add newline above this line.

[iannucci, 2013/05/11 04:12:27]

Done.

+def cover():
+  COVERAGE.start()
+  try:
+    yield
+  finally:
+    COVERAGE.stop()
+
+
+class TestAPI(object):
+  @staticmethod
+  def tryserver_build_properties(**kwargs):
+    ret = {
+      'issue': 12853011,
+      'patchset': 1,
+      'description': 'This is a test',
+      'blamelist': ['cool_dev1337@chromium.org'],
+      'rietveld': 'https://chromiumcodereview.appspot.com',
+    }
+    ret.update(kwargs)
+    return ret
+
+
+class DevNull(object):
+  @staticmethod
+  def write(data):
+    pass
+
+  @staticmethod
+  def flush():
+    pass
+
+
+def test_path_for_recipe(recipe_path):
+  root = os.path.dirname(os.path.dirname(recipe_path))
+  return os.path.join(root, 'recipes_test', os.path.basename(recipe_path))
+
+
+def has_test(recipe_path):
+  return os.path.exists(test_path_for_recipe(recipe_path))
+
+
+def expected_for(recipe_path, test_name):
+  test_base = os.path.splitext(test_path_for_recipe(recipe_path))[0]
+  return "%s.%s.expected" % (test_base, test_name)
+
+
+def exec_test_file(recipe_path):
+  test_path = test_path_for_recipe(recipe_path)
+  gvars = {}
+  execfile(test_path, gvars)
+  return dict(item for item in gvars.iteritems() if item[0].endswith('_test'))
+
+
+def execute_test_case(test_fn, recipe_path):
+  test_data = test_fn(TestAPI())
+  bp = test_data.get('build_properties', {})
+  fp = test_data.get('factory_properties', {})
+  fp['recipe'] = os.path.basename(os.path.splitext(recipe_path)[0])
+
+  with cover():
+    with recipe_util.mock_paths():
+      return annotated_run.make_steps(
+          annotator.StructuredAnnotationStream(stream=DevNull()), bp, fp, True)

[Vadim Shtayura, 2013/05/11 00:33:32]

Open real /dev/null? :)

[iannucci, 2013/05/11 04:12:27]

No bueno on windows :/

+
+
+def train_from_tests(_base_dir_type, recipe_path):
+  if not has_test(recipe_path):
+    print 'FATAL: Recipe %s has NO tests!' % recipe_path
+    return False
+
+  for path in glob(expected_for(recipe_path, '*')):
+    os.unlink(path)
+
+  for name, test_fn in exec_test_file(recipe_path).iteritems():
+    steps = execute_test_case(test_fn, recipe_path)
+    expected_path = expected_for(recipe_path, name)
+    print 'Writing', expected_path
+    with open(expected_path, 'w') as f:
+      json.dump(steps, f, indent=2, sort_keys=True)
+
+  return True
+
+
+def NoTestsForRecipe(recipe_path):
+  def test_DoesntExist(self):
+    self.assert_(False, 'No tests exist for %s' % recipe_path)
+  return test_DoesntExist
+
+
+def RecipeTest(name, expected_path, test_fn, recipe_path):
+  def test_(self):
+    steps = execute_test_case(test_fn, recipe_path)
+    # Roundtrip json to get same string encoding as load
+    steps = json.loads(json.dumps(steps))
+    with open(expected_path, 'r') as f:
+      expected = json.load(f)
+    self.assertEqual(steps, expected)
+
+  test_.__name__ += name
+  return test_
+
+
+def add_tests(_base_dir_type, recipe_path, loader, suite):

[Vadim Shtayura, 2013/05/11 00:33:32]

For readability reasons this stuff probably can be converted to some metaclass's
__init__ method. That function is basically a metaclass. Explicitly stating it
as such will make this code more readable IMO. But I'm not sure :)

[iannucci, 2013/05/11 04:12:27]

Hm... I'll take a whack at this, but I'm not sure it'll actually be better. I'll
still have to dynamically create the test methods. It may allow me to contain
the magic in one spot more though.

+  tests = []
+  if not has_test(recipe_path):
+    tests.append(NoTestsForRecipe(recipe_path))
+  else:
+    for name, test_fn in exec_test_file(recipe_path).iteritems():
+      tests.append(RecipeTest(name, expected_for(recipe_path, name),
+                              test_fn, recipe_path))
+
+  suite.addTest(loader.loadTestsFromTestCase(
+    type('RecipeTest_for_%s' % os.path.basename(recipe_path),
+         (unittest.TestCase,), dict((x.__name__, x) for x in tests))))
+  return True
+
+
+def load_tests(loader, standard_tests, _pattern):
+  assert not standard_tests.countTestCases()
+  suite = unittest.TestSuite()
+  assert not loop_over_recipes(add_tests, loader, suite)
+  return suite
+
+def loop_over_recipes(func, *args, **kwargs):

[Vadim Shtayura, 2013/05/11 00:33:32]

Maybe convert to a generator? 'had_errors' is used only with 'train_from_tests'
so far and so can be implemented right there (generators can't have return
values).

[iannucci, 2013/05/11 04:12:27]

Done.

+  had_errors = False
+  for name, path in BASE_DIRS.iteritems():
+    recipe_dir = os.path.join(path, 'recipes')
+    for root, _dirs, files in os.walk(recipe_dir):
+      for recipe in (f for f in files if f.endswith('.py') and f[0] != '_'):
+        recipe_path = os.path.join(root, recipe)
+        with cover():
+          # Force this file into coverage, even if there's no test for it.
+          execfile(recipe_path, {})

[Vadim Shtayura, 2013/05/11 00:33:32]

Do we care about an error in a single recipe blocking tests of other recipes? If
we are, then we need to wrap 'execfile' in exception blocks everywhere.

[iannucci, 2013/05/11 04:12:27]

I think the answer is 'maybe'. I'm OK with having the exception bubble up to the
top. If the recipe is bailing out with an exception, we pretty much need to know
immediately. If it's bailing on presubmit, then it's basically unsubmittable
code anyhow. Unless you feel strongly here, I'm inclined to leave it as is.

+
+        if not func(name, recipe_path, *args, **kwargs):
+          had_errors = True
+  return had_errors
+
+
+def main(argv):
+  if not os.path.exists(SLAVE_DIR):
+    os.makedirs(SLAVE_DIR)
+
+  os.chdir(SLAVE_DIR)
+
+  training = False
+  if '--help' in argv or '-h' in argv:

[Vadim Shtayura, 2013/05/11 00:33:32]

Use ArgumentParser or OptionParser?

[iannucci, 2013/05/11 04:12:27]

Unfortunately we can't because of the way that unittest.main works :/. unittest
will barf if it sees the extra arg (so we have to delete from argv), and
optparse doesn't like to let not exit after printing help (so you have to catch
SystemExit). It turns out to be 2x the lines.

This is the least-bad way I've found to add arguments to a unittest, but I
agree, it sucks.

+    print 'Pass --train to enter training mode.'
+    print
+  elif '--train' in argv:
+    argv.remove('--train')
+    training = True
+
+  had_errors = loop_over_recipes(train_from_tests) if training else False
+
+  retcode = 1 if had_errors else 0
+
+  if not training:
+    try:
+      unittest.main()
+    except SystemExit as e:
+      retcode = e.code or retcode
+
+  total_covered = COVERAGE.report()
+  if total_covered != 100.0:
+    print 'FATAL: Recipes are not at 100% coverage.'
+    retcode = retcode or 2
+
+  return retcode
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))