simulation_test_ng: initial CL

unittests/test_test.py

+#!/usr/bin/env python
+# Copyright 2017 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT_DIR)
+import recipe_engine.env
+
+
+from recipe_engine import package
+from recipe_engine import package_pb2
+
+
+class TestTest(unittest.TestCase):
+  def setUp(self):
+    root_dir = tempfile.mkdtemp()
+    config_dir = os.path.join(root_dir, 'infra', 'config')
+    os.makedirs(config_dir)
+
+    self._root_dir = root_dir
+    self._recipes_cfg = os.path.join(config_dir, 'recipes.cfg')
+    self._recipe_tool = os.path.join(ROOT_DIR, 'recipes.py')
+
+    test_pkg = package_pb2.Package(
+        api_version=1,
+        project_id='test_pkg',
+        recipes_path='',
+        deps=[
+            package_pb2.DepSpec(
+                project_id='recipe_engine',
+                url='file://'+ROOT_DIR),
+        ],
+    )
+    package.ProtoFile(self._recipes_cfg).write(test_pkg)
+
+  def tearDown(self):
+    shutil.rmtree(self._root_dir)
+
+  def _run_recipes(self, *args):
+    return subprocess.check_output((
+        sys.executable,
+        self._recipe_tool,
+        '--use-bootstrap',
+        '--package', self._recipes_cfg,
+    ) + args)
+
+  def _write_recipe(self, name, deps, step_lines, expect, test_lines=None):
+    if not test_lines:
+      test_lines = ['yield api.test("basic")']
+    recipes_dir = os.path.join(self._root_dir, 'recipes')
+    expect_dir = os.path.join(recipes_dir, '%s.expected' % name)
+    for d in (recipes_dir, expect_dir):
+      if not os.path.exists(d):
+        os.makedirs(d)
+    with open(os.path.join(recipes_dir, '%s.py' % name), 'w') as f:
+      f.write('\n'.join([
+        'from recipe_engine import post_process',
+        '',
+        'DEPS = %r' % deps,
+        '',
+        'def RunSteps(api):',
+      ] + ['  %s' % l for l in step_lines] + [
+        '',
+        'def GenTests(api):',
+      ] + ['  %s' % l for l in test_lines]))
+    with open(os.path.join(expect_dir, 'basic.json'), 'w') as f:
+      json.dump(expect, f)
+
+  def test_list(self):
+    self._write_recipe(
+        'foo', [], ['pass'],
+        [{'name': '$result', 'recipe_result': None, 'status_code': 0}])

[iannucci, 2017/03/08 17:59:20]

This invocation is pretty cryptic. Wdyt about

r = self.Recipe('foo')
# any of these can be omitted
r.Deps = [
  "foo/bar",
]
r.RunStepsBody = """
  pass
"""
r.GenTestsBody = """
  yield api.test('basic')
"""
r.Expectations['basic.json'] = {
  ...
}
r.write()

I think something like that will make it much clearer what's going on, and will
make us much more likely to add more tests in the future :).

+    self.assertEqual(
+        ['foo.basic'],
+        self._run_recipes('test', 'list').splitlines())
+
+  def test_test(self):
+    self._write_recipe(
+        'foo', [], ['pass'],
+        [{'name': '$result', 'recipe_result': None, 'status_code': 0}])
+    self._run_recipes('test', 'run')
+
+  def test_test_diff_failure(self):
+    self._write_recipe(
+        'foo', [], ['pass'],
+        None)
+    with self.assertRaises(subprocess.CalledProcessError) as cm:
+      self._run_recipes('test', 'run')
+    self.assertNotIn('FATAL: Insufficient coverage', cm.exception.output)
+    self.assertNotIn('CHECK(FAIL)', cm.exception.output)
+    self.assertIn(
+        'foo.basic failed',
+        cm.exception.output)
+
+  def test_test_recipe_not_covered(self):
+    self._write_recipe(
+        'foo', [], ['if False:', '  pass'],
+        [{'name': '$result', 'recipe_result': None, 'status_code': 0}])
+    with self.assertRaises(subprocess.CalledProcessError) as cm:
+      self._run_recipes('test', 'run')
+    self.assertIn('FATAL: Insufficient coverage', cm.exception.output)
+    self.assertNotIn('CHECK(FAIL)', cm.exception.output)
+    self.assertNotIn('foo.basic failed', cm.exception.output)
+
+  def test_test_check_failure(self):
+    self._write_recipe(
+        'foo', [], ['pass'],
+        [{'name': '$result', 'recipe_result': None, 'status_code': 0}],
+        test_lines=['yield api.test("basic") + api.post_process(post_process.MustRun, "bar")'])
+    with self.assertRaises(subprocess.CalledProcessError) as cm:
+      self._run_recipes('test', 'run')
+    self.assertNotIn('FATAL: Insufficient coverage', cm.exception.output)
+    self.assertIn('CHECK(FAIL)', cm.exception.output)
+    self.assertIn('foo.basic failed', cm.exception.output)
+
+
+if __name__ == '__main__':
+  sys.exit(unittest.main())