Always identify test names that lead to failures

unittests/test_test.py

 #!/usr/bin/env python
 # Copyright 2017 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import json
 import os
 import shutil
 import subprocess
 import sys
@@ skipping 126 matching lines @@
       'version': 1,
       'valid': True,
     }
 
   def diff_failure(self, test):
     """Simulates a diff failure for |test|."""
     self._result.setdefault('test_failures', {}).setdefault(
         test, {'failures': []})['failures'].append({'diff_failure': {}})
     return self
 
+  def internal_failure(self, test):
+    """Simulates an internal failure for |test|."""
+    self._result.setdefault('test_failures', {}).setdefault(
+        test, {'failures': []})['failures'].append({'internal_failure': {}})
+    return self
+
   def check_failure(self, test, filename, lineno, func, args=None, kwargs=None,
                     name=None):
     """Simulates a check failure for |test|.
 
     Arguments:
       test(str): name of the test
       filename(str): path where check is introduced
       lineno(int): line number where check is introduced
       func(str): function/callable name of the check
       args(list): arguments for |func|
@@ skipping 264 matching lines @@
 
   def test_test_recipe_syntax_error(self):
     rw = RecipeWriter(os.path.join(self._root_dir, 'recipes'), 'foo')
     rw.RunStepsLines = ['baz']
     rw.add_expectation('basic')
     rw.write()
     with self.assertRaises(subprocess.CalledProcessError) as cm:
       self._run_recipes('test', 'run', '--json', self.json_path)
     self.assertIn('NameError: global name \'baz\' is not defined',
                   cm.exception.output)
-    self.assertFalse(self.json_contents.get('valid'))
+    self.assertEqual(
+        self.json_generator
+            .invalid()
+            .internal_failure('foo.basic')
+            .coverage_failure('recipes/foo.py', [6])
+            .unused_expectation('recipes/foo.expected')
+            .unused_expectation('recipes/foo.expected/basic.json').get(),
+        self.json_contents)
 
   def test_test_recipe_module_uncovered(self):
     mw = RecipeModuleWriter(self._root_dir, 'foo')
     mw.write()
     with self.assertRaises(subprocess.CalledProcessError) as cm:
       self._run_recipes('test', 'run', '--json', self.json_path)
     self.assertIn('The following modules lack test coverage: foo',
                   cm.exception.output)
     self.assertEqual(self.json_generator.uncovered_module('foo').get(),
                      self.json_contents)
 
   def test_test_recipe_module_syntax_error(self):
     mw = RecipeModuleWriter(self._root_dir, 'foo_module')
     mw.methods['foo'] = ['baz']
     mw.write()
     mw.example.DEPS = ['foo_module']
     mw.example.RunStepsLines = ['api.foo_module.foo()']
     mw.example.write()
     with self.assertRaises(subprocess.CalledProcessError) as cm:
       self._run_recipes('test', 'run', '--json', self.json_path)
     self.assertIn('NameError: global name \'baz\' is not defined',
                   cm.exception.output)
     self.assertIn('FATAL: Insufficient coverage', cm.exception.output)
-    self.assertFalse(self.json_contents.get('valid'))
+    self.assertEqual(
+        self.json_generator
+            .invalid()
+            .internal_failure('foo_module:example.basic')
+            .coverage_failure('recipe_modules/foo_module/api.py', [6])
+            .coverage_failure('recipe_modules/foo_module/example.py', [6])
+            .get(),
+        self.json_contents)
 
   def test_test_recipe_module_syntax_error_in_example(self):
     mw = RecipeModuleWriter(self._root_dir, 'foo_module')
     mw.methods['foo'] = ['pass']
     mw.write()
     mw.example.DEPS = ['foo_module']
     mw.example.RunStepsLines = ['baz']
     mw.example.write()
     with self.assertRaises(subprocess.CalledProcessError) as cm:
       self._run_recipes('test', 'run', '--json', self.json_path)
     self.assertIn('NameError: global name \'baz\' is not defined',
                   cm.exception.output)
     self.assertIn('FATAL: Insufficient coverage', cm.exception.output)
-    self.assertFalse(self.json_contents.get('valid'))
+    self.assertEqual(
+        self.json_generator
+            .invalid()
+            .internal_failure('foo_module:example.basic')
+            .coverage_failure('recipe_modules/foo_module/api.py', [6])
+            .coverage_failure('recipe_modules/foo_module/example.py', [6])
+            .get(),
+        self.json_contents)
 
   def test_test_recipe_module_example_not_covered(self):
     mw = RecipeModuleWriter(self._root_dir, 'foo_module')
     mw.methods['foo'] = ['pass']
     mw.write()
     mw.example.DEPS = ['foo_module']
     mw.example.RunStepsLines = ['if False:', '  pass']
     mw.example.write()
     with self.assertRaises(subprocess.CalledProcessError) as cm:
       self._run_recipes('test', 'run', '--json', self.json_path)
@@ skipping 379 matching lines @@
 
     g1.diff_failure('foo_diff')
     g2.diff_failure('foo_diff').diff_failure('bar_diff')
 
     g1.check_failure(
         'foo_check', 'foo_file', 1, 'foo_func')
     g2.check_failure(
         'foo_check', 'foo_file', 1, 'foo_func').check_failure(
             'bar_check', 'bar_file', 2, 'bar_func', ['bar_args'])
 
+    g1.internal_failure('foo_internal')
+    g2.internal_failure('foo_internal').internal_failure('bar_internal')
+
     g1.uncovered_module('foo_module')
     g2.uncovered_module('foo_module').uncovered_module('bar_module')
 
     g1.unused_expectation(
         'foo_expectation')
     g2.unused_expectation(
         'foo_expectation').unused_expectation('bar_expectation')
 
     with self.assertRaises(subprocess.CalledProcessError) as cm:
       self._run_recipes(
           'test', 'diff',
           '--baseline', g1.write(),
           '--actual', g2.write(),
           '--json', self.json_path)
     self.assertEqual(
         self.json_generator
             .coverage_failure('bar_coverage', [2])
             .diff_failure('bar_diff')
             .check_failure('bar_check', 'bar_file', 2, 'bar_func', ['bar_args'])
+            .internal_failure('bar_internal')
             .uncovered_module('bar_module')
             .unused_expectation('bar_expectation')
             .get(),
         self.json_contents)
 
 
 if __name__ == '__main__':
   sys.exit(unittest.main())