Add crash detection to test report generation.

generate_test_report.py

 #!/usr/bin/python
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 """Parses and displays the contents of one or more autoserv result directories.
 
 This script parses the contents of one or more autoserv results folders and
 generates test reports.
@@ skipping 95 matching lines @@
     status = 'FAIL'
     if (re.search(r'GOOD.+completed successfully', status_raw) and
         not re.search(r'ABORT|ERROR|FAIL|TEST_NA', status_raw)):
       status = 'PASS'
 
     perf = self._CollectPerf(testdir)
 
     if testdir.startswith(self._options.strip):
       testdir = testdir.replace(self._options.strip, '', 1)
 
-    self._results[testdir] = {'status': status,
+    crashes = []
+    regex = re.compile('Received crash notification for ([-\w]+).+ (sig \d+)')
+    for match in regex.finditer(status_raw):
+      crashes.append('%s %s' % match.groups())
+
+    self._results[testdir] = {'crashes': crashes,
+                              'status': status,
                               'perf': perf}
 
   def _CollectResultsRec(self, resdir):
     """Recursively collect results into the self._results dictionary.
 
     Args:
       resdir: results/test directory to parse results from and recurse into.
     """
 
     self._CollectResult(resdir)
@@ skipping 42 matching lines @@
     results.
     """
     tests = self._results.keys()
     tests.sort()
 
     tests_with_errors = []
 
     width = self.GetTestColumnWidth()
     line = ''.ljust(width + 5, '-')
 
+    crashes = {}
     tests_pass = 0
     print line
     for test in tests:
       # Emit the test/status entry first
       test_entry = test.ljust(width)
       result = self._results[test]
       status_entry = result['status']
       if status_entry == 'PASS':
         color = Color.GREEN
         tests_pass += 1
       else:
         color = Color.RED
         tests_with_errors.append(test)
 
       status_entry = self._color.Color(color, status_entry)
       print test_entry + status_entry
 
       # Emit the perf keyvals entries. There will be no entries if the
       # --no-perf option is specified.
       perf = result['perf']
       perf_keys = perf.keys()
       perf_keys.sort()
 
       for perf_key in perf_keys:
         perf_key_entry = perf_key.ljust(width - self._KEYVAL_INDENT)
         perf_key_entry = perf_key_entry.rjust(width)
         perf_value_entry = self._color.Color(Color.BOLD, perf[perf_key])
         print perf_key_entry + perf_value_entry
 
+      # Ignore top-level entry, since it's just a combination of all the
+      # individual results.
+      if result['crashes'] and test != tests[0]:
+        for crash in result['crashes']:
+          if not crash in crashes:
+            crashes[crash] = set([])
+          crashes[crash].add(test)
+
     print line
 
     total_tests = len(tests)
     percent_pass = 100 * tests_pass / total_tests
     pass_str = '%d/%d (%d%%)' % (tests_pass, total_tests, percent_pass)
     print 'Total PASS: ' + self._color.Color(Color.BOLD, pass_str)
 
+    if self._options.crash_detection:
+      print ''
+      if crashes:
+        print self._color.Color(Color.RED, 'Crashes detected during testing:')
+        print line
+
+        for crash_name, crashed_tests in sorted(crashes.iteritems()):
+          print self._color.Color(Color.RED, crash_name)
+          for crashed_test in crashed_tests:
+            print ' '*self._KEYVAL_INDENT + crashed_test
+
+        print line
+        print 'Total unique crashes: ' + self._color.Color(Color.BOLD,
+                                                           str(len(crashes)))
+      else:
+        print self._color.Color(Color.GREEN,
+                                'No crashes detected during testing.')
+
     # Print out the client debug information for failed tests.
     if self._options.print_debug:
       for test in tests_with_errors:
         debug_file_regex = os.path.join(self._options.strip, test, 'debug',
                                        '%s*.DEBUG' % os.path.basename(test))
         for path in glob.glob(debug_file_regex):
           try:
             fh = open(path)
             print >> sys.stderr, (
                 '\n========== DEBUG FILE %s FOR TEST %s ==============\n' % (
@@ skipping 11 matching lines @@
 
       # Sometimes the builders exit before these buffers are flushed.
       sys.stderr.flush()
       sys.stdout.flush()
 
   def Run(self):
     """Runs report generation."""
     self._CollectResults()
     self._GenerateReportText()
     for v in self._results.itervalues():
-      if v['status'] != 'PASS':
+      if v['status'] != 'PASS' or (self._options.crash_detection
+                                   and v['crashes']):
         sys.exit(1)
 
 
 def main():
   usage = 'Usage: %prog [options] result-directories...'
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('--color', dest='color', action='store_true',
                     default=_STDOUT_IS_TTY,
                     help='Use color for text reports [default if TTY stdout]')
   parser.add_option('--no-color', dest='color', action='store_false',
                     help='Don\'t use color for text reports')
+  parser.add_option('--no-crash-detection', dest='crash_detection',
+                    action='store_false', default=True,
+                    help='Don\'t report crashes or error out when detected')
   parser.add_option('--perf', dest='perf', action='store_true',
                     default=True,
                     help='Include perf keyvals in the report [default]')
   parser.add_option('--no-perf', dest='perf', action='store_false',
                     help='Don\'t include perf keyvals in the report')
   parser.add_option('--strip', dest='strip', type='string', action='store',
                     default='results.',
                     help='Strip a prefix from test directory names'
                     ' [default: \'%default\']')
   parser.add_option('--no-strip', dest='strip', const='', action='store_const',
                     help='Don\'t strip a prefix from test directory names')
   parser.add_option('--no-debug', dest='print_debug', action='store_false',
                     default=True,
-                    help='Do not print out the debug log when a test fails.')
+                    help='Don\'t print out the debug log when a test fails.')
   (options, args) = parser.parse_args()
 
   if not args:
     parser.print_help()
     Die('no result directories provided')
 
   generator = ReportGenerator(options, args)
   generator.Run()
 
 
 if __name__ == '__main__':
   main()