Add crash detection to test report generation.

generate_test_report.py

 #!/usr/bin/python
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 """Parses and displays the contents of one or more autoserv result directories.
 
 This script parses the contents of one or more autoserv results folders and
 generates test reports.
@@ skipping 95 matching lines @@
     status = 'FAIL'
     if (re.search(r'GOOD.+completed successfully', status_raw) and
         not re.search(r'ABORT|ERROR|FAIL|TEST_NA', status_raw)):
       status = 'PASS'
 
     perf = self._CollectPerf(testdir)
 
     if testdir.startswith(self._options.strip):
       testdir = testdir.replace(self._options.strip, '', 1)
 
-    self._results[testdir] = {'status': status,
+    crashes = []
+    regex = re.compile('Received crash notification for (\w+).+ (sig \d+)')

[petkov, 2011/02/16 08:17:40]

btw, for the process name, you probably want to cover '-' too.
maybe \S+ instead of \w+?

[DaleCurtis, 2011/02/16 19:59:16]

\S+ would grab the process id and such too. Switched to [-\w]+

On 2011/02/16 08:17:40, petkov wrote:

+    for match in regex.finditer(status_raw):
+      crashes.append('%s %s' % match.groups())
+
+    self._results[testdir] = {'crashes': crashes,
+                              'status': status,
                               'perf': perf}
 
   def _CollectResultsRec(self, resdir):
     """Recursively collect results into the self._results dictionary.
 
     Args:
       resdir: results/test directory to parse results from and recurse into.
     """
 
     self._CollectResult(resdir)
@@ skipping 42 matching lines @@
     results.
     """
     tests = self._results.keys()
     tests.sort()
 
     tests_with_errors = []
 
     width = self.GetTestColumnWidth()
     line = ''.ljust(width + 5, '-')
 
+    crashes = {}
     tests_pass = 0
     print line
     for test in tests:
       # Emit the test/status entry first
       test_entry = test.ljust(width)
       result = self._results[test]
       status_entry = result['status']
       if status_entry == 'PASS':
         color = Color.GREEN
         tests_pass += 1
       else:
         color = Color.RED
         tests_with_errors.append(test)
 
       status_entry = self._color.Color(color, status_entry)
       print test_entry + status_entry
 
       # Emit the perf keyvals entries. There will be no entries if the
       # --no-perf option is specified.
       perf = result['perf']
       perf_keys = perf.keys()
       perf_keys.sort()
 
       for perf_key in perf_keys:
         perf_key_entry = perf_key.ljust(width - self._KEYVAL_INDENT)
         perf_key_entry = perf_key_entry.rjust(width)
         perf_value_entry = self._color.Color(Color.BOLD, perf[perf_key])
         print perf_key_entry + perf_value_entry
 
+      # Ignore top-level entry, since it's just a combination of all the
+      # individual results.
+      if result['crashes'] and not test == tests[0]:

[petkov, 2011/02/16 03:46:54]

isn't this test != tests[0]?

this filter is a bit hacky but i don't have any better suggestions...

[DaleCurtis, 2011/02/16 19:59:16]

Yup, changed. Python has me losing sense of basic operators.

On 2011/02/16 03:46:54, petkov wrote:

+        for crash in result['crashes']:
+          if not crash in crashes:
+            crashes[crash] = set([])
+          crashes[crash].add(test)
+
     print line
 
     total_tests = len(tests)
     percent_pass = 100 * tests_pass / total_tests
     pass_str = '%d/%d (%d%%)' % (tests_pass, total_tests, percent_pass)
     print 'Total PASS: ' + self._color.Color(Color.BOLD, pass_str)
 
+    if self._options.crash_detection:
+      print ''
+      if crashes:
+        print self._color.Color(Color.RED, 'Crashes detected during testing:')
+        print line
+
+        for crash_name, crashed_tests in crashes.iteritems():
+          print self._color.Color(Color.RED, crash_name)
+          for crashed_test in crashed_tests:
+            print ' '*self._KEYVAL_INDENT + crashed_test
+
+        print line
+        print 'Total unique crashes: ' + self._color.Color(Color.BOLD,
+                                                           str(len(crashes)))
+      else:
+        print self._color.Color(Color.GREEN,
+                                'No crashes detected during testing.')
+
     # Print out the client debug information for failed tests.
     if self._options.print_debug:
       for test in tests_with_errors:
         debug_file_regex = os.path.join(self._options.strip, test, 'debug',
                                        '%s*.DEBUG' % os.path.basename(test))
         for path in glob.glob(debug_file_regex):
           try:
             fh = open(path)
             print >> sys.stderr, (
                 '\n========== DEBUG FILE %s FOR TEST %s ==============\n' % (
@@ skipping 11 matching lines @@
 
       # Sometimes the builders exit before these buffers are flushed.
       sys.stderr.flush()
       sys.stdout.flush()
 
   def Run(self):
     """Runs report generation."""
     self._CollectResults()
     self._GenerateReportText()
     for v in self._results.itervalues():
-      if v['status'] != 'PASS':
+      if v['status'] != 'PASS' or (self._options.crash_detection
+                                   and v['crashes']):
         sys.exit(1)
 
 
 def main():
   usage = 'Usage: %prog [options] result-directories...'
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('--color', dest='color', action='store_true',
                     default=_STDOUT_IS_TTY,
                     help='Use color for text reports [default if TTY stdout]')
   parser.add_option('--no-color', dest='color', action='store_false',
                     help='Don\'t use color for text reports')
+  parser.add_option('--no-crash-detection', dest='crash_detection',
+                    action='store_false', default=True,
+                    help='Don\'t report crashes or error out when detected')
   parser.add_option('--perf', dest='perf', action='store_true',
                     default=True,
                     help='Include perf keyvals in the report [default]')
   parser.add_option('--no-perf', dest='perf', action='store_false',
                     help='Don\'t include perf keyvals in the report')
   parser.add_option('--strip', dest='strip', type='string', action='store',
                     default='results.',
                     help='Strip a prefix from test directory names'
                     ' [default: \'%default\']')
   parser.add_option('--no-strip', dest='strip', const='', action='store_const',
                     help='Don\'t strip a prefix from test directory names')
   parser.add_option('--no-debug', dest='print_debug', action='store_false',
                     default=True,
-                    help='Do not print out the debug log when a test fails.')
+                    help='Don\'t print out the debug log when a test fails.')
   (options, args) = parser.parse_args()
 
   if not args:
     parser.print_help()
     Die('no result directories provided')
 
   generator = ReportGenerator(options, args)
   generator.Run()
 
 
 if __name__ == '__main__':
   main()