Add crash detection to test report generation.

generate_test_report.py

 #!/usr/bin/python
 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 
 """Parses and displays the contents of one or more autoserv result directories.
 
 This script parses the contents of one or more autoserv results folders and
 generates test reports.
@@ skipping 95 matching lines @@
     status = 'FAIL'
     if (re.search(r'GOOD.+completed successfully', status_raw) and
         not re.search(r'ABORT|ERROR|FAIL|TEST_NA', status_raw)):
       status = 'PASS'
 
     perf = self._CollectPerf(testdir)
 
     if testdir.startswith(self._options.strip):
       testdir = testdir.replace(self._options.strip, '', 1)
 
-    self._results[testdir] = {'status': status,
+    crashes = []
+    regex = re.compile('(Received crash notification for .+)')
+    for match in regex.finditer(status_raw):
+      crashes.append(match.group(1).split('\t')[0])

[petkov, 2011/02/16 00:53:56]

what does this do? group(1) is the whole string, right? then split by \t, and
[0] would be "Received crash notification..."?

[DaleCurtis, 2011/02/16 00:59:00]

Autotest attaches the timetamp and a couple other things on the end of the
string.

On 2011/02/16 00:53:56, petkov wrote:

[petkov, 2011/02/16 01:01:17]

Sorry, I still don't get what this does... Can you copy/paste some sample
expected input and output? Thanks.

[DaleCurtis, 2011/02/16 01:03:54]

Line in status.log is this:
"INFO    desktopui_ChromeFirstRender     Received crash notification for
Xorg[9945] sig 11 (ignoring - no consent)       timestamp=1297813740   
localtime=Feb 15 15:49:00"

Regex returns:
"Received crash notification for Xorg[9945] sig 11 (ignoring - no consent)      
timestamp=1297813740    localtime=Feb 15 15:49:00"

We want:
"Received crash notification for Xorg[9945] sig 11 (ignoring - no consent)"

On 2011/02/16 01:01:17, petkov wrote:

[petkov, 2011/02/16 01:06:03]

Thanks. It seems "Received crash notification for " is verbose and redundant.
Maybe replace with just "crash".
Also, (ignoring - no consent) is redundant.

[DaleCurtis, 2011/02/16 01:18:49]

Done.

+
+    self._results[testdir] = {'crashes': crashes,
+                              'status': status,
                               'perf': perf}
 
   def _CollectResultsRec(self, resdir):
     """Recursively collect results into the self._results dictionary.
 
     Args:
       resdir: results/test directory to parse results from and recurse into.
     """
 
     self._CollectResult(resdir)
@@ skipping 42 matching lines @@
     results.
     """
     tests = self._results.keys()
     tests.sort()
 
     tests_with_errors = []
 
     width = self.GetTestColumnWidth()
     line = ''.ljust(width + 5, '-')
 
+    crashes_detected = False
     tests_pass = 0
     print line
     for test in tests:
       # Emit the test/status entry first
       test_entry = test.ljust(width)
       result = self._results[test]
       status_entry = result['status']
       if status_entry == 'PASS':
         color = Color.GREEN
         tests_pass += 1
       else:
         color = Color.RED
         tests_with_errors.append(test)
 
       status_entry = self._color.Color(color, status_entry)
       print test_entry + status_entry
 
       # Emit the perf keyvals entries. There will be no entries if the
       # --no-perf option is specified.
       perf = result['perf']
       perf_keys = perf.keys()
       perf_keys.sort()
 
       for perf_key in perf_keys:
         perf_key_entry = perf_key.ljust(width - self._KEYVAL_INDENT)
         perf_key_entry = perf_key_entry.rjust(width)
         perf_value_entry = self._color.Color(Color.BOLD, perf[perf_key])
         print perf_key_entry + perf_value_entry
 
+      if result['crashes']:
+        crashes_detected = True
+
     print line
 
     total_tests = len(tests)
     percent_pass = 100 * tests_pass / total_tests
     pass_str = '%d/%d (%d%%)' % (tests_pass, total_tests, percent_pass)
     print 'Total PASS: ' + self._color.Color(Color.BOLD, pass_str)
 
+    if self._options.crash_detection:
+      print ''

[petkov, 2011/02/16 00:53:56]

Isn't it better to interleave the crash results into the main results table?
Then you see the data for each test in one place (PASS/FAIL, perf, crashes...)

[DaleCurtis, 2011/02/16 00:59:00]

I thought about this, but cmasone indicated he'd like a strong separation
between the test results and the crash results. I'm okay with interleaving them
if he has no objections.

On 2011/02/16 00:53:56, petkov wrote:

+      if crashes_detected:
+        num_crashes = 0
+        print self._color.Color(Color.RED, 'Crashes detected during testing:')
+        print line
+
+        # Ignore top-level entry, since it's just a combination of all the
+        # individual results.
+        for test in tests[1:]:
+          crashes = self._results[test]['crashes']
+          if not crashes:
+            continue
+          print test
+          for crash in crashes:
+            num_crashes += 1
+            print ' '*self._KEYVAL_INDENT + self._color.Color(Color.RED, crash)
+
+        print line
+        print 'Total crashes: ' + self._color.Color(Color.BOLD,
+                                                    str(num_crashes))
+      else:
+        print self._color.Color(Color.GREEN,
+                                'No crashes detected during testing.')
+
     # Print out the client debug information for failed tests.
     if self._options.print_debug:
       for test in tests_with_errors:
         debug_file_regex = os.path.join(self._options.strip, test, 'debug',
                                        '%s*.DEBUG' % os.path.basename(test))
         for path in glob.glob(debug_file_regex):
           try:
             fh = open(path)
             print >> sys.stderr, (
                 '\n========== DEBUG FILE %s FOR TEST %s ==============\n' % (
@@ skipping 11 matching lines @@
 
       # Sometimes the builders exit before these buffers are flushed.
       sys.stderr.flush()
       sys.stdout.flush()
 
   def Run(self):
     """Runs report generation."""
     self._CollectResults()
     self._GenerateReportText()
     for v in self._results.itervalues():
-      if v['status'] != 'PASS':
+      if v['status'] != 'PASS' or (self._options.crash_detection
+                                   and v['crashes']):
         sys.exit(1)
 
 
 def main():
   usage = 'Usage: %prog [options] result-directories...'
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('--color', dest='color', action='store_true',
                     default=_STDOUT_IS_TTY,
                     help='Use color for text reports [default if TTY stdout]')
   parser.add_option('--no-color', dest='color', action='store_false',
                     help='Don\'t use color for text reports')
+  parser.add_option('--no-crash-detection', dest='crash_detection',
+                    action='store_false', default=True,
+                    help='Don\'t report crashes or error out when detected')
   parser.add_option('--perf', dest='perf', action='store_true',
                     default=True,
                     help='Include perf keyvals in the report [default]')
   parser.add_option('--no-perf', dest='perf', action='store_false',
                     help='Don\'t include perf keyvals in the report')
   parser.add_option('--strip', dest='strip', type='string', action='store',
                     default='results.',
                     help='Strip a prefix from test directory names'
                     ' [default: \'%default\']')
   parser.add_option('--no-strip', dest='strip', const='', action='store_const',
                     help='Don\'t strip a prefix from test directory names')
   parser.add_option('--no-debug', dest='print_debug', action='store_false',
                     default=True,
-                    help='Do not print out the debug log when a test fails.')
+                    help='Don\'t print out the debug log when a test fails.')
   (options, args) = parser.parse_args()
 
   if not args:
     parser.print_help()
     Die('no result directories provided')
 
   generator = ReportGenerator(options, args)
   generator.Run()
 
 
 if __name__ == '__main__':
   main()