Print actual list of failing tests on Android layout test steps

build/android/buildbot/bb_device_steps.py

 #!/usr/bin/env python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import collections
 import glob
 import hashlib
+import json
 import multiprocessing
 import os
 import random
 import re
 import shutil
 import sys
 
 import bb_utils
 import bb_annotations
 
@@ skipping 231 matching lines @@
   for f in options.factory_properties.get('additional_expectations', []):
     cmd_args.extend(
         ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)])
 
   # TODO(dpranke): Remove this block after
   # https://codereview.chromium.org/12927002/ lands.
   for f in options.factory_properties.get('additional_expectations_files', []):
     cmd_args.extend(
         ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)])
 
-  RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args)
+  exit_code = RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] +
+                     cmd_args)
+  if exit_code == 254:  # AKA -1, internal error.
+    bb_annotations.PrintMsg("?? (crashed or hung)")
+  else:
+    full_results_path = os.path.join('..', 'layout-test-results',
+                                     'full_results.json')
+    if os.path.exists(full_results_path):
+      full_results = json.load(open(full_results_path))
+      unexpected_failures, unexpected_flakes, unexpected_passes = \

[Isaac (away), 2013/10/09 18:27:59]

Extend lines using parens, not backslash, per google python conventions.

Could you run this through gpylint and see if anything else obvious comes up?

+          _ParseLayoutTestResults(full_results)
+      if unexpected_failures:
+        _PrintDashboardLink("failed", unexpected_failures,
+                            max_tests=25)
+      elif unexpected_passes:
+        _PrintDashboardLink("unexpected passes", unexpected_passes,
+                            max_tests=10)
+      if unexpected_flakes:
+        _PrintDashboardLink("unexpected flakes", unexpected_flakes,
+                            max_tests=10)
+    else:
+      bb_annotations.PrintMsg("?? (results missing)")
+
 
   if options.factory_properties.get('archive_webkit_results', False):
     bb_annotations.PrintNamedStep('archive_webkit_results')
     base = 'https://storage.googleapis.com/chromium-layout-test-archives'
     builder_name = options.build_properties.get('buildername', '')
     build_number = str(options.build_properties.get('buildnumber', ''))
     bb_annotations.PrintLink('results',
         '%s/%s/%s/layout-test-results/results.html' % (
         base, EscapeBuilderName(builder_name), build_number))
     bb_annotations.PrintLink('(zip)', '%s/%s/%s/layout-test-results.zip' % (
         base, EscapeBuilderName(builder_name), build_number))
     gs_bucket = 'gs://chromium-layout-test-archives'
     RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'chromium',
                          'archive_layout_test_results.py'),
         '--results-dir', '../../layout-test-results',
         '--build-dir', CHROME_OUT_DIR,
         '--build-number', build_number,
         '--builder-name', builder_name,
         '--gs-bucket', gs_bucket])
 
 
+def _ParseLayoutTestResults(results):
+  # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results
+  tests = _ConvertTrieToFlatPaths(results['tests'])
+  failures = {}
+  flakes = {}
+  passes = {}
+  for (test, result) in tests.iteritems():
+    if result.get('is_unexpected'):
+      actual_result = result['actual']
+      if ' PASS' in actual_result:
+        flakes[test] = actual_result
+      elif actual_result == 'PASS':
+        passes[test] = result
+      else:
+        failures[test] = actual_result
+
+  return (passes, failures, flakes)
+
+
+def _ConvertTrieToFlatPaths(trie, prefix=None):
+  # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results
+  result = {}
+  for name, data in trie.iteritems():
+    if prefix:
+      name = prefix + "/" + name
+
+    if len(data) and not "actual" in data and not "expected" in data:
+      result.update(_ConvertTrieToFlatPaths(data, name))
+    else:
+      result[name] = data
+
+  return result
+
+
+def _PrintDashboardLink(link_text, tests, max_tests):
+  if len(tests) > max_tests:
+    test_list_text = ' '.join(tests[:max_tests]) + ' and more'
+  else:
+    test_list_text = ' '.join(tests)
+
+  DASHBOARD_BASE = ("http://test-results.appspot.com"
+                    "/dashboards/flakiness_dashboard.html#"
+                    "master=ChromiumWebkit&tests=")
+
+  bb_annotations.PrintLink('%d %s: %s' %
+                           (len(tests), link_text, test_list_text),
+                           DASHBOARD_BASE + ','.join(tests))
+
+
 def EscapeBuilderName(builder_name):
   return re.sub('[ ()]', '_', builder_name)
 
 
 def SpawnLogcatMonitor():
   shutil.rmtree(LOGCAT_DIR, ignore_errors=True)
   bb_utils.SpawnCmd([
       os.path.join(CHROME_SRC_DIR, 'build', 'android', 'adb_logcat_monitor.py'),
       LOGCAT_DIR])
 
@@ skipping 199 matching lines @@
   setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
   if options.coverage_bucket:
     setattr(options, 'coverage_dir',
             os.path.join(CHROME_OUT_DIR, options.target, 'coverage'))
 
   MainTestWrapper(options)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))