Insert summary results to the JSON. These are the same results spit to stdout...

webkit/tools/layout_tests/layout_package/json_results_generator.py

 # Copyright (c) 2009 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
 import re
 import subprocess
 import sys
 import time
 import xml.dom.minidom
 
 from layout_package import path_utils
 from layout_package import test_failures
 
 sys.path.append(path_utils.PathFromBase('third_party'))
 import simplejson
 
 class JSONResultsGenerator:
 
   MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 500
   # Min time (seconds) that will be added to the JSON.
   MIN_TIME = 1
   JSON_PREFIX = "ADD_RESULTS("
   JSON_SUFFIX = ");"
   WEBKIT_PATH = "WebKit"
   LAYOUT_TESTS_PATH = "layout_tests"
   PASS_RESULT = "P"
+  SKIP_RESULT = "X"
   NO_DATA_RESULT = "N"
   VERSION = 1
   VERSION_KEY = "version"
   RESULTS = "results"
   TIMES = "times"
   BUILD_NUMBERS = "buildNumbers"
   WEBKIT_SVN = "webkitRevision"
   CHROME_SVN = "chromeRevision"
   TIME = "secondsSinceEpoch"
   TESTS = "tests"
+  NON_WONTFIX = "nonWontfixCounts"
+  DEFERRED = "deferredCounts"
+  ALL = "allCounts"
+  FIXABLE_COUNT = "fixableCount"
+  FAILURE_CHARS = ["C", "T", "I", "S", "F", "O"]
 
   def __init__(self, failures, individual_test_timings, builder_name,
-               build_number, results_file_path, all_tests):
+      build_number, results_file_path, all_tests, result_summary):
     """
     failures: Map of test name to list of failures.
     individual_test_times: Map of test name to a tuple containing the
         test_run-time.
     builder_name: The name of the builder the tests are being run on.
     build_number: The build number for this run.
     results_file_path: Absolute path to the results json file.
     all_tests: List of all the tests that were run.
+    result_summary: ResultsSummary object containing failure counts for
+        different groups of tests.
     """
     # Make sure all test paths are relative to the layout test root directory.
     self._failures = {}
     for test in failures:
       test_path = self._GetPathRelativeToLayoutTestRoot(test)
       self._failures[test_path] = failures[test]
 
     self._all_tests = [self._GetPathRelativeToLayoutTestRoot(test)
         for test in all_tests]
 
+    self._result_summary = result_summary
+
     self._test_timings = {}
     for test_tuple in individual_test_timings:
       test_path = self._GetPathRelativeToLayoutTestRoot(test_tuple.filename)
       self._test_timings[test_path] = test_tuple.test_run_time
 
     self._builder_name = builder_name
     self._build_number = build_number
     self._results_file_path = results_file_path
 
   def _GetSVNRevision(self, in_directory=None):
@@ skipping 36 matching lines @@
       relativePath = test[index + 1:]
 
     # Make sure all paths are unix-style.
     return relativePath.replace('\\', '/')
 
   def GetJSON(self):
     """Gets the results for the results.json file."""
     failures_for_json = {}
     for test in self._failures:
       failures_for_json[test] = ResultAndTime(test, self._all_tests)
-      failures_for_json[test].result = self._GetResultsCharForFailure(test)
+      failures_for_json[test].result = self._GetResultsCharForTest(test)
 
     for test in self._test_timings:
       if not test in failures_for_json:
         failures_for_json[test] = ResultAndTime(test, self._all_tests)
       # Floor for now to get time in seconds.
       # TODO(ojan): As we make tests faster, reduce to tenth of a second
       # granularity.
       failures_for_json[test].time = int(self._test_timings[test])
 
     # If results file exists, read it out, put new info in it.
@@ skipping 40 matching lines @@
 
     path_to_chrome_base = path_utils.PathFromBase()
     self._InsertItemIntoRawList(results_for_builder,
         self._GetSVNRevision(path_to_chrome_base),
         self.CHROME_SVN)
 
     self._InsertItemIntoRawList(results_for_builder,
         int(time.time()),
         self.TIME)
 
+    self._InsertFailureSummaries(results_for_builder)
+
     for test in all_failing_tests:
       if test in failures_for_json:
         result_and_time = failures_for_json[test]
       else:
         result_and_time = ResultAndTime(test, self._all_tests)
 
       if test not in tests:
         tests[test] = self._CreateResultsAndTimesJSON()
 
       thisTest = tests[test]
       self._InsertItemRunLengthEncoded(result_and_time.result,
           thisTest[self.RESULTS])
       self._InsertItemRunLengthEncoded(result_and_time.time,
           thisTest[self.TIMES])
       self._NormalizeResultsJSON(thisTest, test, tests)
 
     # Specify separators in order to get compact encoding.
     results_str = simplejson.dumps(results_json, separators=(',', ':'))
     return self.JSON_PREFIX + results_str + self.JSON_SUFFIX
 
+  def _InsertFailureSummaries(self, results_for_builder):
+    """Inserts aggregate pass/failure statistics into the JSON.
+
+    Args:
+      results_for_builder: Dictionary containing the test results for a single
+          builder.
+    """
+    self._InsertItemIntoRawList(results_for_builder,
+        self._result_summary.fixable_count,
+        self.FIXABLE_COUNT)
+
+    self._InsertItemIntoRawList(results_for_builder,
+        self._GetFailureSummaryEntry(self._result_summary.deferred),
+        self.DEFERRED)
+    self._InsertItemIntoRawList(results_for_builder,
+        self._GetFailureSummaryEntry(self._result_summary.non_wontfix),
+        self.NON_WONTFIX)
+    self._InsertItemIntoRawList(results_for_builder,
+        self._GetFailureSummaryEntry(self._result_summary.all),
+        self.ALL)
+
+  def _GetFailureSummaryEntry(self, result_summary_entry):
+    """Creates a summary object to insert into the JSON.
+
+    Args:
+      result_summary_entry: ResultSummaryEntry for a group of tests
+          (e.g. deferred tests).
+    """
+    entry = {}
+    entry[self.SKIP_RESULT] = result_summary_entry.skip_count
+    entry[self.PASS_RESULT] = result_summary_entry.pass_count
+    for char in self.FAILURE_CHARS:
+      # There can be multiple failures that map to "O", so keep existing entry
+      # values if they already exist.
+      if char in entry:
+        count = entry[char]
+      else:
+        count = 0
+
+      for failure in result_summary_entry.failure_counts:
+        if char == self._GetResultsCharForFailure([failure]):
+          count = result_summary_entry.failure_counts[failure]
+      entry[char] = count
+    return entry
+
   def _InsertItemIntoRawList(self, results_for_builder, item, key):
     """Inserts the item into the list with the given key in the results for
     this builder. Creates the list if no such list exists.
 
     Args:
       results_for_builder: Dictionary containing the test results for a single
           builder.
       item: Number or string to insert into the list.
       key: Key in results_for_builder for the list to insert into.
     """
@@ skipping 50 matching lines @@
     results_and_times = {}
     results_and_times[self.RESULTS] = []
     results_and_times[self.TIMES] = []
     return results_and_times
 
   def _CreateResultsForBuilderJSON(self):
     results_for_builder = {}
     results_for_builder[self.TESTS] = {}
     return results_for_builder
 
-  def _GetResultsCharForFailure(self, test):
+  def _GetResultsCharForTest(self, test):
     """Returns the worst failure from the list of failures for this test
     since we can only show one failure per run for each test on the dashboard.
     """
     failures = [failure.__class__ for failure in self._failures[test]]
+    return self._GetResultsCharForFailure(failures)
 
+  def _GetResultsCharForFailure(self, failures):
+    """Returns the worst failure from the list of failures
+    since we can only show one failure per run for each test on the dashboard.
+    """
     if test_failures.FailureCrash in failures:
       return "C"
     elif test_failures.FailureTimeout in failures:
       return "T"
     elif test_failures.FailureImageHashMismatch in failures:
       return "I"
     elif test_failures.FailureSimplifiedTextMismatch in failures:
       return "S"
     elif test_failures.FailureTextMismatch in failures:
       return "F"
@@ skipping 50 matching lines @@
 
 class ResultAndTime:
   """A holder for a single result and runtime for a test."""
   def __init__(self, test, all_tests):
     self.time = 0
     # If the test was run, then we don't want to default the result to nodata.
     if test in all_tests:
       self.result = JSONResultsGenerator.PASS_RESULT
     else:
       self.result = JSONResultsGenerator.NO_DATA_RESULT