Create HTTP-based GM results viewer.

gm/rebaseline_server/results.py

+#!/usr/bin/python
+
+'''
+Copyright 2013 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+'''
+Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
+'''
+
+# System-level imports
+import fnmatch
+import json
+import os
+import re
+import sys
+
+# Imports from within Skia
+#
+# We need to add the 'gm' directory, so that we can import gm_json.py within
+# that directory.  That script allows us to parse the actual-results.json file
+# written out by the GM tool.
+# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+if GM_DIRECTORY not in sys.path:
+  sys.path.append(GM_DIRECTORY)
+import gm_json
+
+IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+
+class Results(object):
+  """ Loads actual and expected results from all builders, supplying combined
+  reports as requested. """
+
+  def __init__(self, actuals_root, expected_root):
+    """
+    params:
+      actuals_root: root directory containing all actual-results.json files
+      expected_root: root directory containing all expected-results.json files
+    """
+    self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
+    self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
+    self._all_results = self._Combine()
+
+  def GetAll(self):
+    """Return results of all tests, as a list in this form:
+
+       [
+         {
+           "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+           "test": "bigmatrix",
+           "config": "8888",
+           "resultType": "failed",
+           "expectedHashType": "bitmap-64bitMD5",
+           "expectedHashDigest": "10894408024079689926",
+           "actualHashType": "bitmap-64bitMD5",
+           "actualHashDigest": "2409857384569",
+         },
+         ...
+       ]
+    """
+    return self._all_results
+
+  @staticmethod
+  def _GetDictsFromRoot(root, pattern='*.json'):
+    """Read all JSON dictionaries within a directory tree, returning them within
+    a meta-dictionary (keyed by the builder name for each dictionary).
+
+    params:
+      root: path to root of directory tree
+      pattern: which files to read within root (fnmatch-style pattern)
+    """
+    meta_dict = {}
+    for dirpath, dirnames, filenames in os.walk(root):
+      for matching_filename in fnmatch.filter(filenames, pattern):
+        builder = os.path.basename(dirpath)
+        if builder.endswith('-Trybot'):
+          continue
+        fullpath = os.path.join(dirpath, matching_filename)
+        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+    return meta_dict
+
+  def _Combine(self):
+    """Returns a list of all tests, across all builders, based on the
+    contents of self._actual_builder_dicts and self._expected_builder_dicts .
+    Returns the list in the same form needed for GetAllResults().
+    """
+    all_tests = []
+    for builder in sorted(self._actual_builder_dicts.keys()):
+      actual_results_for_this_builder = (
+          self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+      for result_type in sorted(actual_results_for_this_builder.keys()):
+        results_of_this_type = actual_results_for_this_builder[result_type]
+        if not results_of_this_type:
+          continue
+        for image_name in sorted(results_of_this_type.keys()):
+          actual_image = results_of_this_type[image_name]
+          try:
+            # TODO(epoger): assumes a single allowed digest per test
+            expected_image = (
+                self._expected_builder_dicts
+                    [builder][gm_json.JSONKEY_EXPECTEDRESULTS]
+                    [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
+                    [0])
+          except (KeyError, TypeError):
+            # There are several cases in which we would expect to find
+            # no expectations for a given test:
+            #
+            # 1. result_type == NOCOMPARISON
+            #   There are no expectations for this test yet!
+            #
+            # 2. ignore-tests.txt
+            #   If a test has been listed in ignore-tests.txt, then its status
+            #   may show as FAILUREIGNORED even if it doesn't have any
+            #   expectations yet.
+            #
+            # 3. alternate rendering mode failures (e.g. serialized)
+            #   In cases like
+            #   https://code.google.com/p/skia/issues/detail?id=1684
+            #   ('tileimagefilter GM test failing in serialized render mode'),
+            #   the gm-actuals will list a failure for the alternate
+            #   rendering mode even though we don't have explicit expectations
+            #   for the test (the implicit expectation is that it must
+            #   render the same in all rendering modes).
+            #
+            # Don't log types 1 or 2, because they are common.
+            # Log other types, because they are rare and we should know about
+            # them, but don't throw an exception, because we need to keep our
+            # tools working in the meanwhile!
+            if result_type not in [
+                gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
+                gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED] :
+              print 'WARNING: No expectations found for test: %s' % {
+                  'builder': builder,
+                  'image_name': image_name,
+                  'result_type': result_type,
+                  }
+            expected_image = [None, None]
+
+          # If this test was recently rebaselined, it will remain in
+          # the "failed" set of actuals until all the bots have
+          # cycled (although the expectations have indeed been set
+          # from the most recent actuals).  Treat these as successes
+          # instead of failures.
+          #
+          # TODO(epoger): Do we need to do something similar in
+          # other cases, such as when we have recently marked a test
+          # as ignoreFailure but it still shows up in the "failed"
+          # category?  Maybe we should not rely on the result_type
+          # categories recorded within the gm_actuals AT ALL, and
+          # instead evaluate the result_type ourselves based on what
+          # we see in expectations vs actual checksum?
+          if expected_image == actual_image:
+            updated_result_type = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
+          else:
+            updated_result_type = result_type
+
+          # TODO(epoger): For now, don't include succeeded results.
+          # There are so many of them that they make the client too slow.
+          if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
+            continue
+
+          (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
+          all_tests.append({
+              "builder": builder,
+              "test": test,
+              "config": config,
+              "resultType": updated_result_type,
+              "actualHashType": actual_image[0],
+              "actualHashDigest": str(actual_image[1]),
+              "expectedHashType": expected_image[0],
+              "expectedHashDigest": str(expected_image[1]),
+          })
+    return all_tests