Create HTTP-based GM results viewer.

gm/rebaseline_server/results.py

+#!/usr/bin/python
+
+'''
+Copyright 2013 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+'''
+Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
+'''
+
+# System-level imports
+import fnmatch
+import json
+import os
+import re
+import sys
+
+# Imports from within Skia
+#
+# We need to add the 'gm' directory, so that we can import gm_json.py within
+# that directory.  That script allows us to parse the actual-results.json file
+# written out by the GM tool.
+# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+if GM_DIRECTORY not in sys.path:
+  sys.path.append(GM_DIRECTORY)
+import gm_json
+
+IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+
+class Results(object):
+  """ Loads actual and expected results from all builders, supplying combined
+  reports as requested. """
+
+  def __init__(self, actuals_root, expected_root):
+    """
+    params:
+      actuals_root: root directory containing all actual-results.json files
+      expected_root: root directory containing all expected-results.json files
+    """
+    self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
+    self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
+
+  @staticmethod
+  def _GetDictsFromRoot(root, pattern='*.json'):
+    """Read all JSON dictionaries within a directory tree, returning them within
+    a meta-dictionary (keyed by the builder type for each dictionary).
+
+    params:
+      root: path to root of directory tree
+      pattern: which files to read within root (fnmatch-style pattern)
+    """
+    meta_dict = {}
+    for dirpath, dirnames, filenames in os.walk(root):
+      for matching_filename in fnmatch.filter(filenames, pattern):
+        builder = os.path.basename(dirpath)
+        # EPOGER: make this check configurable?
+        if builder.endswith('-Trybot'):
+          continue
+        fullpath = os.path.join(dirpath, matching_filename)
+        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+    return meta_dict
+
+  def OfType(self, result_type):
+    """Returns an array of all tests, across all builders, of a certain
+    result_type.  Returns the array in this form:
+
+       [
+         {
+           "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+           "test": "bigmatrix",
+           "config": "8888",
+           "expectedHashType": "bitmap-64bitMD5",
+           "expectedHashDigest": "10894408024079689926",
+           "actualHashType": "bitmap-64bitMD5",
+           "actualHashDigest": "2409857384569",
+         },
+         ...
+       ]
+
+    params:
+      result_type: string describing result type, such as 'failed'; should be
+                   one of the gm_json.JSONKEY_ACTUALRESULTS_* strings.
+    """
+    # EPOGER: validate that result_type is one of the gm_json.JSONKEY_ACTUALRESULTS_* strings?
+    array = []
+    for builder in sorted(self._actual_builder_dicts.keys()):
+      results_of_this_type = self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS][result_type]
+      if not results_of_this_type:
+        continue
+      for image_name in sorted(results_of_this_type.keys()):
+        actual_image = results_of_this_type[image_name]
+        try:
+          # EPOGER: assumes a single allowed digest per test
+          expected_image = self._expected_builder_dicts[builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]
+        except (KeyError, TypeError):
+          # It's OK to find no expectations, if this is a NOCOMPARISON test!
+          # But we still want to look for expectations, in case someone has
+          # committed expectations and the bots just haven't tested them yet.
+          if result_type != gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON:
+            raise
+          expected_image = [None, None]
+
+        # If this test was recently rebaselined, it will remain in the "failed"
+        # set of actuals until all the bots have cycled (although the
+        # expectations have indeed been set from the most recent actuals).
+        # Weed out such cases.
+        # EPOGER: make this optional?
+        if expected_image == actual_image:
+          continue
+
+        # EPOGER: call re.match just once for both test and config?
+        array.append({
+            "builder": builder,
+            "test": IMAGE_FILENAME_RE.match(image_name).group(1),
+            "config": IMAGE_FILENAME_RE.match(image_name).group(2),
+            "actualHashType": actual_image[0],
+            "actualHashDigest": str(actual_image[1]),
+            "expectedHashType": expected_image[0],
+            "expectedHashDigest": str(expected_image[1]),
+        })
+    return array
+
+
+if __name__ == '__main__':
+  # EPOGER: actual_root should point at a separate checkout of gm-actual
+  # EPOGER: expected_root should point at a separately checked-out expectations/gm workspace
+  results = Results(
+      actuals_root='/usr/local/google/home/epoger/src/rebaseline-mockup/gm-actual',
+      expected_root=os.path.join(GM_DIRECTORY, os.pardir, 'expectations', 'gm'))
+  for result_type in ['failed', 'failure-ignored', 'no-comparison']:
+    gm_json.WriteToFile(results.OfType(result_type), os.path.join('EPOGER-results', result_type + '.json'))