Create HTTP-based GM results viewer.

gm/rebaseline_server/results.py

+#!/usr/bin/python
+
+'''
+Copyright 2013 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+'''
+Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
+'''
+
+# System-level imports
+import fnmatch
+import json
+import os
+import re
+import sys
+
+# Imports from within Skia
+#
+# We need to add the 'gm' directory, so that we can import gm_json.py within
+# that directory.  That script allows us to parse the actual-results.json file
+# written out by the GM tool.
+# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+if GM_DIRECTORY not in sys.path:
+  sys.path.append(GM_DIRECTORY)
+import gm_json
+
+IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+
+class Results(object):
+  """ Loads actual and expected results from all builders, supplying combined
+  reports as requested. """
+
+  def __init__(self, actuals_root, expected_root):
+    """
+    params:
+      actuals_root: root directory containing all actual-results.json files
+      expected_root: root directory containing all expected-results.json files
+    """
+    self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
+    self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
+
+  @staticmethod
+  def _GetDictsFromRoot(root, pattern='*.json'):
+    """Read all JSON dictionaries within a directory tree, returning them within
+    a meta-dictionary (keyed by the builder type for each dictionary).

[borenet, 2013/09/25 15:08:08]

nit: "builder name"

[epoger, 2013/09/25 16:21:18]

Done.

+
+    params:
+      root: path to root of directory tree
+      pattern: which files to read within root (fnmatch-style pattern)
+    """
+    meta_dict = {}
+    for dirpath, dirnames, filenames in os.walk(root):
+      for matching_filename in fnmatch.filter(filenames, pattern):
+        builder = os.path.basename(dirpath)
+        if builder.endswith('-Trybot'):
+          continue
+        fullpath = os.path.join(dirpath, matching_filename)
+        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+    return meta_dict
+
+  def OfType(self, result_type):

[borenet, 2013/09/25 15:08:08]

Is this the only method which gets called on Results?  If so, can we just index
the results by result_type initially, so that this function (which is called
many times) can do much less work?

[epoger, 2013/09/25 16:21:18]

That's a good point.  Actually, it opens the wider discussion that I need to
have about this: I think we should make the server return ALL result types (or
at least all types except for success), because the user may want to jump around
within those different result types during the rebaselining process.

In other words... I think it probably makes the most sense for the client side
to make a single query--"give me all results that might need rebaselining:
failed, failure_ignored, and missing_expectations"--and then filter that view
within the client.

What do you think?

[epoger, 2013/09/25 18:26:33]

Brian and I agree that this is the way to go (in a future CL, if not now): The
client will make a single request to get results of ALL types--even success, if
it doesn't take ridiculously long to load them all--and then the view will allow
the user to filter, sort, etc.  By default, the filters will be set to show only
failing tests.

[borenet, 2013/09/25 19:28:17]

I agree.  The page will be more interactive that way.

+    """Returns an array of all tests, across all builders, of a certain
+    result_type.  Returns the array in this form:
+
+       [
+         {
+           "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+           "test": "bigmatrix",
+           "config": "8888",
+           "expectedHashType": "bitmap-64bitMD5",
+           "expectedHashDigest": "10894408024079689926",
+           "actualHashType": "bitmap-64bitMD5",
+           "actualHashDigest": "2409857384569",
+         },
+         ...
+       ]
+
+    params:
+      result_type: string describing result type, such as 'failed'; should be
+                   one of the gm_json.JSONKEY_ACTUALRESULTS_* strings.
+    """
+    # TODO(epoger): validate that result_type is one of the
+    # gm_json.JSONKEY_ACTUALRESULTS_* strings?

[borenet, 2013/09/25 15:08:08]

If not, please change line 93 to use .get() instead of [].

[epoger, 2013/09/25 16:21:18]

Done.

+    array = []
+    for builder in sorted(self._actual_builder_dicts.keys()):
+      results_of_this_type = (
+          self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]
+                                    [result_type])
+      if not results_of_this_type:
+        continue
+      for image_name in sorted(results_of_this_type.keys()):
+        actual_image = results_of_this_type[image_name]
+        try:
+          # TODO(epoger): assumes a single allowed digest per test
+          expected_image = (
+              self._expected_builder_dicts
+                  [builder][gm_json.JSONKEY_EXPECTEDRESULTS]
+                  [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
+                  [0])
+        except (KeyError, TypeError):
+          # It's OK to find no expectations, if this is a NOCOMPARISON test!
+          # But we still want to look for expectations, in case someone has
+          # committed expectations and the bots just haven't tested them yet.
+          if result_type != gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON:
+            raise
+          expected_image = [None, None]
+
+        # If this test was recently rebaselined, it will remain in the "failed"
+        # set of actuals until all the bots have cycled (although the
+        # expectations have indeed been set from the most recent actuals).
+        # Weed out such cases.
+        if expected_image == actual_image:
+          continue
+
+        (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
+        array.append({
+            "builder": builder,
+            "test": test,
+            "config": config,
+            "actualHashType": actual_image[0],
+            "actualHashDigest": str(actual_image[1]),
+            "expectedHashType": expected_image[0],
+            "expectedHashDigest": str(expected_image[1]),
+        })
+    return array