Create HTTP-based GM results viewer.

gm/rebaseline_server/results.py

+#!/usr/bin/python
+
+'''
+Copyright 2013 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+'''
+
+'''
+Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
+'''
+
+# System-level imports
+import fnmatch
+import json
+import os
+import re
+import sys
+
+# Imports from within Skia
+#
+# We need to add the 'gm' directory, so that we can import gm_json.py within
+# that directory.  That script allows us to parse the actual-results.json file
+# written out by the GM tool.
+# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
+# so any dirs that are already in the PYTHONPATH will be preferred.
+GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+if GM_DIRECTORY not in sys.path:
+  sys.path.append(GM_DIRECTORY)
+import gm_json
+
+IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+
+class Results(object):
+  """ Loads actual and expected results from all builders, supplying combined
+  reports as requested. """
+
+  def __init__(self, actuals_root, expected_root):
+    """
+    params:
+      actuals_root: root directory containing all actual-results.json files
+      expected_root: root directory containing all expected-results.json files
+    """
+    self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
+    self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
+    self._all_results = self._Combine()
+
+  def GetAll(self):

[epoger, 2013/09/26 17:26:51]

Replaced the old OfType(result_type) method with GetAll() , that returns results
of *all* types and lets the client filter/sort them as desired.

And generates that set of results ONCE, during __init__(), rather than waiting
until the client requests them.

[borenet, 2013/09/26 19:02:01]

If the client is going to filter by result type, would it be smarter to go ahead
and index by result type, so that the client's work is really fast?

[epoger, 2013/09/26 21:22:21]

The short answer: "Maybe, but I don't think we know yet.  I think we should
commit it as-is and then make adjustments."

The long answer:

- Currently, view.html has a dropdown at the very top of the page within which
the user chooses exactly one result type to see.  But, as noted in a TODO there,
that will change very soon; we want to allow more flexibility in this (for
example, show a mix of both FAILED and FAILUREIGNORED in the same table).

- Each result record includes the resultType, so it should be fairly
straightforward for the client view to be filtered with things exactly as they
are.

- I think we need to add lazy loading of images, so we can see how much of the
performance problems of long lists is due to image loading (as opposed to
processing the list itself).

[borenet, 2013/09/27 15:00:15]

All of the above SGTM

+    """Return results of all tests, as a list in this form:
+
+       [
+         {
+           "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+           "test": "bigmatrix",
+           "config": "8888",
+           "resultType": "failed",
+           "expectedHashType": "bitmap-64bitMD5",
+           "expectedHashDigest": "10894408024079689926",
+           "actualHashType": "bitmap-64bitMD5",
+           "actualHashDigest": "2409857384569",
+         },
+         ...
+       ]
+    """
+    return self._all_results
+
+  @staticmethod
+  def _GetDictsFromRoot(root, pattern='*.json'):
+    """Read all JSON dictionaries within a directory tree, returning them within
+    a meta-dictionary (keyed by the builder name for each dictionary).
+
+    params:
+      root: path to root of directory tree
+      pattern: which files to read within root (fnmatch-style pattern)
+    """
+    meta_dict = {}
+    for dirpath, dirnames, filenames in os.walk(root):
+      for matching_filename in fnmatch.filter(filenames, pattern):
+        builder = os.path.basename(dirpath)
+        if builder.endswith('-Trybot'):
+          continue
+        fullpath = os.path.join(dirpath, matching_filename)
+        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+    return meta_dict
+
+  def _Combine(self):
+    """Returns a list of all tests, across all builders, based on the
+    contents of self._actual_builder_dicts and self._expected_builder_dicts .
+    Returns the list in the same form needed for GetAllResults().
+    """
+    all_tests = []
+    for builder in sorted(self._actual_builder_dicts.keys()):

[epoger, 2013/09/26 17:26:51]

The codereview diff makes this seem more different than the prior patchset than
it really is.

Basically, rather than throwing away results unless they match result_type, we
gather them all together.

+      actual_results_for_this_builder = (
+          self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+      for result_type in sorted(actual_results_for_this_builder.keys()):
+        results_of_this_type = actual_results_for_this_builder[result_type]
+        if not results_of_this_type:
+          continue
+        for image_name in sorted(results_of_this_type.keys()):
+          actual_image = results_of_this_type[image_name]
+          try:
+            # TODO(epoger): assumes a single allowed digest per test
+            expected_image = (
+                self._expected_builder_dicts
+                    [builder][gm_json.JSONKEY_EXPECTEDRESULTS]
+                    [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
+                    [0])
+          except (KeyError, TypeError):
+            # There are several cases in which we would expect to find
+            # no expectations for a given test:
+            #
+            # 1. result_type == NOCOMPARISON
+            #   There are no expectations for this test yet!
+            #
+            # 2. ignore-tests.txt
+            #   If a test has been listed in ignore-tests.txt, then its status
+            #   may show as FAILUREIGNORED even if it doesn't have any
+            #   expectations yet.
+            #
+            # 3. alternate rendering mode failures (e.g. serialized)
+            #   In cases like
+            #   https://code.google.com/p/skia/issues/detail?id=1684
+            #   ('tileimagefilter GM test failing in serialized render mode'),
+            #   the gm-actuals will list a failure for the alternate
+            #   rendering mode even though we don't have explicit expectations
+            #   for the test (the implicit expectation is that it must
+            #   render the same in all rendering modes).
+            #
+            # Don't log types 1 or 2, because they are common.
+            # Log other types, because they are rare and we should know about
+            # them, but don't throw an exception, because we need to keep our
+            # tools working in the meanwhile!
+            if result_type not in [
+                gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
+                gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED] :
+              print 'WARNING: No expectations found for test: %s' % {
+                  'builder': builder,
+                  'image_name': image_name,
+                  'result_type': result_type,
+                  }
+            expected_image = [None, None]
+
+          # If this test was recently rebaselined, it will remain in
+          # the "failed" set of actuals until all the bots have
+          # cycled (although the expectations have indeed been set
+          # from the most recent actuals).  Treat these as successes
+          # instead of failures.
+          #
+          # TODO(epoger): Do we need to do something similar in
+          # other cases, such as when we have recently marked a test
+          # as ignoreFailure but it still shows up in the "failed"
+          # category?  Maybe we should not rely on the result_type
+          # categories recorded within the gm_actuals AT ALL, and
+          # instead evaluate the result_type ourselves based on what
+          # we see in expectations vs actual checksum?
+          if expected_image == actual_image:
+            updated_result_type = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
+          else:
+            updated_result_type = result_type
+
+          # TODO(epoger): For now, don't include succeeded results.

[epoger, 2013/09/26 17:26:51]

Note this temporary restriction to keep the tool usable, until we figure out how
to handle the performance issues around the large number of "succeeded" results.

+          # There are so many of them that they make the client too slow.
+          if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
+            continue
+
+          (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
+          all_tests.append({
+              "builder": builder,
+              "test": test,
+              "config": config,
+              "resultType": updated_result_type,
+              "actualHashType": actual_image[0],
+              "actualHashDigest": str(actual_image[1]),
+              "expectedHashType": expected_image[0],
+              "expectedHashDigest": str(expected_image[1]),
+          })
+    return all_tests