HTTP GM results viewer: server now returns category summaries along with testData

gm/rebaseline_server/results.py

 #!/usr/bin/python
 
 '''
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 '''
 
 '''
@@ skipping 13 matching lines @@
 # that directory.  That script allows us to parse the actual-results.json file
 # written out by the GM tool.
 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
 # so any dirs that are already in the PYTHONPATH will be preferred.
 GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 if GM_DIRECTORY not in sys.path:
   sys.path.append(GM_DIRECTORY)
 import gm_json
 
 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+CATEGORIES_TO_SUMMARIZE = [
+    'builder', 'test', 'config', 'resultType',
+]
 
 class Results(object):
   """ Loads actual and expected results from all builders, supplying combined
   reports as requested. """
 
   def __init__(self, actuals_root, expected_root):
     """
     params:
       actuals_root: root directory containing all actual-results.json files
       expected_root: root directory containing all expected-results.json files
     """
     self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
     self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
-    self._all_results = self._Combine()
+    self._all_results = Results._Combine(
+        actual_builder_dicts=self._actual_builder_dicts,
+        expected_builder_dicts=self._expected_builder_dicts)
 
   def GetAll(self):
-    """Return results of all tests, as a list in this form:
+    """Return results of all tests, as a dictionary in this form:
 
-       [
+       {
+         "categories": # dictionary of category totals, keyed by category name

[epoger, 2013/09/27 18:12:39]

This "categories" dict is the new part.  By including this summary, we allow the
web client to present sorting/filtering UIs without having to parse every test
result.

[borenet, 2013/09/27 19:22:01]

I'm confused - "resultType" and "builder" are the categories?  Can we add some
more comments?  It looks like only "resultType" is being used.

[epoger, 2013/09/27 21:38:13]

I updated the docstring to (hopefully) make it clearer.

Currently, only "resultType" is being used by view.html, but I plan to change
that in another CL soon.

[borenet, 2013/09/30 13:06:02]

Thanks Elliot.  This is much clearer.

+                       # and value
          {
-           "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
-           "test": "bigmatrix",
-           "config": "8888",
-           "resultType": "failed",
-           "expectedHashType": "bitmap-64bitMD5",
-           "expectedHashDigest": "10894408024079689926",
-           "actualHashType": "bitmap-64bitMD5",
-           "actualHashDigest": "2409857384569",
-         },
-         ...
-       ]
+           "resultType": # category name, one of CATEGORIES_TO_SUMMARIZE
+           {
+             "failed": 3, # category value and total number found of that value
+             "failure-ignored": 19,
+             "no-comparison": 120,
+             "succeeded": 1932,
+           },
+           "builder":
+           {
+             "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug": 180,
+             "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release": 180,

[borenet, 2013/09/27 19:22:01]

What does 180 mean?  Is that the total number of results for this builder?

[epoger, 2013/09/27 21:38:13]

That's what the number at that position represents, yes.  180 was a wild guess,
which was off by an order of magnitude.  I have updated all the numbers in this
docstring, using current results.

+             ...
+           },
+         }, # end of "categories" dictionary
+
+         "testData": # list of test results, with a dictionary for each
+         [
+           {
+             "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
+             "test": "bigmatrix",
+             "config": "8888",
+             "resultType": "failed",
+             "expectedHashType": "bitmap-64bitMD5",
+             "expectedHashDigest": "10894408024079689926",
+             "actualHashType": "bitmap-64bitMD5",
+             "actualHashDigest": "2409857384569",
+           },
+           ...
+         ], # end of "testData" list
+       }
     """
     return self._all_results
 
   @staticmethod
   def _GetDictsFromRoot(root, pattern='*.json'):
     """Read all JSON dictionaries within a directory tree, returning them within
     a meta-dictionary (keyed by the builder name for each dictionary).
 
     params:
       root: path to root of directory tree
       pattern: which files to read within root (fnmatch-style pattern)
     """
     meta_dict = {}
     for dirpath, dirnames, filenames in os.walk(root):
       for matching_filename in fnmatch.filter(filenames, pattern):
         builder = os.path.basename(dirpath)
         if builder.endswith('-Trybot'):
           continue
         fullpath = os.path.join(dirpath, matching_filename)
         meta_dict[builder] = gm_json.LoadFromFile(fullpath)
     return meta_dict
 
-  def _Combine(self):
-    """Returns a list of all tests, across all builders, based on the
-    contents of self._actual_builder_dicts and self._expected_builder_dicts .
-    Returns the list in the same form needed for GetAllResults().
+  @staticmethod
+  def _Combine(actual_builder_dicts, expected_builder_dicts):
+    """Gathers the results of all tests, across all builders (based on the
+    contents of actual_builder_dicts and expected_builder_dicts)
+    and returns it in a list in the same form needed for self.GetAll().
+
+    This is a static method, because once we start refreshing results
+    asynchronously, we need to make sure we are not corrupting the object's
+    member variables.
     """
-    all_tests = []
-    for builder in sorted(self._actual_builder_dicts.keys()):
+    test_data = []
+    category_dict = {}
+    for builder in sorted(actual_builder_dicts.keys()):
       actual_results_for_this_builder = (
-          self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
+          actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
       for result_type in sorted(actual_results_for_this_builder.keys()):
         results_of_this_type = actual_results_for_this_builder[result_type]
         if not results_of_this_type:
           continue
         for image_name in sorted(results_of_this_type.keys()):
           actual_image = results_of_this_type[image_name]
           try:
             # TODO(epoger): assumes a single allowed digest per test
             expected_image = (
-                self._expected_builder_dicts
+                expected_builder_dicts
                     [builder][gm_json.JSONKEY_EXPECTEDRESULTS]
                     [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
                     [0])
           except (KeyError, TypeError):
             # There are several cases in which we would expect to find
             # no expectations for a given test:
             #
             # 1. result_type == NOCOMPARISON
             #   There are no expectations for this test yet!
             #
@@ skipping 36 matching lines @@
           # as ignoreFailure but it still shows up in the "failed"
           # category?  Maybe we should not rely on the result_type
           # categories recorded within the gm_actuals AT ALL, and
           # instead evaluate the result_type ourselves based on what
           # we see in expectations vs actual checksum?
           if expected_image == actual_image:
             updated_result_type = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
           else:
             updated_result_type = result_type
 
-          # TODO(epoger): For now, don't include succeeded results.
-          # There are so many of them that they make the client too slow.
-          if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
-            continue
-
           (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
-          all_tests.append({
+          results_for_this_test = {
               "builder": builder,
               "test": test,
               "config": config,
               "resultType": updated_result_type,
               "actualHashType": actual_image[0],
               "actualHashDigest": str(actual_image[1]),
               "expectedHashType": expected_image[0],
               "expectedHashDigest": str(expected_image[1]),
-          })
-    return all_tests
+          }
+          Results._AddToCategoryDict(category_dict, results_for_this_test)
+
+          # TODO(epoger): For now, don't include succeeded results in the raw
+          # data. There are so many of them that they make the client too slow.
+          if updated_result_type != gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
+            test_data.append(results_for_this_test)
+    return {"categories": category_dict, "testData": test_data}
+
+  @staticmethod
+  def _AddToCategoryDict(category_dict, test_results):
+    """Add test_results to the category dictionary we are building
+    (see documentation of self.GetAll() for the format of this dictionary).
+
+    params:
+      category_dict: category dict-of-dicts to add to; modify this in-place
+      test_results: test data with which to update category_list, in a dict:
+         {
+           "category_name": "category_value",
+           "category_name": "category_value",
+           ...
+         }
+    """
+    for category in CATEGORIES_TO_SUMMARIZE:
+      category_value = test_results.get(category)
+      if not category_value:
+        continue  # test_results did not include this category, keep going
+      if not category_dict.get(category):
+        category_dict[category] = {}
+      if not category_dict[category].get(category_value):
+        category_dict[category][category_value] = 0
+      category_dict[category][category_value] += 1