rebaseline_server: generate JSON that can be viewed without a live server

gm/rebaseline_server/results.py

 #!/usr/bin/python
 

[epoger, 2014/03/13 14:37:04]

Probably.  If it's OK with you, I'll wait until I fix those broken image embeds
to get this worked out.

By the way, I have an idea for supplying the broken image embeds:  For now, keep
the specialized server.py running just to serve out those images.  (The JSON can
be loaded from Google Storage, and the web client will contact the special
server to download the diff images.)  I may take a different tack later, but for
now that will allow us to generate multiple JSONs (expected vs actual at HEAD,
expected vs actual at a particular revision, gpu vs cpu, etc.) and serve them
from Google Storage.

What do you think of that idea, for the time being at least?

 """
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
 """
 
 # System-level imports
 import argparse
 import fnmatch
 import json
 import logging
 import os
 import re
 import sys
 import time
 
 # Imports from within Skia
 #
 # We need to add the 'gm' directory, so that we can import gm_json.py within
 # that directory.  That script allows us to parse the actual-results.json file
 # written out by the GM tool.
 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
 # so any dirs that are already in the PYTHONPATH will be preferred.
 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
+TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
 if GM_DIRECTORY not in sys.path:
   sys.path.append(GM_DIRECTORY)
 import gm_json
 import imagediffdb
 import imagepair
 import imagepairset
 
 # Keys used to link an image to a particular GM test.
 # NOTE: Keep these in sync with static/constants.js
 KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
 KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
 KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
 KEY__EXTRACOLUMN__BUILDER = 'builder'
 KEY__EXTRACOLUMN__CONFIG = 'config'
 KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType'
 KEY__EXTRACOLUMN__TEST = 'test'
+KEY__HEADER = 'header'
+KEY__HEADER__DATAHASH = 'dataHash'
+KEY__HEADER__IS_EDITABLE = 'isEditable'
+KEY__HEADER__IS_EXPORTED = 'isExported'
+KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
 KEY__HEADER__RESULTS_ALL = 'all'
 KEY__HEADER__RESULTS_FAILURES = 'failures'
+KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
+KEY__HEADER__TIME_UPDATED = 'timeUpdated'
+KEY__HEADER__TYPE = 'type'
 KEY__NEW_IMAGE_URL = 'newImageUrl'
 KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
 KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
 KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
 KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
 
 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
     KEY__EXPECTATIONS__BUGS,
     KEY__EXPECTATIONS__IGNOREFAILURE,
     KEY__EXPECTATIONS__REVIEWED,
 ]
 
 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
 IMAGE_FILENAME_FORMATTER = '%s_%s.png'  # pass in (testname, config)
 
 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
 
+DEFAULT_ACTUALS_DIR = '.gm-actuals'
+DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
+DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(PARENT_DIRECTORY, 'static',
+                                             'generated-images')
+
 
 class Results(object):
   """ Loads actual and expected GM results into an ImagePairSet.
 
   Loads actual and expected results from all builders, except for those skipped
   by _ignore_builder().
 
   Once this object has been constructed, the results (in self._results[])
   are immutable.  If you want to update the results based on updated JSON
   file contents, you will need to create a new Results object."""
 
-  def __init__(self, actuals_root, expected_root, generated_images_root):
+  def __init__(self, actuals_root=DEFAULT_ACTUALS_DIR,
+               expected_root=DEFAULT_EXPECTATIONS_DIR,
+               generated_images_root=DEFAULT_GENERATED_IMAGES_ROOT):
     """
     Args:
       actuals_root: root directory containing all actual-results.json files
       expected_root: root directory containing all expected-results.json files
       generated_images_root: directory within which to create all pixel diffs;
           if this directory does not yet exist, it will be created
     """
     time_start = int(time.time())
     self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
     self._actuals_root = actuals_root
@@ skipping 55 matching lines @@
           new_expectations[field] = value
       builder_dict = expected_builder_dicts[
           mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__BUILDER]]
       builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
       if not builder_expectations:
         builder_expectations = {}
         builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations
       builder_expectations[image_name] = new_expectations
     Results._write_dicts_to_root(expected_builder_dicts, self._expected_root)
 
-  def get_results_of_type(self, type):
-    """Return results of some/all tests (depending on 'type' parameter).
+  def get_results_of_type(self, results_type):
+    """Return results of some/all tests (depending on 'results_type' parameter).
 
     Args:
-      type: string describing which types of results to include; must be one
-            of the RESULTS_* constants
+      results_type: string describing which types of results to include; must
+          be one of the RESULTS_* constants
 
     Results are returned in a dictionary as output by ImagePairSet.as_dict().
     """
-    return self._results[type]
+    return self._results[results_type]
+
+  def get_packaged_results_of_type(self, results_type, reload_seconds=None,

[epoger, 2014/03/12 21:08:35]

moved here from server.py

+                                   is_editable=False, is_exported=True):
+    """ Package the results of some/all tests as a complete response_dict.
+
+    Args:
+      results_type: string indicating which set of results to return;
+          must be one of the RESULTS_* constants
+      reload_seconds: if specified, note that new results may be available once
+          these results are reload_seconds old
+      is_editable: whether clients are allowed to submit new baselines
+      is_exported: whether these results are being made available to other
+          network hosts
+    """
+    response_dict = self._results[results_type]
+    time_updated = self.get_timestamp()
+    response_dict[KEY__HEADER] = {
+        # Timestamps:
+        # 1. when this data was last updated
+        # 2. when the caller should check back for new data (if ever)
+        KEY__HEADER__TIME_UPDATED: time_updated,
+        KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
+            (time_updated+reload_seconds) if reload_seconds else None),
+
+        # The type we passed to get_results_of_type()
+        KEY__HEADER__TYPE: results_type,
+
+        # Hash of dataset, which the client must return with any edits--
+        # this ensures that the edits were made to a particular dataset.
+        KEY__HEADER__DATAHASH: str(hash(repr(
+            response_dict[imagepairset.KEY__IMAGEPAIRS]))),
+
+        # Whether the server will accept edits back.
+        KEY__HEADER__IS_EDITABLE: is_editable,
+
+        # Whether the service is accessible from other hosts.
+        KEY__HEADER__IS_EXPORTED: is_exported,
+    }
+    return response_dict
 
   @staticmethod
   def _ignore_builder(builder):
     """Returns True if we should ignore expectations and actuals for a builder.
 
     This allows us to ignore builders for which we don't maintain expectations
     (trybots, Valgrind, ASAN, TSAN), and avoid problems like
     https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
     produces error when trying to add baselines for ASAN/TSAN builders')
 
@@ skipping 235 matching lines @@
       KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
     }
 
 
 def main():
   logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                       datefmt='%m/%d/%Y %H:%M:%S',
                       level=logging.INFO)
   parser = argparse.ArgumentParser()
   parser.add_argument(
-      '--actuals', required=True,
+      '--actuals', default=DEFAULT_ACTUALS_DIR,
       help='Directory containing all actual-result JSON files')
   parser.add_argument(
-      '--expectations', required=True,
-      help='Directory containing all expected-result JSON files')
+      '--expectations', default=DEFAULT_EXPECTATIONS_DIR,
+      help='Directory containing all expected-result JSON files; defaults to '
+      '\'%(default)s\' .')
   parser.add_argument(
       '--outfile', required=True,
-      help='File to write result summary into, in JSON format')
+      help='File to write result summary into, in JSON format.')
   parser.add_argument(
-      '--workdir', default='.workdir',
-      help='Directory within which to download images and generate diffs')
+      '--results', default=KEY__HEADER__RESULTS_FAILURES,
+      help='Which result types to include. Defaults to \'%(default)s\'; '
+      'must be one of ' +
+      str([KEY__HEADER__RESULTS_FAILURES, KEY__HEADER__RESULTS_ALL]))
+  parser.add_argument(
+      '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT,
+      help='Directory within which to download images and generate diffs; '
+      'defaults to \'%(default)s\' .')

[rmistry, 2014/03/13 13:11:46]

This is cool, I did not know you could do this to get the default value.

   args = parser.parse_args()
   results = Results(actuals_root=args.actuals,
                     expected_root=args.expectations,
                     generated_images_root=args.workdir)
-  gm_json.WriteToFile(results.get_results_of_type(KEY__HEADER__RESULTS_ALL),
-                      args.outfile)
+  gm_json.WriteToFile(
+      results.get_packaged_results_of_type(results_type=args.results),
+      args.outfile)
 
 
 if __name__ == '__main__':
   main()