rebaseline_server: cache results in long-running ImageDiffDB instance

gm/rebaseline_server/compare_to_expectations.py

 #!/usr/bin/python
 
 """
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
 """
 
 # System-level imports
 import argparse
 import fnmatch
-import json
 import logging
 import os
-import re
-import sys
 import time
 
+# Must fix up PYTHONPATH before importing from within Skia
+# pylint: disable=W0611
+import fix_pythonpath
+# pylint: enable=W0611
+
 # Imports from within Skia
-import fix_pythonpath  # must do this first
 from pyutils import url_utils
 import gm_json
 import imagediffdb
 import imagepair
 import imagepairset
 import results
 
 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
     results.KEY__EXPECTATIONS__BUGS,
     results.KEY__EXPECTATIONS__IGNOREFAILURE,
     results.KEY__EXPECTATIONS__REVIEWED,
 ]
 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
 DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt'
 
 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
 
 
 class ExpectationComparisons(results.BaseComparisons):
   """Loads actual and expected GM results into an ImagePairSet.
 
   Loads actual and expected results from all builders, except for those skipped
   by _ignore_builder().
 
   Once this object has been constructed, the results (in self._results[])
   are immutable.  If you want to update the results based on updated JSON
   file contents, you will need to create a new ExpectationComparisons object."""
 
-  def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
+  def __init__(self, image_diff_db, actuals_root=results.DEFAULT_ACTUALS_DIR,
                expected_root=DEFAULT_EXPECTATIONS_DIR,
                ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE,
-               generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
                diff_base_url=None, builder_regex_list=None):
     """
     Args:
+      image_diff_db: instance of ImageDiffDB we use to cache the image diffs
       actuals_root: root directory containing all actual-results.json files
       expected_root: root directory containing all expected-results.json files
       ignore_failures_file: if a file with this name is found within
           expected_root, ignore failures for any tests listed in the file
-      generated_images_root: directory within which to create all pixel diffs;
-          if this directory does not yet exist, it will be created
       diff_base_url: base URL within which the client should look for diff
           images; if not specified, defaults to a "file:///" URL representation
-          of generated_images_root
+          of image_diff_db's storage_root
       builder_regex_list: List of regular expressions specifying which builders
           we will process. If None, process all builders.
     """
     time_start = int(time.time())
     if builder_regex_list != None:
       self.set_match_builders_pattern_list(builder_regex_list)
-    self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
+    self._image_diff_db = image_diff_db
     self._diff_base_url = (
         diff_base_url or
-        url_utils.create_filepath_url(generated_images_root))
+        url_utils.create_filepath_url(image_diff_db.get_storage_root()))
     self._actuals_root = actuals_root
     self._expected_root = expected_root
     self._ignore_failures_on_these_tests = []
     if ignore_failures_file:
       self._ignore_failures_on_these_tests = (
           ExpectationComparisons._read_noncomment_lines(
               os.path.join(expected_root, ignore_failures_file)))
     self._load_actual_and_expected()
     self._timestamp = int(time.time())
     logging.info('Results complete; took %d seconds.' %
@@ skipping 75 matching lines @@
       pattern: which files to write within root (fnmatch-style pattern)
 
     Raises:
       IOError if root does not refer to an existing directory
       KeyError if the set of per-builder dictionaries written out was
                different than expected
     """
     if not os.path.isdir(root):
       raise IOError('no directory found at path %s' % root)
     actual_builders_written = []
-    for dirpath, dirnames, filenames in os.walk(root):
+    for dirpath, _, filenames in os.walk(root):
       for matching_filename in fnmatch.filter(filenames, pattern):
         builder = os.path.basename(dirpath)
         per_builder_dict = meta_dict.get(builder)
         if per_builder_dict is not None:
           fullpath = os.path.join(dirpath, matching_filename)
           gm_json.WriteToFile(per_builder_dict, fullpath)
           actual_builders_written.append(builder)
 
     # Check: did we write out the set of per-builder dictionaries we
     # expected to?
@@ skipping 147 matching lines @@
                 imageA_relative_url=expected_image_relative_url,
                 imageB_relative_url=actual_image_relative_url,
                 expectations=expectations_dict,
                 extra_columns=extra_columns_dict)
             all_image_pairs.add_image_pair(image_pair)
             if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
               failing_image_pairs.add_image_pair(image_pair)
           except Exception:
             logging.exception('got exception while creating new ImagePair')
 
+    # pylint: disable=W0201
     self._results = {
       results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
       results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
     }
 
 
 def main():
   logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                       datefmt='%m/%d/%Y %H:%M:%S',
                       level=logging.INFO)
@@ skipping 18 matching lines @@
       '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
       help='Which result types to include. Defaults to \'%(default)s\'; '
       'must be one of ' +
       str([results.KEY__HEADER__RESULTS_FAILURES,
            results.KEY__HEADER__RESULTS_ALL]))
   parser.add_argument(
       '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
       help='Directory within which to download images and generate diffs; '
       'defaults to \'%(default)s\' .')
   args = parser.parse_args()
+  image_diff_db = imagediffdb.ImageDiffDB(storage_root=args.workdir)
   results_obj = ExpectationComparisons(
+      image_diff_db=image_diff_db,
       actuals_root=args.actuals,
       expected_root=args.expectations,
-      ignore_failures_file=args.ignore_failures_file,
-      generated_images_root=args.workdir)
+      ignore_failures_file=args.ignore_failures_file)
   gm_json.WriteToFile(
       results_obj.get_packaged_results_of_type(results_type=args.results),
       args.outfile)
 
 
 if __name__ == '__main__':
   main()