rebaseline_server: add --compare-configs option

gm/rebaseline_server/compare_to_expectations.py

Index: gm/rebaseline_server/compare_to_expectations.py
diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py
index e0fa23fb2d6556a4d4dc26e21ced87f9ef6871f0..44138e739b2b230a27612db090b63e982bb7537a 100755
--- a/gm/rebaseline_server/compare_to_expectations.py
+++ b/gm/rebaseline_server/compare_to_expectations.py
@@ -47,16 +47,12 @@ EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
     results.KEY__EXPECTATIONS__IGNOREFAILURE,
     results.KEY__EXPECTATIONS__REVIEWED,
 ]
+DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
 
 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
 
-DEFAULT_ACTUALS_DIR = '.gm-actuals'
-DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
-DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
-    PARENT_DIRECTORY, '.generated-images')
-
 
-class Results(object):
+class ExpectationComparisons(results.BaseComparisons):

[epoger, 2014/03/27 21:31:07]

I moved the common code (used by both ExpectationComparisons and
ConfigComparisons) into a shared BaseComparisons class in results.py.

   """ Loads actual and expected GM results into an ImagePairSet.
 
   Loads actual and expected results from all builders, except for those skipped
@@ -64,11 +60,11 @@ class Results(object):
 
   Once this object has been constructed, the results (in self._results[])
   are immutable.  If you want to update the results based on updated JSON
-  file contents, you will need to create a new Results object."""
+  file contents, you will need to create a new ExpectationComparisons object."""
 
-  def __init__(self, actuals_root=DEFAULT_ACTUALS_DIR,
+  def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
                expected_root=DEFAULT_EXPECTATIONS_DIR,
-               generated_images_root=DEFAULT_GENERATED_IMAGES_ROOT,
+               generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
                diff_base_url=None):
     """
     Args:
@@ -92,19 +88,13 @@ class Results(object):
     logging.info('Results complete; took %d seconds.' %
                  (self._timestamp - time_start))
 
-  def get_timestamp(self):
-    """Return the time at which this object was created, in seconds past epoch
-    (UTC).
-    """
-    return self._timestamp
-
   def edit_expectations(self, modifications):
     """Edit the expectations stored within this object and write them back
     to disk.
 
     Note that this will NOT update the results stored in self._results[] ;
-    in order to see those updates, you must instantiate a new Results object
-    based on the (now updated) files on disk.
+    in order to see those updates, you must instantiate a new
+    ExpectationComparisons object based on the (now updated) files on disk.
 
     Args:
       modifications: a list of dictionaries, one for each expectation to update:
@@ -127,7 +117,8 @@ class Results(object):
          ]
 
     """
-    expected_builder_dicts = Results._read_dicts_from_root(self._expected_root)
+    expected_builder_dicts = ExpectationComparisons._read_dicts_from_root(
+        self._expected_root)
     for mod in modifications:
       image_name = results.IMAGE_FILENAME_FORMATTER % (
           mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
@@ -152,125 +143,8 @@ class Results(object):
         builder_expectations = {}
         builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations
       builder_expectations[image_name] = new_expectations
-    Results._write_dicts_to_root(expected_builder_dicts, self._expected_root)
-
-  def get_results_of_type(self, results_type):
-    """Return results of some/all tests (depending on 'results_type' parameter).
-
-    Args:
-      results_type: string describing which types of results to include; must
-          be one of the RESULTS_* constants
-
-    Results are returned in a dictionary as output by ImagePairSet.as_dict().
-    """
-    return self._results[results_type]
-
-  def get_packaged_results_of_type(self, results_type, reload_seconds=None,
-                                   is_editable=False, is_exported=True):
-    """ Package the results of some/all tests as a complete response_dict.
-
-    Args:
-      results_type: string indicating which set of results to return;
-          must be one of the RESULTS_* constants
-      reload_seconds: if specified, note that new results may be available once
-          these results are reload_seconds old
-      is_editable: whether clients are allowed to submit new baselines
-      is_exported: whether these results are being made available to other
-          network hosts
-    """
-    response_dict = self._results[results_type]
-    time_updated = self.get_timestamp()
-    response_dict[results.KEY__HEADER] = {
-        results.KEY__HEADER__SCHEMA_VERSION: (
-            results.REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
-
-        # Timestamps:
-        # 1. when this data was last updated
-        # 2. when the caller should check back for new data (if ever)
-        results.KEY__HEADER__TIME_UPDATED: time_updated,
-        results.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
-            (time_updated+reload_seconds) if reload_seconds else None),
-
-        # The type we passed to get_results_of_type()
-        results.KEY__HEADER__TYPE: results_type,
-
-        # Hash of dataset, which the client must return with any edits--
-        # this ensures that the edits were made to a particular dataset.
-        results.KEY__HEADER__DATAHASH: str(hash(repr(
-            response_dict[imagepairset.KEY__IMAGEPAIRS]))),
-
-        # Whether the server will accept edits back.
-        results.KEY__HEADER__IS_EDITABLE: is_editable,
-
-        # Whether the service is accessible from other hosts.
-        results.KEY__HEADER__IS_EXPORTED: is_exported,
-    }
-    return response_dict
-
-  @staticmethod
-  def _ignore_builder(builder):
-    """Returns True if we should ignore expectations and actuals for a builder.
-
-    This allows us to ignore builders for which we don't maintain expectations
-    (trybots, Valgrind, ASAN, TSAN), and avoid problems like
-    https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
-    produces error when trying to add baselines for ASAN/TSAN builders')
-
-    Args:
-      builder: name of this builder, as a string
-
-    Returns:
-      True if we should ignore expectations and actuals for this builder.
-    """
-    return (builder.endswith('-Trybot') or
-            ('Valgrind' in builder) or
-            ('TSAN' in builder) or
-            ('ASAN' in builder))
-
-  @staticmethod
-  def _read_dicts_from_root(root, pattern='*.json'):
-    """Read all JSON dictionaries within a directory tree.
-
-    Args:
-      root: path to root of directory tree
-      pattern: which files to read within root (fnmatch-style pattern)
-
-    Returns:
-      A meta-dictionary containing all the JSON dictionaries found within
-      the directory tree, keyed by the builder name of each dictionary.
-
-    Raises:
-      IOError if root does not refer to an existing directory
-    """
-    if not os.path.isdir(root):
-      raise IOError('no directory found at path %s' % root)
-    meta_dict = {}
-    for dirpath, dirnames, filenames in os.walk(root):
-      for matching_filename in fnmatch.filter(filenames, pattern):
-        builder = os.path.basename(dirpath)
-        if Results._ignore_builder(builder):
-          continue
-        fullpath = os.path.join(dirpath, matching_filename)
-        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
-    return meta_dict
-
-  @staticmethod
-  def _create_relative_url(hashtype_and_digest, test_name):
-    """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
-
-    If we don't have a record of this image, returns None.
-
-    Args:
-      hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
-          don't have a record of this image
-      test_name: string; name of the GM test that created this image
-    """
-    if not hashtype_and_digest:
-      return None
-    return gm_json.CreateGmRelativeUrl(
-        test_name=test_name,
-        hash_type=hashtype_and_digest[0],
-        hash_digest=hashtype_and_digest[1])
+    ExpectationComparisons._write_dicts_to_root(
+        expected_builder_dicts, self._expected_root)
 
   @staticmethod
   def _write_dicts_to_root(meta_dict, root, pattern='*.json'):
@@ -300,7 +174,7 @@ class Results(object):
     for dirpath, dirnames, filenames in os.walk(root):
       for matching_filename in fnmatch.filter(filenames, pattern):
         builder = os.path.basename(dirpath)
-        if Results._ignore_builder(builder):
+        if ExpectationComparisons._ignore_builder(builder):
           continue
         per_builder_dict = meta_dict.get(builder)
         if per_builder_dict is not None:
@@ -325,10 +199,12 @@ class Results(object):
     """
     logging.info('Reading actual-results JSON files from %s...' %
                  self._actuals_root)
-    actual_builder_dicts = Results._read_dicts_from_root(self._actuals_root)
+    actual_builder_dicts = ExpectationComparisons._read_dicts_from_root(
+        self._actuals_root)
     logging.info('Reading expected-results JSON files from %s...' %
                  self._expected_root)
-    expected_builder_dicts = Results._read_dicts_from_root(self._expected_root)
+    expected_builder_dicts = ExpectationComparisons._read_dicts_from_root(
+        self._expected_root)
 
     all_image_pairs = imagepairset.ImagePairSet(
         descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
@@ -366,9 +242,10 @@ class Results(object):
           continue
         for image_name in sorted(results_of_this_type.keys()):
           (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
-          actual_image_relative_url = Results._create_relative_url(
-              hashtype_and_digest=results_of_this_type[image_name],
-              test_name=test)
+          actual_image_relative_url = (
+              ExpectationComparisons._create_relative_url(
+                  hashtype_and_digest=results_of_this_type[image_name],
+                  test_name=test))
 
           # Default empty expectations; overwrite these if we find any real ones
           expectations_per_test = None
@@ -383,9 +260,10 @@ class Results(object):
             expected_image_hashtype_and_digest = (
                 expectations_per_test
                 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0])
-            expected_image_relative_url = Results._create_relative_url(
-                hashtype_and_digest=expected_image_hashtype_and_digest,
-                test_name=test)
+            expected_image_relative_url = (
+                ExpectationComparisons._create_relative_url(
+                    hashtype_and_digest=expected_image_hashtype_and_digest,
+                    test_name=test))
             expectations_dict = {}
             for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
               expectations_dict[field] = expectations_per_test.get(field)
@@ -465,7 +343,7 @@ def main():
                       level=logging.INFO)
   parser = argparse.ArgumentParser()
   parser.add_argument(
-      '--actuals', default=DEFAULT_ACTUALS_DIR,
+      '--actuals', default=results.DEFAULT_ACTUALS_DIR,
       help='Directory containing all actual-result JSON files; defaults to '
       '\'%(default)s\' .')
   parser.add_argument(
@@ -482,13 +360,13 @@ def main():
       str([results.KEY__HEADER__RESULTS_FAILURES,
            results.KEY__HEADER__RESULTS_ALL]))
   parser.add_argument(
-      '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT,
+      '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
       help='Directory within which to download images and generate diffs; '
       'defaults to \'%(default)s\' .')
   args = parser.parse_args()
-  results_obj = Results(actuals_root=args.actuals,
-                        expected_root=args.expectations,
-                        generated_images_root=args.workdir)
+  results_obj = ExpectationComparisons(actuals_root=args.actuals,
+                                       expected_root=args.expectations,
+                                       generated_images_root=args.workdir)
   gm_json.WriteToFile(
       results_obj.get_packaged_results_of_type(results_type=args.results),
       args.outfile)