rebaseline_server: rename results.py to compare_to_expectations.py

gm/rebaseline_server/compare_to_expectations.py

 #!/usr/bin/python
 
 """
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
 """
@@ skipping 22 matching lines @@
 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
 if GM_DIRECTORY not in sys.path:
   sys.path.append(GM_DIRECTORY)
 import download_actuals
 import gm_json
 import imagediffdb
 import imagepair
 import imagepairset
-
-# Keys used to link an image to a particular GM test.
-# NOTE: Keep these in sync with static/constants.js
-REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 2
-KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
-KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
-KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
-KEY__EXTRACOLUMN__BUILDER = 'builder'
-KEY__EXTRACOLUMN__CONFIG = 'config'
-KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType'
-KEY__EXTRACOLUMN__TEST = 'test'
-KEY__HEADER = 'header'
-KEY__HEADER__DATAHASH = 'dataHash'
-KEY__HEADER__IS_EDITABLE = 'isEditable'
-KEY__HEADER__IS_EXPORTED = 'isExported'
-KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
-KEY__HEADER__RESULTS_ALL = 'all'
-KEY__HEADER__RESULTS_FAILURES = 'failures'
-KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
-KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
-KEY__HEADER__TIME_UPDATED = 'timeUpdated'
-KEY__HEADER__TYPE = 'type'
-KEY__NEW_IMAGE_URL = 'newImageUrl'
-KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
-KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
-KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
-KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
+import results
 
 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
-    KEY__EXPECTATIONS__BUGS,
-    KEY__EXPECTATIONS__IGNOREFAILURE,
-    KEY__EXPECTATIONS__REVIEWED,
+    results.KEY__EXPECTATIONS__BUGS,
+    results.KEY__EXPECTATIONS__IGNOREFAILURE,
+    results.KEY__EXPECTATIONS__REVIEWED,
 ]
 
-IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
-IMAGE_FILENAME_FORMATTER = '%s_%s.png'  # pass in (testname, config)
-
 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
 
 DEFAULT_ACTUALS_DIR = '.gm-actuals'
 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
     PARENT_DIRECTORY, '.generated-images')
 
 
 class Results(object):
   """ Loads actual and expected GM results into an ImagePairSet.
@@ skipping 44 matching lines @@
     Note that this will NOT update the results stored in self._results[] ;
     in order to see those updates, you must instantiate a new Results object
     based on the (now updated) files on disk.
 
     Args:
       modifications: a list of dictionaries, one for each expectation to update:
 
          [
            {
              imagepair.KEY__EXPECTATIONS_DATA: {
-               KEY__EXPECTATIONS__BUGS: [123, 456],
-               KEY__EXPECTATIONS__IGNOREFAILURE: false,
-               KEY__EXPECTATIONS__REVIEWED: true,
+               results.KEY__EXPECTATIONS__BUGS: [123, 456],
+               results.KEY__EXPECTATIONS__IGNOREFAILURE: false,
+               results.KEY__EXPECTATIONS__REVIEWED: true,
              },
              imagepair.KEY__EXTRA_COLUMN_VALUES: {
-               KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
-               KEY__EXTRACOLUMN__CONFIG: '8888',
-               KEY__EXTRACOLUMN__TEST: 'bigmatrix',
+               results.KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
+               results.KEY__EXTRACOLUMN__CONFIG: '8888',
+               results.KEY__EXTRACOLUMN__TEST: 'bigmatrix',
              },
-             KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png',
+             results.KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png',
            },
            ...
          ]
 
     """
     expected_builder_dicts = Results._read_dicts_from_root(self._expected_root)
     for mod in modifications:
-      image_name = IMAGE_FILENAME_FORMATTER % (
-          mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__TEST],
-          mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__CONFIG])
+      image_name = results.IMAGE_FILENAME_FORMATTER % (
+          mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
+             [results.KEY__EXTRACOLUMN__TEST],
+          mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
+             [results.KEY__EXTRACOLUMN__CONFIG])
       _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl(
-          mod[KEY__NEW_IMAGE_URL])
+          mod[results.KEY__NEW_IMAGE_URL])
       allowed_digests = [[hash_type, int(hash_digest)]]
       new_expectations = {
           gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests,
       }
       for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
         value = mod[imagepair.KEY__EXPECTATIONS_DATA].get(field)
         if value is not None:
           new_expectations[field] = value
       builder_dict = expected_builder_dicts[
-          mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__BUILDER]]
+          mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
+             [results.KEY__EXTRACOLUMN__BUILDER]]
       builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
       if not builder_expectations:
         builder_expectations = {}
         builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations
       builder_expectations[image_name] = new_expectations
     Results._write_dicts_to_root(expected_builder_dicts, self._expected_root)
 
   def get_results_of_type(self, results_type):
     """Return results of some/all tests (depending on 'results_type' parameter).
 
@@ skipping 13 matching lines @@
       results_type: string indicating which set of results to return;
           must be one of the RESULTS_* constants
       reload_seconds: if specified, note that new results may be available once
           these results are reload_seconds old
       is_editable: whether clients are allowed to submit new baselines
       is_exported: whether these results are being made available to other
           network hosts
     """
     response_dict = self._results[results_type]
     time_updated = self.get_timestamp()
-    response_dict[KEY__HEADER] = {
-        KEY__HEADER__SCHEMA_VERSION: REBASELINE_SERVER_SCHEMA_VERSION_NUMBER,
+    response_dict[results.KEY__HEADER] = {
+        results.KEY__HEADER__SCHEMA_VERSION: (
+            results.REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
 
         # Timestamps:
         # 1. when this data was last updated
         # 2. when the caller should check back for new data (if ever)
-        KEY__HEADER__TIME_UPDATED: time_updated,
-        KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
+        results.KEY__HEADER__TIME_UPDATED: time_updated,
+        results.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
             (time_updated+reload_seconds) if reload_seconds else None),
 
         # The type we passed to get_results_of_type()
-        KEY__HEADER__TYPE: results_type,
+        results.KEY__HEADER__TYPE: results_type,
 
         # Hash of dataset, which the client must return with any edits--
         # this ensures that the edits were made to a particular dataset.
-        KEY__HEADER__DATAHASH: str(hash(repr(
+        results.KEY__HEADER__DATAHASH: str(hash(repr(
             response_dict[imagepairset.KEY__IMAGEPAIRS]))),
 
         # Whether the server will accept edits back.
-        KEY__HEADER__IS_EDITABLE: is_editable,
+        results.KEY__HEADER__IS_EDITABLE: is_editable,
 
         # Whether the service is accessible from other hosts.
-        KEY__HEADER__IS_EXPORTED: is_exported,
+        results.KEY__HEADER__IS_EXPORTED: is_exported,
     }
     return response_dict
 
   @staticmethod
   def _ignore_builder(builder):
     """Returns True if we should ignore expectations and actuals for a builder.
 
     This allows us to ignore builders for which we don't maintain expectations
     (trybots, Valgrind, ASAN, TSAN), and avoid problems like
     https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
@@ skipping 114 matching lines @@
     expected_builder_dicts = Results._read_dicts_from_root(self._expected_root)
 
     all_image_pairs = imagepairset.ImagePairSet(
         descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
         diff_base_url=self._diff_base_url)
     failing_image_pairs = imagepairset.ImagePairSet(
         descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
         diff_base_url=self._diff_base_url)
 
     all_image_pairs.ensure_extra_column_values_in_summary(
-        column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[
-            KEY__RESULT_TYPE__FAILED,
-            KEY__RESULT_TYPE__FAILUREIGNORED,
-            KEY__RESULT_TYPE__NOCOMPARISON,
-            KEY__RESULT_TYPE__SUCCEEDED,
+        column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
+            results.KEY__RESULT_TYPE__FAILED,
+            results.KEY__RESULT_TYPE__FAILUREIGNORED,
+            results.KEY__RESULT_TYPE__NOCOMPARISON,
+            results.KEY__RESULT_TYPE__SUCCEEDED,
         ])
     failing_image_pairs.ensure_extra_column_values_in_summary(
-        column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[
-            KEY__RESULT_TYPE__FAILED,
-            KEY__RESULT_TYPE__FAILUREIGNORED,
-            KEY__RESULT_TYPE__NOCOMPARISON,
+        column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
+            results.KEY__RESULT_TYPE__FAILED,
+            results.KEY__RESULT_TYPE__FAILUREIGNORED,
+            results.KEY__RESULT_TYPE__NOCOMPARISON,
         ])
 
     builders = sorted(actual_builder_dicts.keys())
     num_builders = len(builders)
     builder_num = 0
     for builder in builders:
       builder_num += 1
       logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
                    (builder_num, num_builders, builder))
       actual_results_for_this_builder = (
           actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
       for result_type in sorted(actual_results_for_this_builder.keys()):
         results_of_this_type = actual_results_for_this_builder[result_type]
         if not results_of_this_type:
           continue
         for image_name in sorted(results_of_this_type.keys()):
-          (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
+          (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
           actual_image_relative_url = Results._create_relative_url(
               hashtype_and_digest=results_of_this_type[image_name],
               test_name=test)
 
           # Default empty expectations; overwrite these if we find any real ones
           expectations_per_test = None
           expected_image_relative_url = None
           expectations_dict = None
           try:
             expectations_per_test = (
@@ skipping 23 matching lines @@
             #   ('tileimagefilter GM test failing in serialized render mode'),
             #   the gm-actuals will list a failure for the alternate
             #   rendering mode even though we don't have explicit expectations
             #   for the test (the implicit expectation is that it must
             #   render the same in all rendering modes).
             #
             # Don't log type 1, because it is common.
             # Log other types, because they are rare and we should know about
             # them, but don't throw an exception, because we need to keep our
             # tools working in the meanwhile!
-            if result_type != KEY__RESULT_TYPE__NOCOMPARISON:
+            if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON:
               logging.warning('No expectations found for test: %s' % {
-                  KEY__EXTRACOLUMN__BUILDER: builder,
-                  KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
+                  results.KEY__EXTRACOLUMN__BUILDER: builder,
+                  results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
                   'image_name': image_name,
                   })
 
           # If this test was recently rebaselined, it will remain in
           # the 'failed' set of actuals until all the bots have
           # cycled (although the expectations have indeed been set
           # from the most recent actuals).  Treat these as successes
           # instead of failures.
           #
           # TODO(epoger): Do we need to do something similar in
           # other cases, such as when we have recently marked a test
           # as ignoreFailure but it still shows up in the 'failed'
           # category?  Maybe we should not rely on the result_type
           # categories recorded within the gm_actuals AT ALL, and
           # instead evaluate the result_type ourselves based on what
           # we see in expectations vs actual checksum?
           if expected_image_relative_url == actual_image_relative_url:
-            updated_result_type = KEY__RESULT_TYPE__SUCCEEDED
+            updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
           else:
             updated_result_type = result_type
           extra_columns_dict = {
-              KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type,
-              KEY__EXTRACOLUMN__BUILDER: builder,
-              KEY__EXTRACOLUMN__TEST: test,
-              KEY__EXTRACOLUMN__CONFIG: config,
+              results.KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type,
+              results.KEY__EXTRACOLUMN__BUILDER: builder,
+              results.KEY__EXTRACOLUMN__TEST: test,
+              results.KEY__EXTRACOLUMN__CONFIG: config,
           }
           try:
             image_pair = imagepair.ImagePair(
                 image_diff_db=self._image_diff_db,
                 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
                 imageA_relative_url=expected_image_relative_url,
                 imageB_relative_url=actual_image_relative_url,
                 expectations=expectations_dict,
                 extra_columns=extra_columns_dict)
             all_image_pairs.add_image_pair(image_pair)
-            if updated_result_type != KEY__RESULT_TYPE__SUCCEEDED:
+            if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
               failing_image_pairs.add_image_pair(image_pair)
           except Exception:
             logging.exception('got exception while creating new ImagePair')
 
     self._results = {
-      KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
-      KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
+      results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
+      results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
     }
 
 
 def main():
   logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                       datefmt='%m/%d/%Y %H:%M:%S',
                       level=logging.INFO)
   parser = argparse.ArgumentParser()
   parser.add_argument(
       '--actuals', default=DEFAULT_ACTUALS_DIR,
       help='Directory containing all actual-result JSON files')
   parser.add_argument(
       '--expectations', default=DEFAULT_EXPECTATIONS_DIR,
       help='Directory containing all expected-result JSON files; defaults to '
       '\'%(default)s\' .')
   parser.add_argument(
       '--outfile', required=True,
       help='File to write result summary into, in JSON format.')
   parser.add_argument(
-      '--results', default=KEY__HEADER__RESULTS_FAILURES,
+      '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
       help='Which result types to include. Defaults to \'%(default)s\'; '
       'must be one of ' +
-      str([KEY__HEADER__RESULTS_FAILURES, KEY__HEADER__RESULTS_ALL]))
+      str([results.KEY__HEADER__RESULTS_FAILURES,
+           results.KEY__HEADER__RESULTS_ALL]))
   parser.add_argument(
       '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT,
       help='Directory within which to download images and generate diffs; '
       'defaults to \'%(default)s\' .')
   args = parser.parse_args()
   results = Results(actuals_root=args.actuals,
                     expected_root=args.expectations,
                     generated_images_root=args.workdir)
   gm_json.WriteToFile(
       results.get_packaged_results_of_type(results_type=args.results),
       args.outfile)
 
 
 if __name__ == '__main__':
   main()