Index: gm/rebaseline_server/compare_to_expectations.py |
diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/compare_to_expectations.py |
similarity index 82% |
copy from gm/rebaseline_server/results.py |
copy to gm/rebaseline_server/compare_to_expectations.py |
index 66105a49213bc182ac554df882e8ced12404efbd..f047b7b2911e7afbf197e128da9abcda62949c2c 100755 |
--- a/gm/rebaseline_server/results.py |
+++ b/gm/rebaseline_server/compare_to_expectations.py |
@@ -40,43 +40,14 @@ import gm_json |
import imagediffdb |
import imagepair |
import imagepairset |
- |
-# Keys used to link an image to a particular GM test. |
-# NOTE: Keep these in sync with static/constants.js |
-REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 2 |
-KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS |
-KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE |
-KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED |
-KEY__EXTRACOLUMN__BUILDER = 'builder' |
-KEY__EXTRACOLUMN__CONFIG = 'config' |
-KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType' |
-KEY__EXTRACOLUMN__TEST = 'test' |
-KEY__HEADER = 'header' |
-KEY__HEADER__DATAHASH = 'dataHash' |
-KEY__HEADER__IS_EDITABLE = 'isEditable' |
-KEY__HEADER__IS_EXPORTED = 'isExported' |
-KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' |
-KEY__HEADER__RESULTS_ALL = 'all' |
-KEY__HEADER__RESULTS_FAILURES = 'failures' |
-KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' |
-KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' |
-KEY__HEADER__TIME_UPDATED = 'timeUpdated' |
-KEY__HEADER__TYPE = 'type' |
-KEY__NEW_IMAGE_URL = 'newImageUrl' |
-KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED |
-KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED |
-KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON |
-KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED |
+import results |
EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ |
- KEY__EXPECTATIONS__BUGS, |
- KEY__EXPECTATIONS__IGNOREFAILURE, |
- KEY__EXPECTATIONS__REVIEWED, |
+ results.KEY__EXPECTATIONS__BUGS, |
+ results.KEY__EXPECTATIONS__IGNOREFAILURE, |
+ results.KEY__EXPECTATIONS__REVIEWED, |
] |
-IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
-IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) |
- |
IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') |
DEFAULT_ACTUALS_DIR = '.gm-actuals' |
@@ -141,16 +112,16 @@ class Results(object): |
[ |
{ |
imagepair.KEY__EXPECTATIONS_DATA: { |
- KEY__EXPECTATIONS__BUGS: [123, 456], |
- KEY__EXPECTATIONS__IGNOREFAILURE: false, |
- KEY__EXPECTATIONS__REVIEWED: true, |
+ results.KEY__EXPECTATIONS__BUGS: [123, 456], |
+ results.KEY__EXPECTATIONS__IGNOREFAILURE: false, |
+ results.KEY__EXPECTATIONS__REVIEWED: true, |
}, |
imagepair.KEY__EXTRA_COLUMN_VALUES: { |
- KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', |
- KEY__EXTRACOLUMN__CONFIG: '8888', |
- KEY__EXTRACOLUMN__TEST: 'bigmatrix', |
+ results.KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', |
+ results.KEY__EXTRACOLUMN__CONFIG: '8888', |
+ results.KEY__EXTRACOLUMN__TEST: 'bigmatrix', |
}, |
- KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png', |
+ results.KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png', |
}, |
... |
] |
@@ -158,11 +129,13 @@ class Results(object): |
""" |
expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) |
for mod in modifications: |
- image_name = IMAGE_FILENAME_FORMATTER % ( |
- mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__TEST], |
- mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__CONFIG]) |
+ image_name = results.IMAGE_FILENAME_FORMATTER % ( |
+ mod[imagepair.KEY__EXTRA_COLUMN_VALUES] |
+ [results.KEY__EXTRACOLUMN__TEST], |
+ mod[imagepair.KEY__EXTRA_COLUMN_VALUES] |
+ [results.KEY__EXTRACOLUMN__CONFIG]) |
_, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( |
- mod[KEY__NEW_IMAGE_URL]) |
+ mod[results.KEY__NEW_IMAGE_URL]) |
allowed_digests = [[hash_type, int(hash_digest)]] |
new_expectations = { |
gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, |
@@ -172,7 +145,8 @@ class Results(object): |
if value is not None: |
new_expectations[field] = value |
builder_dict = expected_builder_dicts[ |
- mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__BUILDER]] |
+ mod[imagepair.KEY__EXTRA_COLUMN_VALUES] |
+ [results.KEY__EXTRACOLUMN__BUILDER]] |
builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) |
if not builder_expectations: |
builder_expectations = {} |
@@ -206,29 +180,30 @@ class Results(object): |
""" |
response_dict = self._results[results_type] |
time_updated = self.get_timestamp() |
- response_dict[KEY__HEADER] = { |
- KEY__HEADER__SCHEMA_VERSION: REBASELINE_SERVER_SCHEMA_VERSION_NUMBER, |
+ response_dict[results.KEY__HEADER] = { |
+ results.KEY__HEADER__SCHEMA_VERSION: ( |
+ results.REBASELINE_SERVER_SCHEMA_VERSION_NUMBER), |
# Timestamps: |
# 1. when this data was last updated |
# 2. when the caller should check back for new data (if ever) |
- KEY__HEADER__TIME_UPDATED: time_updated, |
- KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( |
+ results.KEY__HEADER__TIME_UPDATED: time_updated, |
+ results.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( |
(time_updated+reload_seconds) if reload_seconds else None), |
# The type we passed to get_results_of_type() |
- KEY__HEADER__TYPE: results_type, |
+ results.KEY__HEADER__TYPE: results_type, |
# Hash of dataset, which the client must return with any edits-- |
# this ensures that the edits were made to a particular dataset. |
- KEY__HEADER__DATAHASH: str(hash(repr( |
+ results.KEY__HEADER__DATAHASH: str(hash(repr( |
response_dict[imagepairset.KEY__IMAGEPAIRS]))), |
# Whether the server will accept edits back. |
- KEY__HEADER__IS_EDITABLE: is_editable, |
+ results.KEY__HEADER__IS_EDITABLE: is_editable, |
# Whether the service is accessible from other hosts. |
- KEY__HEADER__IS_EXPORTED: is_exported, |
+ results.KEY__HEADER__IS_EXPORTED: is_exported, |
} |
return response_dict |
@@ -363,17 +338,17 @@ class Results(object): |
diff_base_url=self._diff_base_url) |
all_image_pairs.ensure_extra_column_values_in_summary( |
- column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[ |
- KEY__RESULT_TYPE__FAILED, |
- KEY__RESULT_TYPE__FAILUREIGNORED, |
- KEY__RESULT_TYPE__NOCOMPARISON, |
- KEY__RESULT_TYPE__SUCCEEDED, |
+ column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ |
+ results.KEY__RESULT_TYPE__FAILED, |
+ results.KEY__RESULT_TYPE__FAILUREIGNORED, |
+ results.KEY__RESULT_TYPE__NOCOMPARISON, |
+ results.KEY__RESULT_TYPE__SUCCEEDED, |
]) |
failing_image_pairs.ensure_extra_column_values_in_summary( |
- column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[ |
- KEY__RESULT_TYPE__FAILED, |
- KEY__RESULT_TYPE__FAILUREIGNORED, |
- KEY__RESULT_TYPE__NOCOMPARISON, |
+ column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ |
+ results.KEY__RESULT_TYPE__FAILED, |
+ results.KEY__RESULT_TYPE__FAILUREIGNORED, |
+ results.KEY__RESULT_TYPE__NOCOMPARISON, |
]) |
builders = sorted(actual_builder_dicts.keys()) |
@@ -390,7 +365,7 @@ class Results(object): |
if not results_of_this_type: |
continue |
for image_name in sorted(results_of_this_type.keys()): |
- (test, config) = IMAGE_FILENAME_RE.match(image_name).groups() |
+ (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() |
actual_image_relative_url = Results._create_relative_url( |
hashtype_and_digest=results_of_this_type[image_name], |
test_name=test) |
@@ -434,10 +409,10 @@ class Results(object): |
# Log other types, because they are rare and we should know about |
# them, but don't throw an exception, because we need to keep our |
# tools working in the meanwhile! |
- if result_type != KEY__RESULT_TYPE__NOCOMPARISON: |
+ if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON: |
logging.warning('No expectations found for test: %s' % { |
- KEY__EXTRACOLUMN__BUILDER: builder, |
- KEY__EXTRACOLUMN__RESULT_TYPE: result_type, |
+ results.KEY__EXTRACOLUMN__BUILDER: builder, |
+ results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type, |
'image_name': image_name, |
}) |
@@ -455,14 +430,14 @@ class Results(object): |
# instead evaluate the result_type ourselves based on what |
# we see in expectations vs actual checksum? |
if expected_image_relative_url == actual_image_relative_url: |
- updated_result_type = KEY__RESULT_TYPE__SUCCEEDED |
+ updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED |
else: |
updated_result_type = result_type |
extra_columns_dict = { |
- KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type, |
- KEY__EXTRACOLUMN__BUILDER: builder, |
- KEY__EXTRACOLUMN__TEST: test, |
- KEY__EXTRACOLUMN__CONFIG: config, |
+ results.KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type, |
+ results.KEY__EXTRACOLUMN__BUILDER: builder, |
+ results.KEY__EXTRACOLUMN__TEST: test, |
+ results.KEY__EXTRACOLUMN__CONFIG: config, |
} |
try: |
image_pair = imagepair.ImagePair( |
@@ -473,14 +448,14 @@ class Results(object): |
expectations=expectations_dict, |
extra_columns=extra_columns_dict) |
all_image_pairs.add_image_pair(image_pair) |
- if updated_result_type != KEY__RESULT_TYPE__SUCCEEDED: |
+ if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED: |
failing_image_pairs.add_image_pair(image_pair) |
except Exception: |
logging.exception('got exception while creating new ImagePair') |
self._results = { |
- KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), |
- KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), |
+ results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), |
+ results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), |
} |
@@ -500,10 +475,11 @@ def main(): |
'--outfile', required=True, |
help='File to write result summary into, in JSON format.') |
parser.add_argument( |
- '--results', default=KEY__HEADER__RESULTS_FAILURES, |
+ '--results', default=results.KEY__HEADER__RESULTS_FAILURES, |
help='Which result types to include. Defaults to \'%(default)s\'; ' |
'must be one of ' + |
- str([KEY__HEADER__RESULTS_FAILURES, KEY__HEADER__RESULTS_ALL])) |
+ str([results.KEY__HEADER__RESULTS_FAILURES, |
+ results.KEY__HEADER__RESULTS_ALL])) |
parser.add_argument( |
'--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT, |
help='Directory within which to download images and generate diffs; ' |