| Index: gm/rebaseline_server/compare_to_expectations.py
|
| diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/compare_to_expectations.py
|
| similarity index 82%
|
| copy from gm/rebaseline_server/results.py
|
| copy to gm/rebaseline_server/compare_to_expectations.py
|
| index 66105a49213bc182ac554df882e8ced12404efbd..f047b7b2911e7afbf197e128da9abcda62949c2c 100755
|
| --- a/gm/rebaseline_server/results.py
|
| +++ b/gm/rebaseline_server/compare_to_expectations.py
|
| @@ -40,43 +40,14 @@ import gm_json
|
| import imagediffdb
|
| import imagepair
|
| import imagepairset
|
| -
|
| -# Keys used to link an image to a particular GM test.
|
| -# NOTE: Keep these in sync with static/constants.js
|
| -REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 2
|
| -KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
|
| -KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
|
| -KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
|
| -KEY__EXTRACOLUMN__BUILDER = 'builder'
|
| -KEY__EXTRACOLUMN__CONFIG = 'config'
|
| -KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType'
|
| -KEY__EXTRACOLUMN__TEST = 'test'
|
| -KEY__HEADER = 'header'
|
| -KEY__HEADER__DATAHASH = 'dataHash'
|
| -KEY__HEADER__IS_EDITABLE = 'isEditable'
|
| -KEY__HEADER__IS_EXPORTED = 'isExported'
|
| -KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
|
| -KEY__HEADER__RESULTS_ALL = 'all'
|
| -KEY__HEADER__RESULTS_FAILURES = 'failures'
|
| -KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
|
| -KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
|
| -KEY__HEADER__TIME_UPDATED = 'timeUpdated'
|
| -KEY__HEADER__TYPE = 'type'
|
| -KEY__NEW_IMAGE_URL = 'newImageUrl'
|
| -KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
|
| -KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
|
| -KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
|
| -KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
|
| +import results
|
|
|
| EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
|
| - KEY__EXPECTATIONS__BUGS,
|
| - KEY__EXPECTATIONS__IGNOREFAILURE,
|
| - KEY__EXPECTATIONS__REVIEWED,
|
| + results.KEY__EXPECTATIONS__BUGS,
|
| + results.KEY__EXPECTATIONS__IGNOREFAILURE,
|
| + results.KEY__EXPECTATIONS__REVIEWED,
|
| ]
|
|
|
| -IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
|
| -IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config)
|
| -
|
| IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
|
|
|
| DEFAULT_ACTUALS_DIR = '.gm-actuals'
|
| @@ -141,16 +112,16 @@ class Results(object):
|
| [
|
| {
|
| imagepair.KEY__EXPECTATIONS_DATA: {
|
| - KEY__EXPECTATIONS__BUGS: [123, 456],
|
| - KEY__EXPECTATIONS__IGNOREFAILURE: false,
|
| - KEY__EXPECTATIONS__REVIEWED: true,
|
| + results.KEY__EXPECTATIONS__BUGS: [123, 456],
|
| + results.KEY__EXPECTATIONS__IGNOREFAILURE: false,
|
| + results.KEY__EXPECTATIONS__REVIEWED: true,
|
| },
|
| imagepair.KEY__EXTRA_COLUMN_VALUES: {
|
| - KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
|
| - KEY__EXTRACOLUMN__CONFIG: '8888',
|
| - KEY__EXTRACOLUMN__TEST: 'bigmatrix',
|
| + results.KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
|
| + results.KEY__EXTRACOLUMN__CONFIG: '8888',
|
| + results.KEY__EXTRACOLUMN__TEST: 'bigmatrix',
|
| },
|
| - KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png',
|
| + results.KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png',
|
| },
|
| ...
|
| ]
|
| @@ -158,11 +129,13 @@ class Results(object):
|
| """
|
| expected_builder_dicts = Results._read_dicts_from_root(self._expected_root)
|
| for mod in modifications:
|
| - image_name = IMAGE_FILENAME_FORMATTER % (
|
| - mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__TEST],
|
| - mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__CONFIG])
|
| + image_name = results.IMAGE_FILENAME_FORMATTER % (
|
| + mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
|
| + [results.KEY__EXTRACOLUMN__TEST],
|
| + mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
|
| + [results.KEY__EXTRACOLUMN__CONFIG])
|
| _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl(
|
| - mod[KEY__NEW_IMAGE_URL])
|
| + mod[results.KEY__NEW_IMAGE_URL])
|
| allowed_digests = [[hash_type, int(hash_digest)]]
|
| new_expectations = {
|
| gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests,
|
| @@ -172,7 +145,8 @@ class Results(object):
|
| if value is not None:
|
| new_expectations[field] = value
|
| builder_dict = expected_builder_dicts[
|
| - mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__BUILDER]]
|
| + mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
|
| + [results.KEY__EXTRACOLUMN__BUILDER]]
|
| builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
|
| if not builder_expectations:
|
| builder_expectations = {}
|
| @@ -206,29 +180,30 @@ class Results(object):
|
| """
|
| response_dict = self._results[results_type]
|
| time_updated = self.get_timestamp()
|
| - response_dict[KEY__HEADER] = {
|
| - KEY__HEADER__SCHEMA_VERSION: REBASELINE_SERVER_SCHEMA_VERSION_NUMBER,
|
| + response_dict[results.KEY__HEADER] = {
|
| + results.KEY__HEADER__SCHEMA_VERSION: (
|
| + results.REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
|
|
|
| # Timestamps:
|
| # 1. when this data was last updated
|
| # 2. when the caller should check back for new data (if ever)
|
| - KEY__HEADER__TIME_UPDATED: time_updated,
|
| - KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
|
| + results.KEY__HEADER__TIME_UPDATED: time_updated,
|
| + results.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
|
| (time_updated+reload_seconds) if reload_seconds else None),
|
|
|
| # The type we passed to get_results_of_type()
|
| - KEY__HEADER__TYPE: results_type,
|
| + results.KEY__HEADER__TYPE: results_type,
|
|
|
| # Hash of dataset, which the client must return with any edits--
|
| # this ensures that the edits were made to a particular dataset.
|
| - KEY__HEADER__DATAHASH: str(hash(repr(
|
| + results.KEY__HEADER__DATAHASH: str(hash(repr(
|
| response_dict[imagepairset.KEY__IMAGEPAIRS]))),
|
|
|
| # Whether the server will accept edits back.
|
| - KEY__HEADER__IS_EDITABLE: is_editable,
|
| + results.KEY__HEADER__IS_EDITABLE: is_editable,
|
|
|
| # Whether the service is accessible from other hosts.
|
| - KEY__HEADER__IS_EXPORTED: is_exported,
|
| + results.KEY__HEADER__IS_EXPORTED: is_exported,
|
| }
|
| return response_dict
|
|
|
| @@ -363,17 +338,17 @@ class Results(object):
|
| diff_base_url=self._diff_base_url)
|
|
|
| all_image_pairs.ensure_extra_column_values_in_summary(
|
| - column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[
|
| - KEY__RESULT_TYPE__FAILED,
|
| - KEY__RESULT_TYPE__FAILUREIGNORED,
|
| - KEY__RESULT_TYPE__NOCOMPARISON,
|
| - KEY__RESULT_TYPE__SUCCEEDED,
|
| + column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
|
| + results.KEY__RESULT_TYPE__FAILED,
|
| + results.KEY__RESULT_TYPE__FAILUREIGNORED,
|
| + results.KEY__RESULT_TYPE__NOCOMPARISON,
|
| + results.KEY__RESULT_TYPE__SUCCEEDED,
|
| ])
|
| failing_image_pairs.ensure_extra_column_values_in_summary(
|
| - column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[
|
| - KEY__RESULT_TYPE__FAILED,
|
| - KEY__RESULT_TYPE__FAILUREIGNORED,
|
| - KEY__RESULT_TYPE__NOCOMPARISON,
|
| + column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
|
| + results.KEY__RESULT_TYPE__FAILED,
|
| + results.KEY__RESULT_TYPE__FAILUREIGNORED,
|
| + results.KEY__RESULT_TYPE__NOCOMPARISON,
|
| ])
|
|
|
| builders = sorted(actual_builder_dicts.keys())
|
| @@ -390,7 +365,7 @@ class Results(object):
|
| if not results_of_this_type:
|
| continue
|
| for image_name in sorted(results_of_this_type.keys()):
|
| - (test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
|
| + (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
|
| actual_image_relative_url = Results._create_relative_url(
|
| hashtype_and_digest=results_of_this_type[image_name],
|
| test_name=test)
|
| @@ -434,10 +409,10 @@ class Results(object):
|
| # Log other types, because they are rare and we should know about
|
| # them, but don't throw an exception, because we need to keep our
|
| # tools working in the meanwhile!
|
| - if result_type != KEY__RESULT_TYPE__NOCOMPARISON:
|
| + if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON:
|
| logging.warning('No expectations found for test: %s' % {
|
| - KEY__EXTRACOLUMN__BUILDER: builder,
|
| - KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
|
| + results.KEY__EXTRACOLUMN__BUILDER: builder,
|
| + results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
|
| 'image_name': image_name,
|
| })
|
|
|
| @@ -455,14 +430,14 @@ class Results(object):
|
| # instead evaluate the result_type ourselves based on what
|
| # we see in expectations vs actual checksum?
|
| if expected_image_relative_url == actual_image_relative_url:
|
| - updated_result_type = KEY__RESULT_TYPE__SUCCEEDED
|
| + updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
|
| else:
|
| updated_result_type = result_type
|
| extra_columns_dict = {
|
| - KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type,
|
| - KEY__EXTRACOLUMN__BUILDER: builder,
|
| - KEY__EXTRACOLUMN__TEST: test,
|
| - KEY__EXTRACOLUMN__CONFIG: config,
|
| + results.KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type,
|
| + results.KEY__EXTRACOLUMN__BUILDER: builder,
|
| + results.KEY__EXTRACOLUMN__TEST: test,
|
| + results.KEY__EXTRACOLUMN__CONFIG: config,
|
| }
|
| try:
|
| image_pair = imagepair.ImagePair(
|
| @@ -473,14 +448,14 @@ class Results(object):
|
| expectations=expectations_dict,
|
| extra_columns=extra_columns_dict)
|
| all_image_pairs.add_image_pair(image_pair)
|
| - if updated_result_type != KEY__RESULT_TYPE__SUCCEEDED:
|
| + if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
|
| failing_image_pairs.add_image_pair(image_pair)
|
| except Exception:
|
| logging.exception('got exception while creating new ImagePair')
|
|
|
| self._results = {
|
| - KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
|
| - KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
|
| + results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
|
| + results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
|
| }
|
|
|
|
|
| @@ -500,10 +475,11 @@ def main():
|
| '--outfile', required=True,
|
| help='File to write result summary into, in JSON format.')
|
| parser.add_argument(
|
| - '--results', default=KEY__HEADER__RESULTS_FAILURES,
|
| + '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
|
| help='Which result types to include. Defaults to \'%(default)s\'; '
|
| 'must be one of ' +
|
| - str([KEY__HEADER__RESULTS_FAILURES, KEY__HEADER__RESULTS_ALL]))
|
| + str([results.KEY__HEADER__RESULTS_FAILURES,
|
| + results.KEY__HEADER__RESULTS_ALL]))
|
| parser.add_argument(
|
| '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT,
|
| help='Directory within which to download images and generate diffs; '
|
|
|