OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 """ | 3 """ |
4 Copyright 2013 Google Inc. | 4 Copyright 2013 Google Inc. |
5 | 5 |
6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
7 found in the LICENSE file. | 7 found in the LICENSE file. |
8 | 8 |
9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. | 9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. |
10 """ | 10 """ |
(...skipping 22 matching lines...) Expand all Loading... |
33 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) | 33 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) |
34 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) | 34 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) |
35 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) | 35 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) |
36 if GM_DIRECTORY not in sys.path: | 36 if GM_DIRECTORY not in sys.path: |
37 sys.path.append(GM_DIRECTORY) | 37 sys.path.append(GM_DIRECTORY) |
38 import download_actuals | 38 import download_actuals |
39 import gm_json | 39 import gm_json |
40 import imagediffdb | 40 import imagediffdb |
41 import imagepair | 41 import imagepair |
42 import imagepairset | 42 import imagepairset |
43 | 43 import results |
44 # Keys used to link an image to a particular GM test. | |
45 # NOTE: Keep these in sync with static/constants.js | |
46 REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 2 | |
47 KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS | |
48 KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE | |
49 KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED | |
50 KEY__EXTRACOLUMN__BUILDER = 'builder' | |
51 KEY__EXTRACOLUMN__CONFIG = 'config' | |
52 KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType' | |
53 KEY__EXTRACOLUMN__TEST = 'test' | |
54 KEY__HEADER = 'header' | |
55 KEY__HEADER__DATAHASH = 'dataHash' | |
56 KEY__HEADER__IS_EDITABLE = 'isEditable' | |
57 KEY__HEADER__IS_EXPORTED = 'isExported' | |
58 KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' | |
59 KEY__HEADER__RESULTS_ALL = 'all' | |
60 KEY__HEADER__RESULTS_FAILURES = 'failures' | |
61 KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' | |
62 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' | |
63 KEY__HEADER__TIME_UPDATED = 'timeUpdated' | |
64 KEY__HEADER__TYPE = 'type' | |
65 KEY__NEW_IMAGE_URL = 'newImageUrl' | |
66 KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED | |
67 KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED | |
68 KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON | |
69 KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED | |
70 | 44 |
71 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ | 45 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ |
72 KEY__EXPECTATIONS__BUGS, | 46 results.KEY__EXPECTATIONS__BUGS, |
73 KEY__EXPECTATIONS__IGNOREFAILURE, | 47 results.KEY__EXPECTATIONS__IGNOREFAILURE, |
74 KEY__EXPECTATIONS__REVIEWED, | 48 results.KEY__EXPECTATIONS__REVIEWED, |
75 ] | 49 ] |
76 | 50 |
77 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) | |
78 IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) | |
79 | |
80 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') | 51 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') |
81 | 52 |
82 DEFAULT_ACTUALS_DIR = '.gm-actuals' | 53 DEFAULT_ACTUALS_DIR = '.gm-actuals' |
83 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') | 54 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') |
84 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( | 55 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( |
85 PARENT_DIRECTORY, '.generated-images') | 56 PARENT_DIRECTORY, '.generated-images') |
86 | 57 |
87 | 58 |
88 class Results(object): | 59 class Results(object): |
89 """ Loads actual and expected GM results into an ImagePairSet. | 60 """ Loads actual and expected GM results into an ImagePairSet. |
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
134 Note that this will NOT update the results stored in self._results[] ; | 105 Note that this will NOT update the results stored in self._results[] ; |
135 in order to see those updates, you must instantiate a new Results object | 106 in order to see those updates, you must instantiate a new Results object |
136 based on the (now updated) files on disk. | 107 based on the (now updated) files on disk. |
137 | 108 |
138 Args: | 109 Args: |
139 modifications: a list of dictionaries, one for each expectation to update: | 110 modifications: a list of dictionaries, one for each expectation to update: |
140 | 111 |
141 [ | 112 [ |
142 { | 113 { |
143 imagepair.KEY__EXPECTATIONS_DATA: { | 114 imagepair.KEY__EXPECTATIONS_DATA: { |
144 KEY__EXPECTATIONS__BUGS: [123, 456], | 115 results.KEY__EXPECTATIONS__BUGS: [123, 456], |
145 KEY__EXPECTATIONS__IGNOREFAILURE: false, | 116 results.KEY__EXPECTATIONS__IGNOREFAILURE: false, |
146 KEY__EXPECTATIONS__REVIEWED: true, | 117 results.KEY__EXPECTATIONS__REVIEWED: true, |
147 }, | 118 }, |
148 imagepair.KEY__EXTRA_COLUMN_VALUES: { | 119 imagepair.KEY__EXTRA_COLUMN_VALUES: { |
149 KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x
86-Debug', | 120 results.KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeFor
ce320M-x86-Debug', |
150 KEY__EXTRACOLUMN__CONFIG: '8888', | 121 results.KEY__EXTRACOLUMN__CONFIG: '8888', |
151 KEY__EXTRACOLUMN__TEST: 'bigmatrix', | 122 results.KEY__EXTRACOLUMN__TEST: 'bigmatrix', |
152 }, | 123 }, |
153 KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926
.png', | 124 results.KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/108944080240
79689926.png', |
154 }, | 125 }, |
155 ... | 126 ... |
156 ] | 127 ] |
157 | 128 |
158 """ | 129 """ |
159 expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) | 130 expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) |
160 for mod in modifications: | 131 for mod in modifications: |
161 image_name = IMAGE_FILENAME_FORMATTER % ( | 132 image_name = results.IMAGE_FILENAME_FORMATTER % ( |
162 mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__TEST], | 133 mod[imagepair.KEY__EXTRA_COLUMN_VALUES] |
163 mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__CONFIG]) | 134 [results.KEY__EXTRACOLUMN__TEST], |
| 135 mod[imagepair.KEY__EXTRA_COLUMN_VALUES] |
| 136 [results.KEY__EXTRACOLUMN__CONFIG]) |
164 _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( | 137 _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( |
165 mod[KEY__NEW_IMAGE_URL]) | 138 mod[results.KEY__NEW_IMAGE_URL]) |
166 allowed_digests = [[hash_type, int(hash_digest)]] | 139 allowed_digests = [[hash_type, int(hash_digest)]] |
167 new_expectations = { | 140 new_expectations = { |
168 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, | 141 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, |
169 } | 142 } |
170 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: | 143 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: |
171 value = mod[imagepair.KEY__EXPECTATIONS_DATA].get(field) | 144 value = mod[imagepair.KEY__EXPECTATIONS_DATA].get(field) |
172 if value is not None: | 145 if value is not None: |
173 new_expectations[field] = value | 146 new_expectations[field] = value |
174 builder_dict = expected_builder_dicts[ | 147 builder_dict = expected_builder_dicts[ |
175 mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__BUILDER]] | 148 mod[imagepair.KEY__EXTRA_COLUMN_VALUES] |
| 149 [results.KEY__EXTRACOLUMN__BUILDER]] |
176 builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) | 150 builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) |
177 if not builder_expectations: | 151 if not builder_expectations: |
178 builder_expectations = {} | 152 builder_expectations = {} |
179 builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations | 153 builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations |
180 builder_expectations[image_name] = new_expectations | 154 builder_expectations[image_name] = new_expectations |
181 Results._write_dicts_to_root(expected_builder_dicts, self._expected_root) | 155 Results._write_dicts_to_root(expected_builder_dicts, self._expected_root) |
182 | 156 |
183 def get_results_of_type(self, results_type): | 157 def get_results_of_type(self, results_type): |
184 """Return results of some/all tests (depending on 'results_type' parameter). | 158 """Return results of some/all tests (depending on 'results_type' parameter). |
185 | 159 |
(...skipping 13 matching lines...) Expand all Loading... |
199 results_type: string indicating which set of results to return; | 173 results_type: string indicating which set of results to return; |
200 must be one of the RESULTS_* constants | 174 must be one of the RESULTS_* constants |
201 reload_seconds: if specified, note that new results may be available once | 175 reload_seconds: if specified, note that new results may be available once |
202 these results are reload_seconds old | 176 these results are reload_seconds old |
203 is_editable: whether clients are allowed to submit new baselines | 177 is_editable: whether clients are allowed to submit new baselines |
204 is_exported: whether these results are being made available to other | 178 is_exported: whether these results are being made available to other |
205 network hosts | 179 network hosts |
206 """ | 180 """ |
207 response_dict = self._results[results_type] | 181 response_dict = self._results[results_type] |
208 time_updated = self.get_timestamp() | 182 time_updated = self.get_timestamp() |
209 response_dict[KEY__HEADER] = { | 183 response_dict[results.KEY__HEADER] = { |
210 KEY__HEADER__SCHEMA_VERSION: REBASELINE_SERVER_SCHEMA_VERSION_NUMBER, | 184 results.KEY__HEADER__SCHEMA_VERSION: ( |
| 185 results.REBASELINE_SERVER_SCHEMA_VERSION_NUMBER), |
211 | 186 |
212 # Timestamps: | 187 # Timestamps: |
213 # 1. when this data was last updated | 188 # 1. when this data was last updated |
214 # 2. when the caller should check back for new data (if ever) | 189 # 2. when the caller should check back for new data (if ever) |
215 KEY__HEADER__TIME_UPDATED: time_updated, | 190 results.KEY__HEADER__TIME_UPDATED: time_updated, |
216 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( | 191 results.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( |
217 (time_updated+reload_seconds) if reload_seconds else None), | 192 (time_updated+reload_seconds) if reload_seconds else None), |
218 | 193 |
219 # The type we passed to get_results_of_type() | 194 # The type we passed to get_results_of_type() |
220 KEY__HEADER__TYPE: results_type, | 195 results.KEY__HEADER__TYPE: results_type, |
221 | 196 |
222 # Hash of dataset, which the client must return with any edits-- | 197 # Hash of dataset, which the client must return with any edits-- |
223 # this ensures that the edits were made to a particular dataset. | 198 # this ensures that the edits were made to a particular dataset. |
224 KEY__HEADER__DATAHASH: str(hash(repr( | 199 results.KEY__HEADER__DATAHASH: str(hash(repr( |
225 response_dict[imagepairset.KEY__IMAGEPAIRS]))), | 200 response_dict[imagepairset.KEY__IMAGEPAIRS]))), |
226 | 201 |
227 # Whether the server will accept edits back. | 202 # Whether the server will accept edits back. |
228 KEY__HEADER__IS_EDITABLE: is_editable, | 203 results.KEY__HEADER__IS_EDITABLE: is_editable, |
229 | 204 |
230 # Whether the service is accessible from other hosts. | 205 # Whether the service is accessible from other hosts. |
231 KEY__HEADER__IS_EXPORTED: is_exported, | 206 results.KEY__HEADER__IS_EXPORTED: is_exported, |
232 } | 207 } |
233 return response_dict | 208 return response_dict |
234 | 209 |
235 @staticmethod | 210 @staticmethod |
236 def _ignore_builder(builder): | 211 def _ignore_builder(builder): |
237 """Returns True if we should ignore expectations and actuals for a builder. | 212 """Returns True if we should ignore expectations and actuals for a builder. |
238 | 213 |
239 This allows us to ignore builders for which we don't maintain expectations | 214 This allows us to ignore builders for which we don't maintain expectations |
240 (trybots, Valgrind, ASAN, TSAN), and avoid problems like | 215 (trybots, Valgrind, ASAN, TSAN), and avoid problems like |
241 https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server | 216 https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server |
(...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
356 expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) | 331 expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) |
357 | 332 |
358 all_image_pairs = imagepairset.ImagePairSet( | 333 all_image_pairs = imagepairset.ImagePairSet( |
359 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, | 334 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, |
360 diff_base_url=self._diff_base_url) | 335 diff_base_url=self._diff_base_url) |
361 failing_image_pairs = imagepairset.ImagePairSet( | 336 failing_image_pairs = imagepairset.ImagePairSet( |
362 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, | 337 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, |
363 diff_base_url=self._diff_base_url) | 338 diff_base_url=self._diff_base_url) |
364 | 339 |
365 all_image_pairs.ensure_extra_column_values_in_summary( | 340 all_image_pairs.ensure_extra_column_values_in_summary( |
366 column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[ | 341 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ |
367 KEY__RESULT_TYPE__FAILED, | 342 results.KEY__RESULT_TYPE__FAILED, |
368 KEY__RESULT_TYPE__FAILUREIGNORED, | 343 results.KEY__RESULT_TYPE__FAILUREIGNORED, |
369 KEY__RESULT_TYPE__NOCOMPARISON, | 344 results.KEY__RESULT_TYPE__NOCOMPARISON, |
370 KEY__RESULT_TYPE__SUCCEEDED, | 345 results.KEY__RESULT_TYPE__SUCCEEDED, |
371 ]) | 346 ]) |
372 failing_image_pairs.ensure_extra_column_values_in_summary( | 347 failing_image_pairs.ensure_extra_column_values_in_summary( |
373 column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[ | 348 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ |
374 KEY__RESULT_TYPE__FAILED, | 349 results.KEY__RESULT_TYPE__FAILED, |
375 KEY__RESULT_TYPE__FAILUREIGNORED, | 350 results.KEY__RESULT_TYPE__FAILUREIGNORED, |
376 KEY__RESULT_TYPE__NOCOMPARISON, | 351 results.KEY__RESULT_TYPE__NOCOMPARISON, |
377 ]) | 352 ]) |
378 | 353 |
379 builders = sorted(actual_builder_dicts.keys()) | 354 builders = sorted(actual_builder_dicts.keys()) |
380 num_builders = len(builders) | 355 num_builders = len(builders) |
381 builder_num = 0 | 356 builder_num = 0 |
382 for builder in builders: | 357 for builder in builders: |
383 builder_num += 1 | 358 builder_num += 1 |
384 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % | 359 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % |
385 (builder_num, num_builders, builder)) | 360 (builder_num, num_builders, builder)) |
386 actual_results_for_this_builder = ( | 361 actual_results_for_this_builder = ( |
387 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) | 362 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) |
388 for result_type in sorted(actual_results_for_this_builder.keys()): | 363 for result_type in sorted(actual_results_for_this_builder.keys()): |
389 results_of_this_type = actual_results_for_this_builder[result_type] | 364 results_of_this_type = actual_results_for_this_builder[result_type] |
390 if not results_of_this_type: | 365 if not results_of_this_type: |
391 continue | 366 continue |
392 for image_name in sorted(results_of_this_type.keys()): | 367 for image_name in sorted(results_of_this_type.keys()): |
393 (test, config) = IMAGE_FILENAME_RE.match(image_name).groups() | 368 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() |
394 actual_image_relative_url = Results._create_relative_url( | 369 actual_image_relative_url = Results._create_relative_url( |
395 hashtype_and_digest=results_of_this_type[image_name], | 370 hashtype_and_digest=results_of_this_type[image_name], |
396 test_name=test) | 371 test_name=test) |
397 | 372 |
398 # Default empty expectations; overwrite these if we find any real ones | 373 # Default empty expectations; overwrite these if we find any real ones |
399 expectations_per_test = None | 374 expectations_per_test = None |
400 expected_image_relative_url = None | 375 expected_image_relative_url = None |
401 expectations_dict = None | 376 expectations_dict = None |
402 try: | 377 try: |
403 expectations_per_test = ( | 378 expectations_per_test = ( |
(...skipping 23 matching lines...) Expand all Loading... |
427 # ('tileimagefilter GM test failing in serialized render mode'), | 402 # ('tileimagefilter GM test failing in serialized render mode'), |
428 # the gm-actuals will list a failure for the alternate | 403 # the gm-actuals will list a failure for the alternate |
429 # rendering mode even though we don't have explicit expectations | 404 # rendering mode even though we don't have explicit expectations |
430 # for the test (the implicit expectation is that it must | 405 # for the test (the implicit expectation is that it must |
431 # render the same in all rendering modes). | 406 # render the same in all rendering modes). |
432 # | 407 # |
433 # Don't log type 1, because it is common. | 408 # Don't log type 1, because it is common. |
434 # Log other types, because they are rare and we should know about | 409 # Log other types, because they are rare and we should know about |
435 # them, but don't throw an exception, because we need to keep our | 410 # them, but don't throw an exception, because we need to keep our |
436 # tools working in the meanwhile! | 411 # tools working in the meanwhile! |
437 if result_type != KEY__RESULT_TYPE__NOCOMPARISON: | 412 if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON: |
438 logging.warning('No expectations found for test: %s' % { | 413 logging.warning('No expectations found for test: %s' % { |
439 KEY__EXTRACOLUMN__BUILDER: builder, | 414 results.KEY__EXTRACOLUMN__BUILDER: builder, |
440 KEY__EXTRACOLUMN__RESULT_TYPE: result_type, | 415 results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type, |
441 'image_name': image_name, | 416 'image_name': image_name, |
442 }) | 417 }) |
443 | 418 |
444 # If this test was recently rebaselined, it will remain in | 419 # If this test was recently rebaselined, it will remain in |
445 # the 'failed' set of actuals until all the bots have | 420 # the 'failed' set of actuals until all the bots have |
446 # cycled (although the expectations have indeed been set | 421 # cycled (although the expectations have indeed been set |
447 # from the most recent actuals). Treat these as successes | 422 # from the most recent actuals). Treat these as successes |
448 # instead of failures. | 423 # instead of failures. |
449 # | 424 # |
450 # TODO(epoger): Do we need to do something similar in | 425 # TODO(epoger): Do we need to do something similar in |
451 # other cases, such as when we have recently marked a test | 426 # other cases, such as when we have recently marked a test |
452 # as ignoreFailure but it still shows up in the 'failed' | 427 # as ignoreFailure but it still shows up in the 'failed' |
453 # category? Maybe we should not rely on the result_type | 428 # category? Maybe we should not rely on the result_type |
454 # categories recorded within the gm_actuals AT ALL, and | 429 # categories recorded within the gm_actuals AT ALL, and |
455 # instead evaluate the result_type ourselves based on what | 430 # instead evaluate the result_type ourselves based on what |
456 # we see in expectations vs actual checksum? | 431 # we see in expectations vs actual checksum? |
457 if expected_image_relative_url == actual_image_relative_url: | 432 if expected_image_relative_url == actual_image_relative_url: |
458 updated_result_type = KEY__RESULT_TYPE__SUCCEEDED | 433 updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED |
459 else: | 434 else: |
460 updated_result_type = result_type | 435 updated_result_type = result_type |
461 extra_columns_dict = { | 436 extra_columns_dict = { |
462 KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type, | 437 results.KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type, |
463 KEY__EXTRACOLUMN__BUILDER: builder, | 438 results.KEY__EXTRACOLUMN__BUILDER: builder, |
464 KEY__EXTRACOLUMN__TEST: test, | 439 results.KEY__EXTRACOLUMN__TEST: test, |
465 KEY__EXTRACOLUMN__CONFIG: config, | 440 results.KEY__EXTRACOLUMN__CONFIG: config, |
466 } | 441 } |
467 try: | 442 try: |
468 image_pair = imagepair.ImagePair( | 443 image_pair = imagepair.ImagePair( |
469 image_diff_db=self._image_diff_db, | 444 image_diff_db=self._image_diff_db, |
470 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, | 445 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, |
471 imageA_relative_url=expected_image_relative_url, | 446 imageA_relative_url=expected_image_relative_url, |
472 imageB_relative_url=actual_image_relative_url, | 447 imageB_relative_url=actual_image_relative_url, |
473 expectations=expectations_dict, | 448 expectations=expectations_dict, |
474 extra_columns=extra_columns_dict) | 449 extra_columns=extra_columns_dict) |
475 all_image_pairs.add_image_pair(image_pair) | 450 all_image_pairs.add_image_pair(image_pair) |
476 if updated_result_type != KEY__RESULT_TYPE__SUCCEEDED: | 451 if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED: |
477 failing_image_pairs.add_image_pair(image_pair) | 452 failing_image_pairs.add_image_pair(image_pair) |
478 except Exception: | 453 except Exception: |
479 logging.exception('got exception while creating new ImagePair') | 454 logging.exception('got exception while creating new ImagePair') |
480 | 455 |
481 self._results = { | 456 self._results = { |
482 KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), | 457 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), |
483 KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), | 458 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), |
484 } | 459 } |
485 | 460 |
486 | 461 |
487 def main(): | 462 def main(): |
488 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', | 463 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', |
489 datefmt='%m/%d/%Y %H:%M:%S', | 464 datefmt='%m/%d/%Y %H:%M:%S', |
490 level=logging.INFO) | 465 level=logging.INFO) |
491 parser = argparse.ArgumentParser() | 466 parser = argparse.ArgumentParser() |
492 parser.add_argument( | 467 parser.add_argument( |
493 '--actuals', default=DEFAULT_ACTUALS_DIR, | 468 '--actuals', default=DEFAULT_ACTUALS_DIR, |
494 help='Directory containing all actual-result JSON files') | 469 help='Directory containing all actual-result JSON files') |
495 parser.add_argument( | 470 parser.add_argument( |
496 '--expectations', default=DEFAULT_EXPECTATIONS_DIR, | 471 '--expectations', default=DEFAULT_EXPECTATIONS_DIR, |
497 help='Directory containing all expected-result JSON files; defaults to ' | 472 help='Directory containing all expected-result JSON files; defaults to ' |
498 '\'%(default)s\' .') | 473 '\'%(default)s\' .') |
499 parser.add_argument( | 474 parser.add_argument( |
500 '--outfile', required=True, | 475 '--outfile', required=True, |
501 help='File to write result summary into, in JSON format.') | 476 help='File to write result summary into, in JSON format.') |
502 parser.add_argument( | 477 parser.add_argument( |
503 '--results', default=KEY__HEADER__RESULTS_FAILURES, | 478 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, |
504 help='Which result types to include. Defaults to \'%(default)s\'; ' | 479 help='Which result types to include. Defaults to \'%(default)s\'; ' |
505 'must be one of ' + | 480 'must be one of ' + |
506 str([KEY__HEADER__RESULTS_FAILURES, KEY__HEADER__RESULTS_ALL])) | 481 str([results.KEY__HEADER__RESULTS_FAILURES, |
| 482 results.KEY__HEADER__RESULTS_ALL])) |
507 parser.add_argument( | 483 parser.add_argument( |
508 '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT, | 484 '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT, |
509 help='Directory within which to download images and generate diffs; ' | 485 help='Directory within which to download images and generate diffs; ' |
510 'defaults to \'%(default)s\' .') | 486 'defaults to \'%(default)s\' .') |
511 args = parser.parse_args() | 487 args = parser.parse_args() |
512 results = Results(actuals_root=args.actuals, | 488 results = Results(actuals_root=args.actuals, |
513 expected_root=args.expectations, | 489 expected_root=args.expectations, |
514 generated_images_root=args.workdir) | 490 generated_images_root=args.workdir) |
515 gm_json.WriteToFile( | 491 gm_json.WriteToFile( |
516 results.get_packaged_results_of_type(results_type=args.results), | 492 results.get_packaged_results_of_type(results_type=args.results), |
517 args.outfile) | 493 args.outfile) |
518 | 494 |
519 | 495 |
520 if __name__ == '__main__': | 496 if __name__ == '__main__': |
521 main() | 497 main() |
OLD | NEW |