OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 """ | 3 """ |
4 Copyright 2013 Google Inc. | 4 Copyright 2013 Google Inc. |
5 | 5 |
6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
7 found in the LICENSE file. | 7 found in the LICENSE file. |
8 | 8 |
9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. | 9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. |
10 """ | 10 """ |
11 | 11 |
12 # System-level imports | 12 # System-level imports |
13 import argparse | |
14 import fnmatch | |
15 import json | |
16 import logging | |
17 import os | 13 import os |
18 import re | 14 import re |
19 import sys | 15 import sys |
20 import time | |
21 | 16 |
22 # Imports from within Skia | 17 # Imports from within Skia |
23 # | 18 # |
24 # TODO(epoger): Once we move the create_filepath_url() function out of | |
25 # download_actuals into a shared utility module, we won't need to import | |
26 # download_actuals anymore. | |
27 # | |
28 # We need to add the 'gm' directory, so that we can import gm_json.py within | 19 # We need to add the 'gm' directory, so that we can import gm_json.py within |
29 # that directory. That script allows us to parse the actual-results.json file | 20 # that directory. That script allows us to parse the actual-results.json file |
30 # written out by the GM tool. | 21 # written out by the GM tool. |
31 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* | 22 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* |
32 # so any dirs that are already in the PYTHONPATH will be preferred. | 23 # so any dirs that are already in the PYTHONPATH will be preferred. |
33 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) | 24 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) |
34 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) | 25 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) |
35 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) | |
36 if GM_DIRECTORY not in sys.path: | 26 if GM_DIRECTORY not in sys.path: |
37 sys.path.append(GM_DIRECTORY) | 27 sys.path.append(GM_DIRECTORY) |
38 import download_actuals | |
39 import gm_json | 28 import gm_json |
40 import imagediffdb | |
41 import imagepair | |
42 import imagepairset | |
43 | 29 |
44 # Keys used to link an image to a particular GM test. | 30 # Keys used to link an image to a particular GM test. |
45 # NOTE: Keep these in sync with static/constants.js | 31 # NOTE: Keep these in sync with static/constants.js |
46 REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 2 | 32 REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 2 |
47 KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS | 33 KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS |
48 KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE | 34 KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE |
49 KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED | 35 KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED |
50 KEY__EXTRACOLUMN__BUILDER = 'builder' | 36 KEY__EXTRACOLUMN__BUILDER = 'builder' |
51 KEY__EXTRACOLUMN__CONFIG = 'config' | 37 KEY__EXTRACOLUMN__CONFIG = 'config' |
52 KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType' | 38 KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType' |
53 KEY__EXTRACOLUMN__TEST = 'test' | 39 KEY__EXTRACOLUMN__TEST = 'test' |
54 KEY__HEADER = 'header' | 40 KEY__HEADER = 'header' |
55 KEY__HEADER__DATAHASH = 'dataHash' | 41 KEY__HEADER__DATAHASH = 'dataHash' |
56 KEY__HEADER__IS_EDITABLE = 'isEditable' | 42 KEY__HEADER__IS_EDITABLE = 'isEditable' |
57 KEY__HEADER__IS_EXPORTED = 'isExported' | 43 KEY__HEADER__IS_EXPORTED = 'isExported' |
58 KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' | 44 KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' |
59 KEY__HEADER__RESULTS_ALL = 'all' | 45 KEY__HEADER__RESULTS_ALL = 'all' |
60 KEY__HEADER__RESULTS_FAILURES = 'failures' | 46 KEY__HEADER__RESULTS_FAILURES = 'failures' |
61 KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' | 47 KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' |
62 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' | 48 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' |
63 KEY__HEADER__TIME_UPDATED = 'timeUpdated' | 49 KEY__HEADER__TIME_UPDATED = 'timeUpdated' |
64 KEY__HEADER__TYPE = 'type' | 50 KEY__HEADER__TYPE = 'type' |
65 KEY__NEW_IMAGE_URL = 'newImageUrl' | 51 KEY__NEW_IMAGE_URL = 'newImageUrl' |
66 KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED | 52 KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED |
67 KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED | 53 KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED |
68 KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON | 54 KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON |
69 KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED | 55 KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED |
70 | 56 |
71 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ | |
72 KEY__EXPECTATIONS__BUGS, | |
73 KEY__EXPECTATIONS__IGNOREFAILURE, | |
74 KEY__EXPECTATIONS__REVIEWED, | |
75 ] | |
76 | |
77 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) | 57 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
78 IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) | 58 IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) |
79 | |
80 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') | |
81 | |
82 DEFAULT_ACTUALS_DIR = '.gm-actuals' | |
83 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') | |
84 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( | |
85 PARENT_DIRECTORY, '.generated-images') | |
86 | |
87 | |
88 class Results(object): | |
89 """ Loads actual and expected GM results into an ImagePairSet. | |
90 | |
91 Loads actual and expected results from all builders, except for those skipped | |
92 by _ignore_builder(). | |
93 | |
94 Once this object has been constructed, the results (in self._results[]) | |
95 are immutable. If you want to update the results based on updated JSON | |
96 file contents, you will need to create a new Results object.""" | |
97 | |
98 def __init__(self, actuals_root=DEFAULT_ACTUALS_DIR, | |
99 expected_root=DEFAULT_EXPECTATIONS_DIR, | |
100 generated_images_root=DEFAULT_GENERATED_IMAGES_ROOT, | |
101 diff_base_url=None): | |
102 """ | |
103 Args: | |
104 actuals_root: root directory containing all actual-results.json files | |
105 expected_root: root directory containing all expected-results.json files | |
106 generated_images_root: directory within which to create all pixel diffs; | |
107 if this directory does not yet exist, it will be created | |
108 diff_base_url: base URL within which the client should look for diff | |
109 images; if not specified, defaults to a "file:///" URL representation | |
110 of generated_images_root | |
111 """ | |
112 time_start = int(time.time()) | |
113 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) | |
114 self._diff_base_url = ( | |
115 diff_base_url or | |
116 download_actuals.create_filepath_url(generated_images_root)) | |
117 self._actuals_root = actuals_root | |
118 self._expected_root = expected_root | |
119 self._load_actual_and_expected() | |
120 self._timestamp = int(time.time()) | |
121 logging.info('Results complete; took %d seconds.' % | |
122 (self._timestamp - time_start)) | |
123 | |
124 def get_timestamp(self): | |
125 """Return the time at which this object was created, in seconds past epoch | |
126 (UTC). | |
127 """ | |
128 return self._timestamp | |
129 | |
130 def edit_expectations(self, modifications): | |
131 """Edit the expectations stored within this object and write them back | |
132 to disk. | |
133 | |
134 Note that this will NOT update the results stored in self._results[] ; | |
135 in order to see those updates, you must instantiate a new Results object | |
136 based on the (now updated) files on disk. | |
137 | |
138 Args: | |
139 modifications: a list of dictionaries, one for each expectation to update: | |
140 | |
141 [ | |
142 { | |
143 imagepair.KEY__EXPECTATIONS_DATA: { | |
144 KEY__EXPECTATIONS__BUGS: [123, 456], | |
145 KEY__EXPECTATIONS__IGNOREFAILURE: false, | |
146 KEY__EXPECTATIONS__REVIEWED: true, | |
147 }, | |
148 imagepair.KEY__EXTRA_COLUMN_VALUES: { | |
149 KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x
86-Debug', | |
150 KEY__EXTRACOLUMN__CONFIG: '8888', | |
151 KEY__EXTRACOLUMN__TEST: 'bigmatrix', | |
152 }, | |
153 KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926
.png', | |
154 }, | |
155 ... | |
156 ] | |
157 | |
158 """ | |
159 expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) | |
160 for mod in modifications: | |
161 image_name = IMAGE_FILENAME_FORMATTER % ( | |
162 mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__TEST], | |
163 mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__CONFIG]) | |
164 _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( | |
165 mod[KEY__NEW_IMAGE_URL]) | |
166 allowed_digests = [[hash_type, int(hash_digest)]] | |
167 new_expectations = { | |
168 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, | |
169 } | |
170 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: | |
171 value = mod[imagepair.KEY__EXPECTATIONS_DATA].get(field) | |
172 if value is not None: | |
173 new_expectations[field] = value | |
174 builder_dict = expected_builder_dicts[ | |
175 mod[imagepair.KEY__EXTRA_COLUMN_VALUES][KEY__EXTRACOLUMN__BUILDER]] | |
176 builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) | |
177 if not builder_expectations: | |
178 builder_expectations = {} | |
179 builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations | |
180 builder_expectations[image_name] = new_expectations | |
181 Results._write_dicts_to_root(expected_builder_dicts, self._expected_root) | |
182 | |
183 def get_results_of_type(self, results_type): | |
184 """Return results of some/all tests (depending on 'results_type' parameter). | |
185 | |
186 Args: | |
187 results_type: string describing which types of results to include; must | |
188 be one of the RESULTS_* constants | |
189 | |
190 Results are returned in a dictionary as output by ImagePairSet.as_dict(). | |
191 """ | |
192 return self._results[results_type] | |
193 | |
194 def get_packaged_results_of_type(self, results_type, reload_seconds=None, | |
195 is_editable=False, is_exported=True): | |
196 """ Package the results of some/all tests as a complete response_dict. | |
197 | |
198 Args: | |
199 results_type: string indicating which set of results to return; | |
200 must be one of the RESULTS_* constants | |
201 reload_seconds: if specified, note that new results may be available once | |
202 these results are reload_seconds old | |
203 is_editable: whether clients are allowed to submit new baselines | |
204 is_exported: whether these results are being made available to other | |
205 network hosts | |
206 """ | |
207 response_dict = self._results[results_type] | |
208 time_updated = self.get_timestamp() | |
209 response_dict[KEY__HEADER] = { | |
210 KEY__HEADER__SCHEMA_VERSION: REBASELINE_SERVER_SCHEMA_VERSION_NUMBER, | |
211 | |
212 # Timestamps: | |
213 # 1. when this data was last updated | |
214 # 2. when the caller should check back for new data (if ever) | |
215 KEY__HEADER__TIME_UPDATED: time_updated, | |
216 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( | |
217 (time_updated+reload_seconds) if reload_seconds else None), | |
218 | |
219 # The type we passed to get_results_of_type() | |
220 KEY__HEADER__TYPE: results_type, | |
221 | |
222 # Hash of dataset, which the client must return with any edits-- | |
223 # this ensures that the edits were made to a particular dataset. | |
224 KEY__HEADER__DATAHASH: str(hash(repr( | |
225 response_dict[imagepairset.KEY__IMAGEPAIRS]))), | |
226 | |
227 # Whether the server will accept edits back. | |
228 KEY__HEADER__IS_EDITABLE: is_editable, | |
229 | |
230 # Whether the service is accessible from other hosts. | |
231 KEY__HEADER__IS_EXPORTED: is_exported, | |
232 } | |
233 return response_dict | |
234 | |
235 @staticmethod | |
236 def _ignore_builder(builder): | |
237 """Returns True if we should ignore expectations and actuals for a builder. | |
238 | |
239 This allows us to ignore builders for which we don't maintain expectations | |
240 (trybots, Valgrind, ASAN, TSAN), and avoid problems like | |
241 https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server | |
242 produces error when trying to add baselines for ASAN/TSAN builders') | |
243 | |
244 Args: | |
245 builder: name of this builder, as a string | |
246 | |
247 Returns: | |
248 True if we should ignore expectations and actuals for this builder. | |
249 """ | |
250 return (builder.endswith('-Trybot') or | |
251 ('Valgrind' in builder) or | |
252 ('TSAN' in builder) or | |
253 ('ASAN' in builder)) | |
254 | |
255 @staticmethod | |
256 def _read_dicts_from_root(root, pattern='*.json'): | |
257 """Read all JSON dictionaries within a directory tree. | |
258 | |
259 Args: | |
260 root: path to root of directory tree | |
261 pattern: which files to read within root (fnmatch-style pattern) | |
262 | |
263 Returns: | |
264 A meta-dictionary containing all the JSON dictionaries found within | |
265 the directory tree, keyed by the builder name of each dictionary. | |
266 | |
267 Raises: | |
268 IOError if root does not refer to an existing directory | |
269 """ | |
270 if not os.path.isdir(root): | |
271 raise IOError('no directory found at path %s' % root) | |
272 meta_dict = {} | |
273 for dirpath, dirnames, filenames in os.walk(root): | |
274 for matching_filename in fnmatch.filter(filenames, pattern): | |
275 builder = os.path.basename(dirpath) | |
276 if Results._ignore_builder(builder): | |
277 continue | |
278 fullpath = os.path.join(dirpath, matching_filename) | |
279 meta_dict[builder] = gm_json.LoadFromFile(fullpath) | |
280 return meta_dict | |
281 | |
282 @staticmethod | |
283 def _create_relative_url(hashtype_and_digest, test_name): | |
284 """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL. | |
285 | |
286 If we don't have a record of this image, returns None. | |
287 | |
288 Args: | |
289 hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we | |
290 don't have a record of this image | |
291 test_name: string; name of the GM test that created this image | |
292 """ | |
293 if not hashtype_and_digest: | |
294 return None | |
295 return gm_json.CreateGmRelativeUrl( | |
296 test_name=test_name, | |
297 hash_type=hashtype_and_digest[0], | |
298 hash_digest=hashtype_and_digest[1]) | |
299 | |
300 @staticmethod | |
301 def _write_dicts_to_root(meta_dict, root, pattern='*.json'): | |
302 """Write all per-builder dictionaries within meta_dict to files under | |
303 the root path. | |
304 | |
305 Security note: this will only write to files that already exist within | |
306 the root path (as found by os.walk() within root), so we don't need to | |
307 worry about malformed content writing to disk outside of root. | |
308 However, the data written to those files is not double-checked, so it | |
309 could contain poisonous data. | |
310 | |
311 Args: | |
312 meta_dict: a builder-keyed meta-dictionary containing all the JSON | |
313 dictionaries we want to write out | |
314 root: path to root of directory tree within which to write files | |
315 pattern: which files to write within root (fnmatch-style pattern) | |
316 | |
317 Raises: | |
318 IOError if root does not refer to an existing directory | |
319 KeyError if the set of per-builder dictionaries written out was | |
320 different than expected | |
321 """ | |
322 if not os.path.isdir(root): | |
323 raise IOError('no directory found at path %s' % root) | |
324 actual_builders_written = [] | |
325 for dirpath, dirnames, filenames in os.walk(root): | |
326 for matching_filename in fnmatch.filter(filenames, pattern): | |
327 builder = os.path.basename(dirpath) | |
328 if Results._ignore_builder(builder): | |
329 continue | |
330 per_builder_dict = meta_dict.get(builder) | |
331 if per_builder_dict is not None: | |
332 fullpath = os.path.join(dirpath, matching_filename) | |
333 gm_json.WriteToFile(per_builder_dict, fullpath) | |
334 actual_builders_written.append(builder) | |
335 | |
336 # Check: did we write out the set of per-builder dictionaries we | |
337 # expected to? | |
338 expected_builders_written = sorted(meta_dict.keys()) | |
339 actual_builders_written.sort() | |
340 if expected_builders_written != actual_builders_written: | |
341 raise KeyError( | |
342 'expected to write dicts for builders %s, but actually wrote them ' | |
343 'for builders %s' % ( | |
344 expected_builders_written, actual_builders_written)) | |
345 | |
346 def _load_actual_and_expected(self): | |
347 """Loads the results of all tests, across all builders (based on the | |
348 files within self._actuals_root and self._expected_root), | |
349 and stores them in self._results. | |
350 """ | |
351 logging.info('Reading actual-results JSON files from %s...' % | |
352 self._actuals_root) | |
353 actual_builder_dicts = Results._read_dicts_from_root(self._actuals_root) | |
354 logging.info('Reading expected-results JSON files from %s...' % | |
355 self._expected_root) | |
356 expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) | |
357 | |
358 all_image_pairs = imagepairset.ImagePairSet( | |
359 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, | |
360 diff_base_url=self._diff_base_url) | |
361 failing_image_pairs = imagepairset.ImagePairSet( | |
362 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, | |
363 diff_base_url=self._diff_base_url) | |
364 | |
365 all_image_pairs.ensure_extra_column_values_in_summary( | |
366 column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[ | |
367 KEY__RESULT_TYPE__FAILED, | |
368 KEY__RESULT_TYPE__FAILUREIGNORED, | |
369 KEY__RESULT_TYPE__NOCOMPARISON, | |
370 KEY__RESULT_TYPE__SUCCEEDED, | |
371 ]) | |
372 failing_image_pairs.ensure_extra_column_values_in_summary( | |
373 column_id=KEY__EXTRACOLUMN__RESULT_TYPE, values=[ | |
374 KEY__RESULT_TYPE__FAILED, | |
375 KEY__RESULT_TYPE__FAILUREIGNORED, | |
376 KEY__RESULT_TYPE__NOCOMPARISON, | |
377 ]) | |
378 | |
379 builders = sorted(actual_builder_dicts.keys()) | |
380 num_builders = len(builders) | |
381 builder_num = 0 | |
382 for builder in builders: | |
383 builder_num += 1 | |
384 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % | |
385 (builder_num, num_builders, builder)) | |
386 actual_results_for_this_builder = ( | |
387 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) | |
388 for result_type in sorted(actual_results_for_this_builder.keys()): | |
389 results_of_this_type = actual_results_for_this_builder[result_type] | |
390 if not results_of_this_type: | |
391 continue | |
392 for image_name in sorted(results_of_this_type.keys()): | |
393 (test, config) = IMAGE_FILENAME_RE.match(image_name).groups() | |
394 actual_image_relative_url = Results._create_relative_url( | |
395 hashtype_and_digest=results_of_this_type[image_name], | |
396 test_name=test) | |
397 | |
398 # Default empty expectations; overwrite these if we find any real ones | |
399 expectations_per_test = None | |
400 expected_image_relative_url = None | |
401 expectations_dict = None | |
402 try: | |
403 expectations_per_test = ( | |
404 expected_builder_dicts | |
405 [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name]) | |
406 # TODO(epoger): assumes a single allowed digest per test, which is | |
407 # fine; see https://code.google.com/p/skia/issues/detail?id=1787 | |
408 expected_image_hashtype_and_digest = ( | |
409 expectations_per_test | |
410 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]) | |
411 expected_image_relative_url = Results._create_relative_url( | |
412 hashtype_and_digest=expected_image_hashtype_and_digest, | |
413 test_name=test) | |
414 expectations_dict = {} | |
415 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: | |
416 expectations_dict[field] = expectations_per_test.get(field) | |
417 except (KeyError, TypeError): | |
418 # There are several cases in which we would expect to find | |
419 # no expectations for a given test: | |
420 # | |
421 # 1. result_type == NOCOMPARISON | |
422 # There are no expectations for this test yet! | |
423 # | |
424 # 2. alternate rendering mode failures (e.g. serialized) | |
425 # In cases like | |
426 # https://code.google.com/p/skia/issues/detail?id=1684 | |
427 # ('tileimagefilter GM test failing in serialized render mode'), | |
428 # the gm-actuals will list a failure for the alternate | |
429 # rendering mode even though we don't have explicit expectations | |
430 # for the test (the implicit expectation is that it must | |
431 # render the same in all rendering modes). | |
432 # | |
433 # Don't log type 1, because it is common. | |
434 # Log other types, because they are rare and we should know about | |
435 # them, but don't throw an exception, because we need to keep our | |
436 # tools working in the meanwhile! | |
437 if result_type != KEY__RESULT_TYPE__NOCOMPARISON: | |
438 logging.warning('No expectations found for test: %s' % { | |
439 KEY__EXTRACOLUMN__BUILDER: builder, | |
440 KEY__EXTRACOLUMN__RESULT_TYPE: result_type, | |
441 'image_name': image_name, | |
442 }) | |
443 | |
444 # If this test was recently rebaselined, it will remain in | |
445 # the 'failed' set of actuals until all the bots have | |
446 # cycled (although the expectations have indeed been set | |
447 # from the most recent actuals). Treat these as successes | |
448 # instead of failures. | |
449 # | |
450 # TODO(epoger): Do we need to do something similar in | |
451 # other cases, such as when we have recently marked a test | |
452 # as ignoreFailure but it still shows up in the 'failed' | |
453 # category? Maybe we should not rely on the result_type | |
454 # categories recorded within the gm_actuals AT ALL, and | |
455 # instead evaluate the result_type ourselves based on what | |
456 # we see in expectations vs actual checksum? | |
457 if expected_image_relative_url == actual_image_relative_url: | |
458 updated_result_type = KEY__RESULT_TYPE__SUCCEEDED | |
459 else: | |
460 updated_result_type = result_type | |
461 extra_columns_dict = { | |
462 KEY__EXTRACOLUMN__RESULT_TYPE: updated_result_type, | |
463 KEY__EXTRACOLUMN__BUILDER: builder, | |
464 KEY__EXTRACOLUMN__TEST: test, | |
465 KEY__EXTRACOLUMN__CONFIG: config, | |
466 } | |
467 try: | |
468 image_pair = imagepair.ImagePair( | |
469 image_diff_db=self._image_diff_db, | |
470 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, | |
471 imageA_relative_url=expected_image_relative_url, | |
472 imageB_relative_url=actual_image_relative_url, | |
473 expectations=expectations_dict, | |
474 extra_columns=extra_columns_dict) | |
475 all_image_pairs.add_image_pair(image_pair) | |
476 if updated_result_type != KEY__RESULT_TYPE__SUCCEEDED: | |
477 failing_image_pairs.add_image_pair(image_pair) | |
478 except Exception: | |
479 logging.exception('got exception while creating new ImagePair') | |
480 | |
481 self._results = { | |
482 KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), | |
483 KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), | |
484 } | |
485 | |
486 | |
487 def main(): | |
488 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', | |
489 datefmt='%m/%d/%Y %H:%M:%S', | |
490 level=logging.INFO) | |
491 parser = argparse.ArgumentParser() | |
492 parser.add_argument( | |
493 '--actuals', default=DEFAULT_ACTUALS_DIR, | |
494 help='Directory containing all actual-result JSON files') | |
495 parser.add_argument( | |
496 '--expectations', default=DEFAULT_EXPECTATIONS_DIR, | |
497 help='Directory containing all expected-result JSON files; defaults to ' | |
498 '\'%(default)s\' .') | |
499 parser.add_argument( | |
500 '--outfile', required=True, | |
501 help='File to write result summary into, in JSON format.') | |
502 parser.add_argument( | |
503 '--results', default=KEY__HEADER__RESULTS_FAILURES, | |
504 help='Which result types to include. Defaults to \'%(default)s\'; ' | |
505 'must be one of ' + | |
506 str([KEY__HEADER__RESULTS_FAILURES, KEY__HEADER__RESULTS_ALL])) | |
507 parser.add_argument( | |
508 '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT, | |
509 help='Directory within which to download images and generate diffs; ' | |
510 'defaults to \'%(default)s\' .') | |
511 args = parser.parse_args() | |
512 results = Results(actuals_root=args.actuals, | |
513 expected_root=args.expectations, | |
514 generated_images_root=args.workdir) | |
515 gm_json.WriteToFile( | |
516 results.get_packaged_results_of_type(results_type=args.results), | |
517 args.outfile) | |
518 | |
519 | |
520 if __name__ == '__main__': | |
521 main() | |
OLD | NEW |