Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(333)

Side by Side Diff: gm/rebaseline_server/compare_to_expectations.py

Issue 215503002: rebaseline_server: add --compare-configs option (Closed) Base URL: https://skia.googlesource.com/skia.git@master
Patch Set: rebase Created 6 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 2
3 """ 3 """
4 Copyright 2013 Google Inc. 4 Copyright 2013 Google Inc.
5 5
6 Use of this source code is governed by a BSD-style license that can be 6 Use of this source code is governed by a BSD-style license that can be
7 found in the LICENSE file. 7 found in the LICENSE file.
8 8
9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. 9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
10 """ 10 """
(...skipping 29 matching lines...) Expand all
40 import imagediffdb 40 import imagediffdb
41 import imagepair 41 import imagepair
42 import imagepairset 42 import imagepairset
43 import results 43 import results
44 44
45 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ 45 EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
46 results.KEY__EXPECTATIONS__BUGS, 46 results.KEY__EXPECTATIONS__BUGS,
47 results.KEY__EXPECTATIONS__IGNOREFAILURE, 47 results.KEY__EXPECTATIONS__IGNOREFAILURE,
48 results.KEY__EXPECTATIONS__REVIEWED, 48 results.KEY__EXPECTATIONS__REVIEWED,
49 ] 49 ]
50 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
50 51
51 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') 52 IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')
52 53
53 DEFAULT_ACTUALS_DIR = '.gm-actuals'
54 DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
55 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
56 PARENT_DIRECTORY, '.generated-images')
57 54
58 55 class ExpectationComparisons(results.BaseComparisons):
59 class Results(object): 56 """Loads actual and expected GM results into an ImagePairSet.
60 """ Loads actual and expected GM results into an ImagePairSet.
61 57
62 Loads actual and expected results from all builders, except for those skipped 58 Loads actual and expected results from all builders, except for those skipped
63 by _ignore_builder(). 59 by _ignore_builder().
64 60
65 Once this object has been constructed, the results (in self._results[]) 61 Once this object has been constructed, the results (in self._results[])
66 are immutable. If you want to update the results based on updated JSON 62 are immutable. If you want to update the results based on updated JSON
67 file contents, you will need to create a new Results object.""" 63 file contents, you will need to create a new ExpectationComparisons object."""
68 64
69 def __init__(self, actuals_root=DEFAULT_ACTUALS_DIR, 65 def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR,
70 expected_root=DEFAULT_EXPECTATIONS_DIR, 66 expected_root=DEFAULT_EXPECTATIONS_DIR,
71 generated_images_root=DEFAULT_GENERATED_IMAGES_ROOT, 67 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
72 diff_base_url=None): 68 diff_base_url=None):
73 """ 69 """
74 Args: 70 Args:
75 actuals_root: root directory containing all actual-results.json files 71 actuals_root: root directory containing all actual-results.json files
76 expected_root: root directory containing all expected-results.json files 72 expected_root: root directory containing all expected-results.json files
77 generated_images_root: directory within which to create all pixel diffs; 73 generated_images_root: directory within which to create all pixel diffs;
78 if this directory does not yet exist, it will be created 74 if this directory does not yet exist, it will be created
79 diff_base_url: base URL within which the client should look for diff 75 diff_base_url: base URL within which the client should look for diff
80 images; if not specified, defaults to a "file:///" URL representation 76 images; if not specified, defaults to a "file:///" URL representation
81 of generated_images_root 77 of generated_images_root
82 """ 78 """
83 time_start = int(time.time()) 79 time_start = int(time.time())
84 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) 80 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
85 self._diff_base_url = ( 81 self._diff_base_url = (
86 diff_base_url or 82 diff_base_url or
87 download_actuals.create_filepath_url(generated_images_root)) 83 download_actuals.create_filepath_url(generated_images_root))
88 self._actuals_root = actuals_root 84 self._actuals_root = actuals_root
89 self._expected_root = expected_root 85 self._expected_root = expected_root
90 self._load_actual_and_expected() 86 self._load_actual_and_expected()
91 self._timestamp = int(time.time()) 87 self._timestamp = int(time.time())
92 logging.info('Results complete; took %d seconds.' % 88 logging.info('Results complete; took %d seconds.' %
93 (self._timestamp - time_start)) 89 (self._timestamp - time_start))
94 90
95 def get_timestamp(self):
96 """Return the time at which this object was created, in seconds past epoch
97 (UTC).
98 """
99 return self._timestamp
100
101 def edit_expectations(self, modifications): 91 def edit_expectations(self, modifications):
102 """Edit the expectations stored within this object and write them back 92 """Edit the expectations stored within this object and write them back
103 to disk. 93 to disk.
104 94
105 Note that this will NOT update the results stored in self._results[] ; 95 Note that this will NOT update the results stored in self._results[] ;
106 in order to see those updates, you must instantiate a new Results object 96 in order to see those updates, you must instantiate a new
107 based on the (now updated) files on disk. 97 ExpectationComparisons object based on the (now updated) files on disk.
108 98
109 Args: 99 Args:
110 modifications: a list of dictionaries, one for each expectation to update: 100 modifications: a list of dictionaries, one for each expectation to update:
111 101
112 [ 102 [
113 { 103 {
114 imagepair.KEY__EXPECTATIONS_DATA: { 104 imagepair.KEY__EXPECTATIONS_DATA: {
115 results.KEY__EXPECTATIONS__BUGS: [123, 456], 105 results.KEY__EXPECTATIONS__BUGS: [123, 456],
116 results.KEY__EXPECTATIONS__IGNOREFAILURE: false, 106 results.KEY__EXPECTATIONS__IGNOREFAILURE: false,
117 results.KEY__EXPECTATIONS__REVIEWED: true, 107 results.KEY__EXPECTATIONS__REVIEWED: true,
118 }, 108 },
119 imagepair.KEY__EXTRA_COLUMN_VALUES: { 109 imagepair.KEY__EXTRA_COLUMN_VALUES: {
120 results.KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeFor ce320M-x86-Debug', 110 results.KEY__EXTRACOLUMN__BUILDER: 'Test-Mac10.6-MacMini4.1-GeFor ce320M-x86-Debug',
121 results.KEY__EXTRACOLUMN__CONFIG: '8888', 111 results.KEY__EXTRACOLUMN__CONFIG: '8888',
122 results.KEY__EXTRACOLUMN__TEST: 'bigmatrix', 112 results.KEY__EXTRACOLUMN__TEST: 'bigmatrix',
123 }, 113 },
124 results.KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/108944080240 79689926.png', 114 results.KEY__NEW_IMAGE_URL: 'bitmap-64bitMD5/bigmatrix/108944080240 79689926.png',
125 }, 115 },
126 ... 116 ...
127 ] 117 ]
128 118
129 """ 119 """
130 expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) 120 expected_builder_dicts = ExpectationComparisons._read_dicts_from_root(
121 self._expected_root)
131 for mod in modifications: 122 for mod in modifications:
132 image_name = results.IMAGE_FILENAME_FORMATTER % ( 123 image_name = results.IMAGE_FILENAME_FORMATTER % (
133 mod[imagepair.KEY__EXTRA_COLUMN_VALUES] 124 mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
134 [results.KEY__EXTRACOLUMN__TEST], 125 [results.KEY__EXTRACOLUMN__TEST],
135 mod[imagepair.KEY__EXTRA_COLUMN_VALUES] 126 mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
136 [results.KEY__EXTRACOLUMN__CONFIG]) 127 [results.KEY__EXTRACOLUMN__CONFIG])
137 _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( 128 _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl(
138 mod[results.KEY__NEW_IMAGE_URL]) 129 mod[results.KEY__NEW_IMAGE_URL])
139 allowed_digests = [[hash_type, int(hash_digest)]] 130 allowed_digests = [[hash_type, int(hash_digest)]]
140 new_expectations = { 131 new_expectations = {
141 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, 132 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests,
142 } 133 }
143 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: 134 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
144 value = mod[imagepair.KEY__EXPECTATIONS_DATA].get(field) 135 value = mod[imagepair.KEY__EXPECTATIONS_DATA].get(field)
145 if value is not None: 136 if value is not None:
146 new_expectations[field] = value 137 new_expectations[field] = value
147 builder_dict = expected_builder_dicts[ 138 builder_dict = expected_builder_dicts[
148 mod[imagepair.KEY__EXTRA_COLUMN_VALUES] 139 mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
149 [results.KEY__EXTRACOLUMN__BUILDER]] 140 [results.KEY__EXTRACOLUMN__BUILDER]]
150 builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) 141 builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
151 if not builder_expectations: 142 if not builder_expectations:
152 builder_expectations = {} 143 builder_expectations = {}
153 builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations 144 builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations
154 builder_expectations[image_name] = new_expectations 145 builder_expectations[image_name] = new_expectations
155 Results._write_dicts_to_root(expected_builder_dicts, self._expected_root) 146 ExpectationComparisons._write_dicts_to_root(
156 147 expected_builder_dicts, self._expected_root)
157 def get_results_of_type(self, results_type):
158 """Return results of some/all tests (depending on 'results_type' parameter).
159
160 Args:
161 results_type: string describing which types of results to include; must
162 be one of the RESULTS_* constants
163
164 Results are returned in a dictionary as output by ImagePairSet.as_dict().
165 """
166 return self._results[results_type]
167
168 def get_packaged_results_of_type(self, results_type, reload_seconds=None,
169 is_editable=False, is_exported=True):
170 """ Package the results of some/all tests as a complete response_dict.
171
172 Args:
173 results_type: string indicating which set of results to return;
174 must be one of the RESULTS_* constants
175 reload_seconds: if specified, note that new results may be available once
176 these results are reload_seconds old
177 is_editable: whether clients are allowed to submit new baselines
178 is_exported: whether these results are being made available to other
179 network hosts
180 """
181 response_dict = self._results[results_type]
182 time_updated = self.get_timestamp()
183 response_dict[results.KEY__HEADER] = {
184 results.KEY__HEADER__SCHEMA_VERSION: (
185 results.REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
186
187 # Timestamps:
188 # 1. when this data was last updated
189 # 2. when the caller should check back for new data (if ever)
190 results.KEY__HEADER__TIME_UPDATED: time_updated,
191 results.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
192 (time_updated+reload_seconds) if reload_seconds else None),
193
194 # The type we passed to get_results_of_type()
195 results.KEY__HEADER__TYPE: results_type,
196
197 # Hash of dataset, which the client must return with any edits--
198 # this ensures that the edits were made to a particular dataset.
199 results.KEY__HEADER__DATAHASH: str(hash(repr(
200 response_dict[imagepairset.KEY__IMAGEPAIRS]))),
201
202 # Whether the server will accept edits back.
203 results.KEY__HEADER__IS_EDITABLE: is_editable,
204
205 # Whether the service is accessible from other hosts.
206 results.KEY__HEADER__IS_EXPORTED: is_exported,
207 }
208 return response_dict
209
210 @staticmethod
211 def _ignore_builder(builder):
212 """Returns True if we should ignore expectations and actuals for a builder.
213
214 This allows us to ignore builders for which we don't maintain expectations
215 (trybots, Valgrind, ASAN, TSAN), and avoid problems like
216 https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
217 produces error when trying to add baselines for ASAN/TSAN builders')
218
219 Args:
220 builder: name of this builder, as a string
221
222 Returns:
223 True if we should ignore expectations and actuals for this builder.
224 """
225 return (builder.endswith('-Trybot') or
226 ('Valgrind' in builder) or
227 ('TSAN' in builder) or
228 ('ASAN' in builder))
229
230 @staticmethod
231 def _read_dicts_from_root(root, pattern='*.json'):
232 """Read all JSON dictionaries within a directory tree.
233
234 Args:
235 root: path to root of directory tree
236 pattern: which files to read within root (fnmatch-style pattern)
237
238 Returns:
239 A meta-dictionary containing all the JSON dictionaries found within
240 the directory tree, keyed by the builder name of each dictionary.
241
242 Raises:
243 IOError if root does not refer to an existing directory
244 """
245 if not os.path.isdir(root):
246 raise IOError('no directory found at path %s' % root)
247 meta_dict = {}
248 for dirpath, dirnames, filenames in os.walk(root):
249 for matching_filename in fnmatch.filter(filenames, pattern):
250 builder = os.path.basename(dirpath)
251 if Results._ignore_builder(builder):
252 continue
253 fullpath = os.path.join(dirpath, matching_filename)
254 meta_dict[builder] = gm_json.LoadFromFile(fullpath)
255 return meta_dict
256
257 @staticmethod
258 def _create_relative_url(hashtype_and_digest, test_name):
259 """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
260
261 If we don't have a record of this image, returns None.
262
263 Args:
264 hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
265 don't have a record of this image
266 test_name: string; name of the GM test that created this image
267 """
268 if not hashtype_and_digest:
269 return None
270 return gm_json.CreateGmRelativeUrl(
271 test_name=test_name,
272 hash_type=hashtype_and_digest[0],
273 hash_digest=hashtype_and_digest[1])
274 148
275 @staticmethod 149 @staticmethod
276 def _write_dicts_to_root(meta_dict, root, pattern='*.json'): 150 def _write_dicts_to_root(meta_dict, root, pattern='*.json'):
277 """Write all per-builder dictionaries within meta_dict to files under 151 """Write all per-builder dictionaries within meta_dict to files under
278 the root path. 152 the root path.
279 153
280 Security note: this will only write to files that already exist within 154 Security note: this will only write to files that already exist within
281 the root path (as found by os.walk() within root), so we don't need to 155 the root path (as found by os.walk() within root), so we don't need to
282 worry about malformed content writing to disk outside of root. 156 worry about malformed content writing to disk outside of root.
283 However, the data written to those files is not double-checked, so it 157 However, the data written to those files is not double-checked, so it
284 could contain poisonous data. 158 could contain poisonous data.
285 159
286 Args: 160 Args:
287 meta_dict: a builder-keyed meta-dictionary containing all the JSON 161 meta_dict: a builder-keyed meta-dictionary containing all the JSON
288 dictionaries we want to write out 162 dictionaries we want to write out
289 root: path to root of directory tree within which to write files 163 root: path to root of directory tree within which to write files
290 pattern: which files to write within root (fnmatch-style pattern) 164 pattern: which files to write within root (fnmatch-style pattern)
291 165
292 Raises: 166 Raises:
293 IOError if root does not refer to an existing directory 167 IOError if root does not refer to an existing directory
294 KeyError if the set of per-builder dictionaries written out was 168 KeyError if the set of per-builder dictionaries written out was
295 different than expected 169 different than expected
296 """ 170 """
297 if not os.path.isdir(root): 171 if not os.path.isdir(root):
298 raise IOError('no directory found at path %s' % root) 172 raise IOError('no directory found at path %s' % root)
299 actual_builders_written = [] 173 actual_builders_written = []
300 for dirpath, dirnames, filenames in os.walk(root): 174 for dirpath, dirnames, filenames in os.walk(root):
301 for matching_filename in fnmatch.filter(filenames, pattern): 175 for matching_filename in fnmatch.filter(filenames, pattern):
302 builder = os.path.basename(dirpath) 176 builder = os.path.basename(dirpath)
303 if Results._ignore_builder(builder): 177 if ExpectationComparisons._ignore_builder(builder):
304 continue 178 continue
305 per_builder_dict = meta_dict.get(builder) 179 per_builder_dict = meta_dict.get(builder)
306 if per_builder_dict is not None: 180 if per_builder_dict is not None:
307 fullpath = os.path.join(dirpath, matching_filename) 181 fullpath = os.path.join(dirpath, matching_filename)
308 gm_json.WriteToFile(per_builder_dict, fullpath) 182 gm_json.WriteToFile(per_builder_dict, fullpath)
309 actual_builders_written.append(builder) 183 actual_builders_written.append(builder)
310 184
311 # Check: did we write out the set of per-builder dictionaries we 185 # Check: did we write out the set of per-builder dictionaries we
312 # expected to? 186 # expected to?
313 expected_builders_written = sorted(meta_dict.keys()) 187 expected_builders_written = sorted(meta_dict.keys())
314 actual_builders_written.sort() 188 actual_builders_written.sort()
315 if expected_builders_written != actual_builders_written: 189 if expected_builders_written != actual_builders_written:
316 raise KeyError( 190 raise KeyError(
317 'expected to write dicts for builders %s, but actually wrote them ' 191 'expected to write dicts for builders %s, but actually wrote them '
318 'for builders %s' % ( 192 'for builders %s' % (
319 expected_builders_written, actual_builders_written)) 193 expected_builders_written, actual_builders_written))
320 194
321 def _load_actual_and_expected(self): 195 def _load_actual_and_expected(self):
322 """Loads the results of all tests, across all builders (based on the 196 """Loads the results of all tests, across all builders (based on the
323 files within self._actuals_root and self._expected_root), 197 files within self._actuals_root and self._expected_root),
324 and stores them in self._results. 198 and stores them in self._results.
325 """ 199 """
326 logging.info('Reading actual-results JSON files from %s...' % 200 logging.info('Reading actual-results JSON files from %s...' %
327 self._actuals_root) 201 self._actuals_root)
328 actual_builder_dicts = Results._read_dicts_from_root(self._actuals_root) 202 actual_builder_dicts = ExpectationComparisons._read_dicts_from_root(
203 self._actuals_root)
329 logging.info('Reading expected-results JSON files from %s...' % 204 logging.info('Reading expected-results JSON files from %s...' %
330 self._expected_root) 205 self._expected_root)
331 expected_builder_dicts = Results._read_dicts_from_root(self._expected_root) 206 expected_builder_dicts = ExpectationComparisons._read_dicts_from_root(
207 self._expected_root)
332 208
333 all_image_pairs = imagepairset.ImagePairSet( 209 all_image_pairs = imagepairset.ImagePairSet(
334 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, 210 descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
335 diff_base_url=self._diff_base_url) 211 diff_base_url=self._diff_base_url)
336 failing_image_pairs = imagepairset.ImagePairSet( 212 failing_image_pairs = imagepairset.ImagePairSet(
337 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, 213 descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
338 diff_base_url=self._diff_base_url) 214 diff_base_url=self._diff_base_url)
339 215
340 all_image_pairs.ensure_extra_column_values_in_summary( 216 all_image_pairs.ensure_extra_column_values_in_summary(
341 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ 217 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
(...skipping 17 matching lines...) Expand all
359 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % 235 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
360 (builder_num, num_builders, builder)) 236 (builder_num, num_builders, builder))
361 actual_results_for_this_builder = ( 237 actual_results_for_this_builder = (
362 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) 238 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
363 for result_type in sorted(actual_results_for_this_builder.keys()): 239 for result_type in sorted(actual_results_for_this_builder.keys()):
364 results_of_this_type = actual_results_for_this_builder[result_type] 240 results_of_this_type = actual_results_for_this_builder[result_type]
365 if not results_of_this_type: 241 if not results_of_this_type:
366 continue 242 continue
367 for image_name in sorted(results_of_this_type.keys()): 243 for image_name in sorted(results_of_this_type.keys()):
368 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() 244 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
369 actual_image_relative_url = Results._create_relative_url( 245 actual_image_relative_url = (
370 hashtype_and_digest=results_of_this_type[image_name], 246 ExpectationComparisons._create_relative_url(
371 test_name=test) 247 hashtype_and_digest=results_of_this_type[image_name],
248 test_name=test))
372 249
373 # Default empty expectations; overwrite these if we find any real ones 250 # Default empty expectations; overwrite these if we find any real ones
374 expectations_per_test = None 251 expectations_per_test = None
375 expected_image_relative_url = None 252 expected_image_relative_url = None
376 expectations_dict = None 253 expectations_dict = None
377 try: 254 try:
378 expectations_per_test = ( 255 expectations_per_test = (
379 expected_builder_dicts 256 expected_builder_dicts
380 [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name]) 257 [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name])
381 # TODO(epoger): assumes a single allowed digest per test, which is 258 # TODO(epoger): assumes a single allowed digest per test, which is
382 # fine; see https://code.google.com/p/skia/issues/detail?id=1787 259 # fine; see https://code.google.com/p/skia/issues/detail?id=1787
383 expected_image_hashtype_and_digest = ( 260 expected_image_hashtype_and_digest = (
384 expectations_per_test 261 expectations_per_test
385 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]) 262 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0])
386 expected_image_relative_url = Results._create_relative_url( 263 expected_image_relative_url = (
387 hashtype_and_digest=expected_image_hashtype_and_digest, 264 ExpectationComparisons._create_relative_url(
388 test_name=test) 265 hashtype_and_digest=expected_image_hashtype_and_digest,
266 test_name=test))
389 expectations_dict = {} 267 expectations_dict = {}
390 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: 268 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
391 expectations_dict[field] = expectations_per_test.get(field) 269 expectations_dict[field] = expectations_per_test.get(field)
392 except (KeyError, TypeError): 270 except (KeyError, TypeError):
393 # There are several cases in which we would expect to find 271 # There are several cases in which we would expect to find
394 # no expectations for a given test: 272 # no expectations for a given test:
395 # 273 #
396 # 1. result_type == NOCOMPARISON 274 # 1. result_type == NOCOMPARISON
397 # There are no expectations for this test yet! 275 # There are no expectations for this test yet!
398 # 276 #
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after
458 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), 336 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
459 } 337 }
460 338
461 339
462 def main(): 340 def main():
463 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', 341 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
464 datefmt='%m/%d/%Y %H:%M:%S', 342 datefmt='%m/%d/%Y %H:%M:%S',
465 level=logging.INFO) 343 level=logging.INFO)
466 parser = argparse.ArgumentParser() 344 parser = argparse.ArgumentParser()
467 parser.add_argument( 345 parser.add_argument(
468 '--actuals', default=DEFAULT_ACTUALS_DIR, 346 '--actuals', default=results.DEFAULT_ACTUALS_DIR,
469 help='Directory containing all actual-result JSON files; defaults to ' 347 help='Directory containing all actual-result JSON files; defaults to '
470 '\'%(default)s\' .') 348 '\'%(default)s\' .')
471 parser.add_argument( 349 parser.add_argument(
472 '--expectations', default=DEFAULT_EXPECTATIONS_DIR, 350 '--expectations', default=DEFAULT_EXPECTATIONS_DIR,
473 help='Directory containing all expected-result JSON files; defaults to ' 351 help='Directory containing all expected-result JSON files; defaults to '
474 '\'%(default)s\' .') 352 '\'%(default)s\' .')
475 parser.add_argument( 353 parser.add_argument(
476 '--outfile', required=True, 354 '--outfile', required=True,
477 help='File to write result summary into, in JSON format.') 355 help='File to write result summary into, in JSON format.')
478 parser.add_argument( 356 parser.add_argument(
479 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, 357 '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
480 help='Which result types to include. Defaults to \'%(default)s\'; ' 358 help='Which result types to include. Defaults to \'%(default)s\'; '
481 'must be one of ' + 359 'must be one of ' +
482 str([results.KEY__HEADER__RESULTS_FAILURES, 360 str([results.KEY__HEADER__RESULTS_FAILURES,
483 results.KEY__HEADER__RESULTS_ALL])) 361 results.KEY__HEADER__RESULTS_ALL]))
484 parser.add_argument( 362 parser.add_argument(
485 '--workdir', default=DEFAULT_GENERATED_IMAGES_ROOT, 363 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
486 help='Directory within which to download images and generate diffs; ' 364 help='Directory within which to download images and generate diffs; '
487 'defaults to \'%(default)s\' .') 365 'defaults to \'%(default)s\' .')
488 args = parser.parse_args() 366 args = parser.parse_args()
489 results_obj = Results(actuals_root=args.actuals, 367 results_obj = ExpectationComparisons(actuals_root=args.actuals,
490 expected_root=args.expectations, 368 expected_root=args.expectations,
491 generated_images_root=args.workdir) 369 generated_images_root=args.workdir)
492 gm_json.WriteToFile( 370 gm_json.WriteToFile(
493 results_obj.get_packaged_results_of_type(results_type=args.results), 371 results_obj.get_packaged_results_of_type(results_type=args.results),
494 args.outfile) 372 args.outfile)
495 373
496 374
497 if __name__ == '__main__': 375 if __name__ == '__main__':
498 main() 376 main()
OLDNEW
« no previous file with comments | « gm/rebaseline_server/compare_configs_test.py ('k') | gm/rebaseline_server/compare_to_expectations_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698