OLD | NEW |
| (Empty) |
1 #!/usr/bin/python | |
2 | |
3 ''' | |
4 Copyright 2012 Google Inc. | |
5 | |
6 Use of this source code is governed by a BSD-style license that can be | |
7 found in the LICENSE file. | |
8 ''' | |
9 | |
10 ''' | |
11 Rebaselines the given GM tests, on all bots and all configurations. | |
12 ''' | |
13 | |
14 # System-level imports | |
15 import argparse | |
16 import json | |
17 import os | |
18 import re | |
19 import subprocess | |
20 import sys | |
21 import urllib2 | |
22 | |
23 # Imports from within Skia | |
24 # | |
25 # We need to add the 'gm' directory, so that we can import gm_json.py within | |
26 # that directory. That script allows us to parse the actual-results.json file | |
27 # written out by the GM tool. | |
28 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* | |
29 # so any dirs that are already in the PYTHONPATH will be preferred. | |
30 # | |
31 # This assumes that the 'gm' directory has been checked out as a sibling of | |
32 # the 'tools' directory containing this script, which will be the case if | |
33 # 'trunk' was checked out as a single unit. | |
34 GM_DIRECTORY = os.path.realpath( | |
35 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) | |
36 if GM_DIRECTORY not in sys.path: | |
37 sys.path.append(GM_DIRECTORY) | |
38 import gm_json | |
39 | |
40 # TODO(epoger): In the long run, we want to build this list automatically, | |
41 # but for now we hard-code it until we can properly address | |
42 # https://code.google.com/p/skia/issues/detail?id=1544 | |
43 # ('live query of builder list makes rebaseline.py slow to start up') | |
44 TEST_BUILDERS = [ | |
45 'Test-Android-GalaxyNexus-SGX540-Arm7-Debug', | |
46 'Test-Android-GalaxyNexus-SGX540-Arm7-Release', | |
47 'Test-Android-IntelRhb-SGX544-x86-Debug', | |
48 'Test-Android-IntelRhb-SGX544-x86-Release', | |
49 'Test-Android-Nexus10-MaliT604-Arm7-Debug', | |
50 'Test-Android-Nexus10-MaliT604-Arm7-Release', | |
51 'Test-Android-Nexus4-Adreno320-Arm7-Debug', | |
52 'Test-Android-Nexus4-Adreno320-Arm7-Release', | |
53 'Test-Android-Nexus7-Tegra3-Arm7-Debug', | |
54 'Test-Android-Nexus7-Tegra3-Arm7-Release', | |
55 'Test-Android-NexusS-SGX540-Arm7-Debug', | |
56 'Test-Android-NexusS-SGX540-Arm7-Release', | |
57 'Test-Android-Xoom-Tegra2-Arm7-Debug', | |
58 'Test-Android-Xoom-Tegra2-Arm7-Release', | |
59 'Test-ChromeOS-Alex-GMA3150-x86-Debug', | |
60 'Test-ChromeOS-Alex-GMA3150-x86-Release', | |
61 'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug', | |
62 'Test-ChromeOS-Daisy-MaliT604-Arm7-Release', | |
63 'Test-ChromeOS-Link-HD4000-x86_64-Debug', | |
64 'Test-ChromeOS-Link-HD4000-x86_64-Release', | |
65 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', | |
66 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release', | |
67 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug', | |
68 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release', | |
69 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug', | |
70 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release', | |
71 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug', | |
72 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release', | |
73 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug', | |
74 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release', | |
75 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug', | |
76 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release', | |
77 'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind', | |
78 'Test-Ubuntu12-ShuttleA-GTX660-x86-Debug', | |
79 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release', | |
80 'Test-Ubuntu12-ShuttleA-GTX660-x86_64-Debug', | |
81 'Test-Ubuntu12-ShuttleA-GTX660-x86_64-Release', | |
82 'Test-Ubuntu13.10-ShuttleA-NoGPU-x86_64-Debug', | |
83 'Test-Win7-ShuttleA-HD2000-x86-Debug', | |
84 'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE', | |
85 'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite', | |
86 'Test-Win7-ShuttleA-HD2000-x86-Release', | |
87 'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE', | |
88 'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite', | |
89 'Test-Win7-ShuttleA-HD2000-x86_64-Debug', | |
90 'Test-Win7-ShuttleA-HD2000-x86_64-Release', | |
91 'Test-Win8-ShuttleA-GTX660-x86-Debug', | |
92 'Test-Win8-ShuttleA-GTX660-x86-Release', | |
93 'Test-Win8-ShuttleA-GTX660-x86_64-Debug', | |
94 'Test-Win8-ShuttleA-GTX660-x86_64-Release', | |
95 'Test-Win8-ShuttleA-HD7770-x86-Debug', | |
96 'Test-Win8-ShuttleA-HD7770-x86-Release', | |
97 'Test-Win8-ShuttleA-HD7770-x86_64-Debug', | |
98 'Test-Win8-ShuttleA-HD7770-x86_64-Release', | |
99 ] | |
100 | |
101 # TODO: Get this from builder_name_schema in buildbot. | |
102 TRYBOT_SUFFIX = '-Trybot' | |
103 | |
104 | |
105 class _InternalException(Exception): | |
106 pass | |
107 | |
108 class ExceptionHandler(object): | |
109 """ Object that handles exceptions, either raising them immediately or | |
110 collecting them to display later on.""" | |
111 | |
112 # params: | |
113 def __init__(self, keep_going_on_failure=False): | |
114 """ | |
115 params: | |
116 keep_going_on_failure: if False, report failures and quit right away; | |
117 if True, collect failures until | |
118 ReportAllFailures() is called | |
119 """ | |
120 self._keep_going_on_failure = keep_going_on_failure | |
121 self._failures_encountered = [] | |
122 | |
123 def RaiseExceptionOrContinue(self): | |
124 """ We have encountered an exception; either collect the info and keep | |
125 going, or exit the program right away.""" | |
126 # Get traceback information about the most recently raised exception. | |
127 exc_info = sys.exc_info() | |
128 | |
129 if self._keep_going_on_failure: | |
130 print >> sys.stderr, ('WARNING: swallowing exception %s' % | |
131 repr(exc_info[1])) | |
132 self._failures_encountered.append(exc_info) | |
133 else: | |
134 print >> sys.stderr, ( | |
135 '\nHalting at first exception.\n' + | |
136 'Please file a bug to epoger@google.com at ' + | |
137 'https://code.google.com/p/skia/issues/entry, containing the ' + | |
138 'command you ran and the following stack trace.\n\n' + | |
139 'Afterwards, you can re-run with the --keep-going-on-failure ' + | |
140 'option set.\n') | |
141 raise exc_info[1], None, exc_info[2] | |
142 | |
143 def ReportAllFailures(self): | |
144 if self._failures_encountered: | |
145 print >> sys.stderr, ('Encountered %d failures (see above).' % | |
146 len(self._failures_encountered)) | |
147 sys.exit(1) | |
148 | |
149 | |
150 # Object that rebaselines a JSON expectations file (not individual image files). | |
151 class JsonRebaseliner(object): | |
152 | |
153 # params: | |
154 # expectations_root: root directory of all expectations JSON files | |
155 # expectations_input_filename: filename (under expectations_root) of JSON | |
156 # expectations file to read; typically | |
157 # "expected-results.json" | |
158 # expectations_output_filename: filename (under expectations_root) to | |
159 # which updated expectations should be | |
160 # written; typically the same as | |
161 # expectations_input_filename, to overwrite | |
162 # the old content | |
163 # actuals_base_url: base URL from which to read actual-result JSON files | |
164 # actuals_filename: filename (under actuals_base_url) from which to read a | |
165 # summary of results; typically "actual-results.json" | |
166 # exception_handler: reference to rebaseline.ExceptionHandler object | |
167 # tests: list of tests to rebaseline, or None if we should rebaseline | |
168 # whatever files the JSON results summary file tells us to | |
169 # configs: which configs to run for each test, or None if we should | |
170 # rebaseline whatever configs the JSON results summary file tells | |
171 # us to | |
172 # add_new: if True, add expectations for tests which don't have any yet | |
173 # add_ignored: if True, add expectations for tests for which failures are | |
174 # currently ignored | |
175 # bugs: optional list of bug numbers which pertain to these expectations | |
176 # notes: free-form text notes to add to all updated expectations | |
177 # mark_unreviewed: if True, mark these expectations as NOT having been | |
178 # reviewed by a human; otherwise, leave that field blank. | |
179 # Currently, there is no way to make this script mark | |
180 # expectations as reviewed-by-human=True. | |
181 # TODO(epoger): Add that capability to a review tool. | |
182 # mark_ignore_failure: if True, mark failures of a given test as being | |
183 # ignored. | |
184 # from_trybot: if True, read actual-result JSON files generated from a | |
185 # trybot run rather than a waterfall run. | |
186 def __init__(self, expectations_root, expectations_input_filename, | |
187 expectations_output_filename, actuals_base_url, | |
188 actuals_filename, exception_handler, | |
189 tests=None, configs=None, add_new=False, add_ignored=False, | |
190 bugs=None, notes=None, mark_unreviewed=None, | |
191 mark_ignore_failure=False, from_trybot=False): | |
192 self._expectations_root = expectations_root | |
193 self._expectations_input_filename = expectations_input_filename | |
194 self._expectations_output_filename = expectations_output_filename | |
195 self._tests = tests | |
196 self._configs = configs | |
197 self._actuals_base_url = actuals_base_url | |
198 self._actuals_filename = actuals_filename | |
199 self._exception_handler = exception_handler | |
200 self._add_new = add_new | |
201 self._add_ignored = add_ignored | |
202 self._bugs = bugs | |
203 self._notes = notes | |
204 self._mark_unreviewed = mark_unreviewed | |
205 self._mark_ignore_failure = mark_ignore_failure; | |
206 if self._tests or self._configs: | |
207 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) | |
208 else: | |
209 self._image_filename_re = None | |
210 self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) | |
211 self._from_trybot = from_trybot | |
212 | |
213 # Executes subprocess.call(cmd). | |
214 # Raises an Exception if the command fails. | |
215 def _Call(self, cmd): | |
216 if subprocess.call(cmd) != 0: | |
217 raise _InternalException('error running command: ' + ' '.join(cmd)) | |
218 | |
219 # Returns the full contents of filepath, as a single string. | |
220 # If filepath looks like a URL, try to read it that way instead of as | |
221 # a path on local storage. | |
222 # | |
223 # Raises _InternalException if there is a problem. | |
224 def _GetFileContents(self, filepath): | |
225 if filepath.startswith('http:') or filepath.startswith('https:'): | |
226 try: | |
227 return urllib2.urlopen(filepath).read() | |
228 except urllib2.HTTPError as e: | |
229 raise _InternalException('unable to read URL %s: %s' % ( | |
230 filepath, e)) | |
231 else: | |
232 return open(filepath, 'r').read() | |
233 | |
234 # Returns a dictionary of actual results from actual-results.json file. | |
235 # | |
236 # The dictionary returned has this format: | |
237 # { | |
238 # u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322], | |
239 # u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152], | |
240 # u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716] | |
241 # } | |
242 # | |
243 # If the JSON actual result summary file cannot be loaded, logs a warning | |
244 # message and returns None. | |
245 # If the JSON actual result summary file can be loaded, but we have | |
246 # trouble parsing it, raises an Exception. | |
247 # | |
248 # params: | |
249 # json_url: URL pointing to a JSON actual result summary file | |
250 # sections: a list of section names to include in the results, e.g. | |
251 # [gm_json.JSONKEY_ACTUALRESULTS_FAILED, | |
252 # gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ; | |
253 # if None, then include ALL sections. | |
254 def _GetActualResults(self, json_url, sections=None): | |
255 try: | |
256 json_contents = self._GetFileContents(json_url) | |
257 except _InternalException: | |
258 print >> sys.stderr, ( | |
259 'could not read json_url %s ; skipping this platform.' % | |
260 json_url) | |
261 return None | |
262 json_dict = gm_json.LoadFromString(json_contents) | |
263 results_to_return = {} | |
264 actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] | |
265 if not sections: | |
266 sections = actual_results.keys() | |
267 for section in sections: | |
268 section_results = actual_results[section] | |
269 if section_results: | |
270 results_to_return.update(section_results) | |
271 return results_to_return | |
272 | |
273 # Rebaseline all tests/types we specified in the constructor, | |
274 # within this builder's subdirectory in expectations/gm . | |
275 # | |
276 # params: | |
277 # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' | |
278 def RebaselineSubdir(self, builder): | |
279 # Read in the actual result summary, and extract all the tests whose | |
280 # results we need to update. | |
281 results_builder = str(builder) | |
282 if self._from_trybot: | |
283 results_builder = results_builder + TRYBOT_SUFFIX | |
284 actuals_url = '/'.join([self._actuals_base_url, results_builder, | |
285 self._actuals_filename]) | |
286 # Only update results for tests that are currently failing. | |
287 # We don't want to rewrite results for tests that are already succeeding, | |
288 # because we don't want to add annotation fields (such as | |
289 # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we | |
290 # are actually modifying. | |
291 sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED] | |
292 if self._add_new: | |
293 sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) | |
294 if self._add_ignored: | |
295 sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED) | |
296 results_to_update = self._GetActualResults(json_url=actuals_url, | |
297 sections=sections) | |
298 | |
299 # Read in current expectations. | |
300 expectations_input_filepath = os.path.join( | |
301 self._expectations_root, builder, self._expectations_input_filename) | |
302 expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) | |
303 expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) | |
304 if not expected_results: | |
305 expected_results = {} | |
306 expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results | |
307 | |
308 # Update the expectations in memory, skipping any tests/configs that | |
309 # the caller asked to exclude. | |
310 skipped_images = [] | |
311 if results_to_update: | |
312 for (image_name, image_results) in results_to_update.iteritems(): | |
313 if self._image_filename_re: | |
314 (test, config) = self._image_filename_re.match(image_name).groups() | |
315 if self._tests: | |
316 if test not in self._tests: | |
317 skipped_images.append(image_name) | |
318 continue | |
319 if self._configs: | |
320 if config not in self._configs: | |
321 skipped_images.append(image_name) | |
322 continue | |
323 if not expected_results.get(image_name): | |
324 expected_results[image_name] = {} | |
325 expected_results[image_name]\ | |
326 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\ | |
327 = [image_results] | |
328 if self._mark_unreviewed: | |
329 expected_results[image_name]\ | |
330 [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\ | |
331 = False | |
332 if self._mark_ignore_failure: | |
333 expected_results[image_name]\ | |
334 [gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE]\ | |
335 = True | |
336 if self._bugs: | |
337 expected_results[image_name]\ | |
338 [gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\ | |
339 = self._bugs | |
340 if self._notes: | |
341 expected_results[image_name]\ | |
342 [gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\ | |
343 = self._notes | |
344 | |
345 # Write out updated expectations. | |
346 expectations_output_filepath = os.path.join( | |
347 self._expectations_root, builder, self._expectations_output_filename) | |
348 gm_json.WriteToFile(expectations_dict, expectations_output_filepath) | |
349 | |
350 # Mark the JSON file as plaintext, so text-style diffs can be applied. | |
351 # Fixes https://code.google.com/p/skia/issues/detail?id=1442 | |
352 if self._using_svn: | |
353 self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', | |
354 'text/x-json', expectations_output_filepath]) | |
355 | |
356 # main... | |
357 | |
358 parser = argparse.ArgumentParser( | |
359 formatter_class=argparse.RawDescriptionHelpFormatter, | |
360 epilog='Here is the full set of builders we know about:' + | |
361 '\n '.join([''] + sorted(TEST_BUILDERS))) | |
362 parser.add_argument('--actuals-base-url', | |
363 help=('base URL from which to read files containing JSON ' | |
364 'summaries of actual GM results; defaults to ' | |
365 '%(default)s. To get a specific revision (useful for ' | |
366 'trybots) replace "svn" with "svn-history/r123". ' | |
367 'If SKIMAGE is True, defaults to ' + | |
368 gm_json.SKIMAGE_ACTUALS_BASE_URL), | |
369 default='http://skia-autogen.googlecode.com/svn/gm-actual') | |
370 parser.add_argument('--actuals-filename', | |
371 help=('filename (within builder-specific subdirectories ' | |
372 'of ACTUALS_BASE_URL) to read a summary of results ' | |
373 'from; defaults to %(default)s'), | |
374 default='actual-results.json') | |
375 parser.add_argument('--add-new', action='store_true', | |
376 help=('in addition to the standard behavior of ' | |
377 'updating expectations for failing tests, add ' | |
378 'expectations for tests which don\'t have ' | |
379 'expectations yet.')) | |
380 parser.add_argument('--add-ignored', action='store_true', | |
381 help=('in addition to the standard behavior of ' | |
382 'updating expectations for failing tests, add ' | |
383 'expectations for tests for which failures are ' | |
384 'currently ignored.')) | |
385 parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+', | |
386 help=('Skia bug numbers (under ' | |
387 'https://code.google.com/p/skia/issues/list ) which ' | |
388 'pertain to this set of rebaselines.')) | |
389 parser.add_argument('--builders', metavar='BUILDER', nargs='+', | |
390 help=('which platforms to rebaseline; ' | |
391 'if unspecified, rebaseline all known platforms ' | |
392 '(see below for a list)')) | |
393 # TODO(epoger): Add test that exercises --configs argument. | |
394 parser.add_argument('--configs', metavar='CONFIG', nargs='+', | |
395 help=('which configurations to rebaseline, e.g. ' | |
396 '"--configs 565 8888", as a filter over the full set ' | |
397 'of results in ACTUALS_FILENAME; if unspecified, ' | |
398 'rebaseline *all* configs that are available.')) | |
399 parser.add_argument('--deprecated', action='store_true', | |
400 help=('run the tool even though it has been deprecated; ' | |
401 'see http://tinyurl.com/SkiaRebaselineServer for ' | |
402 'the recommended/supported process')) | |
403 parser.add_argument('--expectations-filename', | |
404 help=('filename (under EXPECTATIONS_ROOT) to read ' | |
405 'current expectations from, and to write new ' | |
406 'expectations into (unless a separate ' | |
407 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' | |
408 'defaults to %(default)s'), | |
409 default='expected-results.json') | |
410 parser.add_argument('--expectations-filename-output', | |
411 help=('filename (under EXPECTATIONS_ROOT) to write ' | |
412 'updated expectations into; by default, overwrites ' | |
413 'the input file (EXPECTATIONS_FILENAME)'), | |
414 default='') | |
415 parser.add_argument('--expectations-root', | |
416 help=('root of expectations directory to update-- should ' | |
417 'contain one or more builder subdirectories. ' | |
418 'Defaults to %(default)s. If SKIMAGE is set, ' | |
419 ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT), | |
420 default=os.path.join('expectations', 'gm')) | |
421 parser.add_argument('--keep-going-on-failure', action='store_true', | |
422 help=('instead of halting at the first error encountered, ' | |
423 'keep going and rebaseline as many tests as ' | |
424 'possible, and then report the full set of errors ' | |
425 'at the end')) | |
426 parser.add_argument('--notes', | |
427 help=('free-form text notes to add to all updated ' | |
428 'expectations')) | |
429 # TODO(epoger): Add test that exercises --tests argument. | |
430 parser.add_argument('--tests', metavar='TEST', nargs='+', | |
431 help=('which tests to rebaseline, e.g. ' | |
432 '"--tests aaclip bigmatrix", as a filter over the ' | |
433 'full set of results in ACTUALS_FILENAME; if ' | |
434 'unspecified, rebaseline *all* tests that are ' | |
435 'available.')) | |
436 parser.add_argument('--unreviewed', action='store_true', | |
437 help=('mark all expectations modified by this run as ' | |
438 '"%s": False' % | |
439 gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED)) | |
440 parser.add_argument('--ignore-failure', action='store_true', | |
441 help=('mark all expectations modified by this run as ' | |
442 '"%s": True' % | |
443 gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)) | |
444 parser.add_argument('--from-trybot', action='store_true', | |
445 help=('pull the actual-results.json file from the ' | |
446 'corresponding trybot, rather than the main builder')) | |
447 parser.add_argument('--skimage', action='store_true', | |
448 help=('Rebaseline skimage results instead of gm. Defaults ' | |
449 'to False. If True, TESTS and CONFIGS are ignored, ' | |
450 'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set ' | |
451 'to alternate defaults, specific to skimage.')) | |
452 args = parser.parse_args() | |
453 if not args.deprecated: | |
454 raise Exception( | |
455 'This tool has been deprecated; see' | |
456 ' http://tinyurl.com/SkiaRebaselineServer for the recommended/supported' | |
457 ' process, or re-run with the --deprecated option to press on.') | |
458 exception_handler = ExceptionHandler( | |
459 keep_going_on_failure=args.keep_going_on_failure) | |
460 if args.builders: | |
461 builders = args.builders | |
462 missing_json_is_fatal = True | |
463 else: | |
464 builders = sorted(TEST_BUILDERS) | |
465 missing_json_is_fatal = False | |
466 if args.skimage: | |
467 # Use a different default if --skimage is specified. | |
468 if args.actuals_base_url == parser.get_default('actuals_base_url'): | |
469 args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL | |
470 if args.expectations_root == parser.get_default('expectations_root'): | |
471 args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT | |
472 for builder in builders: | |
473 if not builder in TEST_BUILDERS: | |
474 raise Exception(('unrecognized builder "%s"; ' + | |
475 'should be one of %s') % ( | |
476 builder, TEST_BUILDERS)) | |
477 | |
478 expectations_json_file = os.path.join(args.expectations_root, builder, | |
479 args.expectations_filename) | |
480 if os.path.isfile(expectations_json_file): | |
481 rebaseliner = JsonRebaseliner( | |
482 expectations_root=args.expectations_root, | |
483 expectations_input_filename=args.expectations_filename, | |
484 expectations_output_filename=(args.expectations_filename_output or | |
485 args.expectations_filename), | |
486 tests=args.tests, configs=args.configs, | |
487 actuals_base_url=args.actuals_base_url, | |
488 actuals_filename=args.actuals_filename, | |
489 exception_handler=exception_handler, | |
490 add_new=args.add_new, add_ignored=args.add_ignored, | |
491 bugs=args.bugs, notes=args.notes, | |
492 mark_unreviewed=args.unreviewed, | |
493 mark_ignore_failure=args.ignore_failure, | |
494 from_trybot=args.from_trybot) | |
495 try: | |
496 rebaseliner.RebaselineSubdir(builder=builder) | |
497 except: | |
498 exception_handler.RaiseExceptionOrContinue() | |
499 else: | |
500 try: | |
501 raise _InternalException('expectations_json_file %s not found' % | |
502 expectations_json_file) | |
503 except: | |
504 exception_handler.RaiseExceptionOrContinue() | |
505 | |
506 exception_handler.ReportAllFailures() | |
OLD | NEW |