OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 ''' | 3 ''' |
4 Copyright 2012 Google Inc. | 4 Copyright 2012 Google Inc. |
5 | 5 |
6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
7 found in the LICENSE file. | 7 found in the LICENSE file. |
8 ''' | 8 ''' |
9 | 9 |
10 ''' | 10 ''' |
(...skipping 154 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
165 # add_new: if True, add expectations for tests which don't have any yet | 165 # add_new: if True, add expectations for tests which don't have any yet |
166 # bugs: optional list of bug numbers which pertain to these expectations | 166 # bugs: optional list of bug numbers which pertain to these expectations |
167 # notes: free-form text notes to add to all updated expectations | 167 # notes: free-form text notes to add to all updated expectations |
168 # mark_unreviewed: if True, mark these expectations as NOT having been | 168 # mark_unreviewed: if True, mark these expectations as NOT having been |
169 # reviewed by a human; otherwise, leave that field blank. | 169 # reviewed by a human; otherwise, leave that field blank. |
170 # Currently, there is no way to make this script mark | 170 # Currently, there is no way to make this script mark |
171 # expectations as reviewed-by-human=True. | 171 # expectations as reviewed-by-human=True. |
172 # TODO(epoger): Add that capability to a review tool. | 172 # TODO(epoger): Add that capability to a review tool. |
173 # mark_ignore_failure: if True, mark failures of a given test as being | 173 # mark_ignore_failure: if True, mark failures of a given test as being |
174 # ignored. | 174 # ignored. |
| 175 # from_trybot: if True, read actual-result JSON files generated from a |
| 176 # trybot run rather than a waterfall run. |
175 def __init__(self, expectations_root, expectations_input_filename, | 177 def __init__(self, expectations_root, expectations_input_filename, |
176 expectations_output_filename, actuals_base_url, | 178 expectations_output_filename, actuals_base_url, |
177 actuals_filename, exception_handler, | 179 actuals_filename, exception_handler, |
178 tests=None, configs=None, add_new=False, bugs=None, notes=None, | 180 tests=None, configs=None, add_new=False, bugs=None, notes=None, |
179 mark_unreviewed=None, mark_ignore_failure=False, | 181 mark_unreviewed=None, mark_ignore_failure=False, |
180 from_trybot=False): | 182 from_trybot=False): |
181 self._expectations_root = expectations_root | 183 self._expectations_root = expectations_root |
182 self._expectations_input_filename = expectations_input_filename | 184 self._expectations_input_filename = expectations_input_filename |
183 self._expectations_output_filename = expectations_output_filename | 185 self._expectations_output_filename = expectations_output_filename |
184 self._tests = tests | 186 self._tests = tests |
185 self._configs = configs | 187 self._configs = configs |
186 self._actuals_base_url = actuals_base_url | 188 self._actuals_base_url = actuals_base_url |
187 self._actuals_filename = actuals_filename | 189 self._actuals_filename = actuals_filename |
188 self._exception_handler = exception_handler | 190 self._exception_handler = exception_handler |
189 self._add_new = add_new | 191 self._add_new = add_new |
190 self._bugs = bugs | 192 self._bugs = bugs |
191 self._notes = notes | 193 self._notes = notes |
192 self._mark_unreviewed = mark_unreviewed | 194 self._mark_unreviewed = mark_unreviewed |
193 self._mark_ignore_failure = mark_ignore_failure; | 195 self._mark_ignore_failure = mark_ignore_failure; |
194 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) | 196 if self._tests or self._configs: |
| 197 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
| 198 else: |
| 199 self._image_filename_re = None |
195 self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) | 200 self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) |
196 self._from_trybot = from_trybot | 201 self._from_trybot = from_trybot |
197 | 202 |
198 # Executes subprocess.call(cmd). | 203 # Executes subprocess.call(cmd). |
199 # Raises an Exception if the command fails. | 204 # Raises an Exception if the command fails. |
200 def _Call(self, cmd): | 205 def _Call(self, cmd): |
201 if subprocess.call(cmd) != 0: | 206 if subprocess.call(cmd) != 0: |
202 raise _InternalException('error running command: ' + ' '.join(cmd)) | 207 raise _InternalException('error running command: ' + ' '.join(cmd)) |
203 | 208 |
204 # Returns the full contents of filepath, as a single string. | 209 # Returns the full contents of filepath, as a single string. |
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
286 expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) | 291 expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) |
287 if not expected_results: | 292 if not expected_results: |
288 expected_results = {} | 293 expected_results = {} |
289 expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results | 294 expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results |
290 | 295 |
291 # Update the expectations in memory, skipping any tests/configs that | 296 # Update the expectations in memory, skipping any tests/configs that |
292 # the caller asked to exclude. | 297 # the caller asked to exclude. |
293 skipped_images = [] | 298 skipped_images = [] |
294 if results_to_update: | 299 if results_to_update: |
295 for (image_name, image_results) in results_to_update.iteritems(): | 300 for (image_name, image_results) in results_to_update.iteritems(): |
296 (test, config) = self._image_filename_re.match(image_name).groups() | 301 if self._image_filename_re: |
297 if self._tests: | 302 (test, config) = self._image_filename_re.match(image_name).groups() |
298 if test not in self._tests: | 303 if self._tests: |
299 skipped_images.append(image_name) | 304 if test not in self._tests: |
300 continue | 305 skipped_images.append(image_name) |
301 if self._configs: | 306 continue |
302 if config not in self._configs: | 307 if self._configs: |
303 skipped_images.append(image_name) | 308 if config not in self._configs: |
304 continue | 309 skipped_images.append(image_name) |
| 310 continue |
305 if not expected_results.get(image_name): | 311 if not expected_results.get(image_name): |
306 expected_results[image_name] = {} | 312 expected_results[image_name] = {} |
307 expected_results[image_name]\ | 313 expected_results[image_name]\ |
308 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\ | 314 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\ |
309 = [image_results] | 315 = [image_results] |
310 if self._mark_unreviewed: | 316 if self._mark_unreviewed: |
311 expected_results[image_name]\ | 317 expected_results[image_name]\ |
312 [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\ | 318 [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\ |
313 = False | 319 = False |
314 if self._mark_ignore_failure: | 320 if self._mark_ignore_failure: |
(...skipping 22 matching lines...) Expand all Loading... |
337 | 343 |
338 # main... | 344 # main... |
339 | 345 |
340 parser = argparse.ArgumentParser( | 346 parser = argparse.ArgumentParser( |
341 formatter_class=argparse.RawDescriptionHelpFormatter, | 347 formatter_class=argparse.RawDescriptionHelpFormatter, |
342 epilog='Here is the full set of builders we know about:' + | 348 epilog='Here is the full set of builders we know about:' + |
343 '\n '.join([''] + sorted(TEST_BUILDERS))) | 349 '\n '.join([''] + sorted(TEST_BUILDERS))) |
344 parser.add_argument('--actuals-base-url', | 350 parser.add_argument('--actuals-base-url', |
345 help=('base URL from which to read files containing JSON ' | 351 help=('base URL from which to read files containing JSON ' |
346 'summaries of actual GM results; defaults to ' | 352 'summaries of actual GM results; defaults to ' |
347 '%(default)s. To get a specific revision (useful for' | 353 '%(default)s. To get a specific revision (useful for ' |
348 'trybots) replace "svn" with "svn-history/r123".'), | 354 'trybots) replace "svn" with "svn-history/r123". ' |
| 355 'If SKIMAGE is True, defaults to ' + |
| 356 gm_json.SKIMAGE_ACTUALS_BASE_URL), |
349 default='http://skia-autogen.googlecode.com/svn/gm-actual') | 357 default='http://skia-autogen.googlecode.com/svn/gm-actual') |
350 parser.add_argument('--actuals-filename', | 358 parser.add_argument('--actuals-filename', |
351 help=('filename (within builder-specific subdirectories ' | 359 help=('filename (within builder-specific subdirectories ' |
352 'of ACTUALS_BASE_URL) to read a summary of results ' | 360 'of ACTUALS_BASE_URL) to read a summary of results ' |
353 'from; defaults to %(default)s'), | 361 'from; defaults to %(default)s'), |
354 default='actual-results.json') | 362 default='actual-results.json') |
355 parser.add_argument('--add-new', action='store_true', | 363 parser.add_argument('--add-new', action='store_true', |
356 help=('in addition to the standard behavior of ' | 364 help=('in addition to the standard behavior of ' |
357 'updating expectations for failing tests, add ' | 365 'updating expectations for failing tests, add ' |
358 'expectations for tests which don\'t have ' | 366 'expectations for tests which don\'t have ' |
359 'expectations yet.')) | 367 'expectations yet.')) |
360 parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+', | 368 parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+', |
361 help=('Skia bug numbers (under ' | 369 help=('Skia bug numbers (under ' |
362 'https://code.google.com/p/skia/issues/list ) which ' | 370 'https://code.google.com/p/skia/issues/list ) which ' |
363 'pertain to this set of rebaselines.')) | 371 'pertain to this set of rebaselines.')) |
364 parser.add_argument('--builders', metavar='BUILDER', nargs='+', | 372 parser.add_argument('--builders', metavar='BUILDER', nargs='+', |
365 help=('which platforms to rebaseline; ' | 373 help=('which platforms to rebaseline; ' |
366 'if unspecified, rebaseline all known platforms ' | 374 'if unspecified, rebaseline all known platforms ' |
367 '(see below for a list)')) | 375 '(see below for a list)')) |
368 # TODO(epoger): Add test that exercises --configs argument. | 376 # TODO(epoger): Add test that exercises --configs argument. |
369 parser.add_argument('--configs', metavar='CONFIG', nargs='+', | 377 parser.add_argument('--configs', metavar='CONFIG', nargs='+', |
370 help=('which configurations to rebaseline, e.g. ' | 378 help=('which configurations to rebaseline, e.g. ' |
371 '"--configs 565 8888", as a filter over the full set ' | 379 '"--configs 565 8888", as a filter over the full set ' |
372 'of results in ACTUALS_FILENAME; if unspecified, ' | 380 'of results in ACTUALS_FILENAME; if unspecified, ' |
373 'rebaseline *all* configs that are available.')) | 381 'rebaseline *all* configs that are available. ' |
| 382 'Ignored if SKIMAGE is True.')) |
374 parser.add_argument('--expectations-filename', | 383 parser.add_argument('--expectations-filename', |
375 help=('filename (under EXPECTATIONS_ROOT) to read ' | 384 help=('filename (under EXPECTATIONS_ROOT) to read ' |
376 'current expectations from, and to write new ' | 385 'current expectations from, and to write new ' |
377 'expectations into (unless a separate ' | 386 'expectations into (unless a separate ' |
378 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' | 387 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' |
379 'defaults to %(default)s'), | 388 'defaults to %(default)s'), |
380 default='expected-results.json') | 389 default='expected-results.json') |
381 parser.add_argument('--expectations-filename-output', | 390 parser.add_argument('--expectations-filename-output', |
382 help=('filename (under EXPECTATIONS_ROOT) to write ' | 391 help=('filename (under EXPECTATIONS_ROOT) to write ' |
383 'updated expectations into; by default, overwrites ' | 392 'updated expectations into; by default, overwrites ' |
384 'the input file (EXPECTATIONS_FILENAME)'), | 393 'the input file (EXPECTATIONS_FILENAME)'), |
385 default='') | 394 default='') |
386 parser.add_argument('--expectations-root', | 395 parser.add_argument('--expectations-root', |
387 help=('root of expectations directory to update-- should ' | 396 help=('root of expectations directory to update-- should ' |
388 'contain one or more builder subdirectories. ' | 397 'contain one or more builder subdirectories. ' |
389 'Defaults to %(default)s'), | 398 'Defaults to %(default)s. If SKIMAGE is set, ' |
| 399 ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT), |
390 default=os.path.join('expectations', 'gm')) | 400 default=os.path.join('expectations', 'gm')) |
391 parser.add_argument('--keep-going-on-failure', action='store_true', | 401 parser.add_argument('--keep-going-on-failure', action='store_true', |
392 help=('instead of halting at the first error encountered, ' | 402 help=('instead of halting at the first error encountered, ' |
393 'keep going and rebaseline as many tests as ' | 403 'keep going and rebaseline as many tests as ' |
394 'possible, and then report the full set of errors ' | 404 'possible, and then report the full set of errors ' |
395 'at the end')) | 405 'at the end')) |
396 parser.add_argument('--notes', | 406 parser.add_argument('--notes', |
397 help=('free-form text notes to add to all updated ' | 407 help=('free-form text notes to add to all updated ' |
398 'expectations')) | 408 'expectations')) |
399 # TODO(epoger): Add test that exercises --tests argument. | 409 # TODO(epoger): Add test that exercises --tests argument. |
400 parser.add_argument('--tests', metavar='TEST', nargs='+', | 410 parser.add_argument('--tests', metavar='TEST', nargs='+', |
401 help=('which tests to rebaseline, e.g. ' | 411 help=('which tests to rebaseline, e.g. ' |
402 '"--tests aaclip bigmatrix", as a filter over the ' | 412 '"--tests aaclip bigmatrix", as a filter over the ' |
403 'full set of results in ACTUALS_FILENAME; if ' | 413 'full set of results in ACTUALS_FILENAME; if ' |
404 'unspecified, rebaseline *all* tests that are ' | 414 'unspecified, rebaseline *all* tests that are ' |
405 'available.')) | 415 'available. Ignored if SKIMAGE is True.')) |
406 parser.add_argument('--unreviewed', action='store_true', | 416 parser.add_argument('--unreviewed', action='store_true', |
407 help=('mark all expectations modified by this run as ' | 417 help=('mark all expectations modified by this run as ' |
408 '"%s": False' % | 418 '"%s": False' % |
409 gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED)) | 419 gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED)) |
410 parser.add_argument('--ignore-failure', action='store_true', | 420 parser.add_argument('--ignore-failure', action='store_true', |
411 help=('mark all expectations modified by this run as ' | 421 help=('mark all expectations modified by this run as ' |
412 '"%s": True' % | 422 '"%s": True' % |
413 gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)) | 423 gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)) |
414 parser.add_argument('--from-trybot', action='store_true', | 424 parser.add_argument('--from-trybot', action='store_true', |
415 help=('pull the actual-results.json file from the ' | 425 help=('pull the actual-results.json file from the ' |
416 'corresponding trybot, rather than the main builder')) | 426 'corresponding trybot, rather than the main builder')) |
| 427 parser.add_argument('--skimage', action='store_true', |
| 428 help=('Rebaseline skimage results instead of gm. Defaults ' |
| 429 'to False. If True, TESTS and CONFIGS are ignored, ' |
| 430 'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set ' |
| 431 'to alternate defaults, specific to skimage.')) |
417 args = parser.parse_args() | 432 args = parser.parse_args() |
418 exception_handler = ExceptionHandler( | 433 exception_handler = ExceptionHandler( |
419 keep_going_on_failure=args.keep_going_on_failure) | 434 keep_going_on_failure=args.keep_going_on_failure) |
420 if args.builders: | 435 if args.builders: |
421 builders = args.builders | 436 builders = args.builders |
422 missing_json_is_fatal = True | 437 missing_json_is_fatal = True |
423 else: | 438 else: |
424 builders = sorted(TEST_BUILDERS) | 439 builders = sorted(TEST_BUILDERS) |
425 missing_json_is_fatal = False | 440 missing_json_is_fatal = False |
| 441 if args.skimage: |
| 442 # Use a different default if --skimage is specified. |
| 443 if args.actuals_base_url == parser.get_default('actuals_base_url'): |
| 444 args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL |
| 445 if args.expectations_root == parser.get_default('expectations_root'): |
| 446 args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT |
| 447 # Also ignore TESTS and CONFIGS |
| 448 args.tests = None |
| 449 args.configs = None |
426 for builder in builders: | 450 for builder in builders: |
427 if not builder in TEST_BUILDERS: | 451 if not builder in TEST_BUILDERS: |
428 raise Exception(('unrecognized builder "%s"; ' + | 452 raise Exception(('unrecognized builder "%s"; ' + |
429 'should be one of %s') % ( | 453 'should be one of %s') % ( |
430 builder, TEST_BUILDERS)) | 454 builder, TEST_BUILDERS)) |
431 | 455 |
432 expectations_json_file = os.path.join(args.expectations_root, builder, | 456 expectations_json_file = os.path.join(args.expectations_root, builder, |
433 args.expectations_filename) | 457 args.expectations_filename) |
434 if os.path.isfile(expectations_json_file): | 458 if os.path.isfile(expectations_json_file): |
435 rebaseliner = JsonRebaseliner( | 459 rebaseliner = JsonRebaseliner( |
(...skipping 14 matching lines...) Expand all Loading... |
450 except: | 474 except: |
451 exception_handler.RaiseExceptionOrContinue() | 475 exception_handler.RaiseExceptionOrContinue() |
452 else: | 476 else: |
453 try: | 477 try: |
454 raise _InternalException('expectations_json_file %s not found' % | 478 raise _InternalException('expectations_json_file %s not found' % |
455 expectations_json_file) | 479 expectations_json_file) |
456 except: | 480 except: |
457 exception_handler.RaiseExceptionOrContinue() | 481 exception_handler.RaiseExceptionOrContinue() |
458 | 482 |
459 exception_handler.ReportAllFailures() | 483 exception_handler.ReportAllFailures() |
OLD | NEW |