| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/python | |
| 2 | |
| 3 ''' | |
| 4 Copyright 2012 Google Inc. | |
| 5 | |
| 6 Use of this source code is governed by a BSD-style license that can be | |
| 7 found in the LICENSE file. | |
| 8 ''' | |
| 9 | |
| 10 ''' | |
| 11 Rebaselines the given GM tests, on all bots and all configurations. | |
| 12 ''' | |
| 13 | |
| 14 # System-level imports | |
| 15 import argparse | |
| 16 import json | |
| 17 import os | |
| 18 import re | |
| 19 import subprocess | |
| 20 import sys | |
| 21 import urllib2 | |
| 22 | |
| 23 # Imports from within Skia | |
| 24 # | |
| 25 # We need to add the 'gm' directory, so that we can import gm_json.py within | |
| 26 # that directory. That script allows us to parse the actual-results.json file | |
| 27 # written out by the GM tool. | |
| 28 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* | |
| 29 # so any dirs that are already in the PYTHONPATH will be preferred. | |
| 30 # | |
| 31 # This assumes that the 'gm' directory has been checked out as a sibling of | |
| 32 # the 'tools' directory containing this script, which will be the case if | |
| 33 # 'trunk' was checked out as a single unit. | |
| 34 GM_DIRECTORY = os.path.realpath( | |
| 35 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) | |
| 36 if GM_DIRECTORY not in sys.path: | |
| 37 sys.path.append(GM_DIRECTORY) | |
| 38 import gm_json | |
| 39 | |
| 40 # TODO(epoger): In the long run, we want to build this list automatically, | |
| 41 # but for now we hard-code it until we can properly address | |
| 42 # https://code.google.com/p/skia/issues/detail?id=1544 | |
| 43 # ('live query of builder list makes rebaseline.py slow to start up') | |
| 44 TEST_BUILDERS = [ | |
| 45 'Test-Android-GalaxyNexus-SGX540-Arm7-Debug', | |
| 46 'Test-Android-GalaxyNexus-SGX540-Arm7-Release', | |
| 47 'Test-Android-IntelRhb-SGX544-x86-Debug', | |
| 48 'Test-Android-IntelRhb-SGX544-x86-Release', | |
| 49 'Test-Android-Nexus10-MaliT604-Arm7-Debug', | |
| 50 'Test-Android-Nexus10-MaliT604-Arm7-Release', | |
| 51 'Test-Android-Nexus4-Adreno320-Arm7-Debug', | |
| 52 'Test-Android-Nexus4-Adreno320-Arm7-Release', | |
| 53 'Test-Android-Nexus7-Tegra3-Arm7-Debug', | |
| 54 'Test-Android-Nexus7-Tegra3-Arm7-Release', | |
| 55 'Test-Android-NexusS-SGX540-Arm7-Debug', | |
| 56 'Test-Android-NexusS-SGX540-Arm7-Release', | |
| 57 'Test-Android-Xoom-Tegra2-Arm7-Debug', | |
| 58 'Test-Android-Xoom-Tegra2-Arm7-Release', | |
| 59 'Test-ChromeOS-Alex-GMA3150-x86-Debug', | |
| 60 'Test-ChromeOS-Alex-GMA3150-x86-Release', | |
| 61 'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug', | |
| 62 'Test-ChromeOS-Daisy-MaliT604-Arm7-Release', | |
| 63 'Test-ChromeOS-Link-HD4000-x86_64-Debug', | |
| 64 'Test-ChromeOS-Link-HD4000-x86_64-Release', | |
| 65 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', | |
| 66 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release', | |
| 67 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug', | |
| 68 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release', | |
| 69 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug', | |
| 70 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release', | |
| 71 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug', | |
| 72 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release', | |
| 73 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug', | |
| 74 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release', | |
| 75 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug', | |
| 76 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release', | |
| 77 'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind', | |
| 78 'Test-Ubuntu12-ShuttleA-GTX660-x86-Debug', | |
| 79 'Test-Ubuntu12-ShuttleA-GTX660-x86-Release', | |
| 80 'Test-Ubuntu12-ShuttleA-GTX660-x86_64-Debug', | |
| 81 'Test-Ubuntu12-ShuttleA-GTX660-x86_64-Release', | |
| 82 'Test-Ubuntu13.10-ShuttleA-NoGPU-x86_64-Debug', | |
| 83 'Test-Win7-ShuttleA-HD2000-x86-Debug', | |
| 84 'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE', | |
| 85 'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite', | |
| 86 'Test-Win7-ShuttleA-HD2000-x86-Release', | |
| 87 'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE', | |
| 88 'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite', | |
| 89 'Test-Win7-ShuttleA-HD2000-x86_64-Debug', | |
| 90 'Test-Win7-ShuttleA-HD2000-x86_64-Release', | |
| 91 'Test-Win8-ShuttleA-GTX660-x86-Debug', | |
| 92 'Test-Win8-ShuttleA-GTX660-x86-Release', | |
| 93 'Test-Win8-ShuttleA-GTX660-x86_64-Debug', | |
| 94 'Test-Win8-ShuttleA-GTX660-x86_64-Release', | |
| 95 'Test-Win8-ShuttleA-HD7770-x86-Debug', | |
| 96 'Test-Win8-ShuttleA-HD7770-x86-Release', | |
| 97 'Test-Win8-ShuttleA-HD7770-x86_64-Debug', | |
| 98 'Test-Win8-ShuttleA-HD7770-x86_64-Release', | |
| 99 ] | |
| 100 | |
| 101 # TODO: Get this from builder_name_schema in buildbot. | |
| 102 TRYBOT_SUFFIX = '-Trybot' | |
| 103 | |
| 104 | |
| 105 class _InternalException(Exception): | |
| 106 pass | |
| 107 | |
| 108 class ExceptionHandler(object): | |
| 109 """ Object that handles exceptions, either raising them immediately or | |
| 110 collecting them to display later on.""" | |
| 111 | |
| 112 # params: | |
| 113 def __init__(self, keep_going_on_failure=False): | |
| 114 """ | |
| 115 params: | |
| 116 keep_going_on_failure: if False, report failures and quit right away; | |
| 117 if True, collect failures until | |
| 118 ReportAllFailures() is called | |
| 119 """ | |
| 120 self._keep_going_on_failure = keep_going_on_failure | |
| 121 self._failures_encountered = [] | |
| 122 | |
| 123 def RaiseExceptionOrContinue(self): | |
| 124 """ We have encountered an exception; either collect the info and keep | |
| 125 going, or exit the program right away.""" | |
| 126 # Get traceback information about the most recently raised exception. | |
| 127 exc_info = sys.exc_info() | |
| 128 | |
| 129 if self._keep_going_on_failure: | |
| 130 print >> sys.stderr, ('WARNING: swallowing exception %s' % | |
| 131 repr(exc_info[1])) | |
| 132 self._failures_encountered.append(exc_info) | |
| 133 else: | |
| 134 print >> sys.stderr, ( | |
| 135 '\nHalting at first exception.\n' + | |
| 136 'Please file a bug to epoger@google.com at ' + | |
| 137 'https://code.google.com/p/skia/issues/entry, containing the ' + | |
| 138 'command you ran and the following stack trace.\n\n' + | |
| 139 'Afterwards, you can re-run with the --keep-going-on-failure ' + | |
| 140 'option set.\n') | |
| 141 raise exc_info[1], None, exc_info[2] | |
| 142 | |
| 143 def ReportAllFailures(self): | |
| 144 if self._failures_encountered: | |
| 145 print >> sys.stderr, ('Encountered %d failures (see above).' % | |
| 146 len(self._failures_encountered)) | |
| 147 sys.exit(1) | |
| 148 | |
| 149 | |
| 150 # Object that rebaselines a JSON expectations file (not individual image files). | |
| 151 class JsonRebaseliner(object): | |
| 152 | |
| 153 # params: | |
| 154 # expectations_root: root directory of all expectations JSON files | |
| 155 # expectations_input_filename: filename (under expectations_root) of JSON | |
| 156 # expectations file to read; typically | |
| 157 # "expected-results.json" | |
| 158 # expectations_output_filename: filename (under expectations_root) to | |
| 159 # which updated expectations should be | |
| 160 # written; typically the same as | |
| 161 # expectations_input_filename, to overwrite | |
| 162 # the old content | |
| 163 # actuals_base_url: base URL from which to read actual-result JSON files | |
| 164 # actuals_filename: filename (under actuals_base_url) from which to read a | |
| 165 # summary of results; typically "actual-results.json" | |
| 166 # exception_handler: reference to rebaseline.ExceptionHandler object | |
| 167 # tests: list of tests to rebaseline, or None if we should rebaseline | |
| 168 # whatever files the JSON results summary file tells us to | |
| 169 # configs: which configs to run for each test, or None if we should | |
| 170 # rebaseline whatever configs the JSON results summary file tells | |
| 171 # us to | |
| 172 # add_new: if True, add expectations for tests which don't have any yet | |
| 173 # add_ignored: if True, add expectations for tests for which failures are | |
| 174 # currently ignored | |
| 175 # bugs: optional list of bug numbers which pertain to these expectations | |
| 176 # notes: free-form text notes to add to all updated expectations | |
| 177 # mark_unreviewed: if True, mark these expectations as NOT having been | |
| 178 # reviewed by a human; otherwise, leave that field blank. | |
| 179 # Currently, there is no way to make this script mark | |
| 180 # expectations as reviewed-by-human=True. | |
| 181 # TODO(epoger): Add that capability to a review tool. | |
| 182 # mark_ignore_failure: if True, mark failures of a given test as being | |
| 183 # ignored. | |
| 184 # from_trybot: if True, read actual-result JSON files generated from a | |
| 185 # trybot run rather than a waterfall run. | |
| 186 def __init__(self, expectations_root, expectations_input_filename, | |
| 187 expectations_output_filename, actuals_base_url, | |
| 188 actuals_filename, exception_handler, | |
| 189 tests=None, configs=None, add_new=False, add_ignored=False, | |
| 190 bugs=None, notes=None, mark_unreviewed=None, | |
| 191 mark_ignore_failure=False, from_trybot=False): | |
| 192 self._expectations_root = expectations_root | |
| 193 self._expectations_input_filename = expectations_input_filename | |
| 194 self._expectations_output_filename = expectations_output_filename | |
| 195 self._tests = tests | |
| 196 self._configs = configs | |
| 197 self._actuals_base_url = actuals_base_url | |
| 198 self._actuals_filename = actuals_filename | |
| 199 self._exception_handler = exception_handler | |
| 200 self._add_new = add_new | |
| 201 self._add_ignored = add_ignored | |
| 202 self._bugs = bugs | |
| 203 self._notes = notes | |
| 204 self._mark_unreviewed = mark_unreviewed | |
| 205 self._mark_ignore_failure = mark_ignore_failure; | |
| 206 if self._tests or self._configs: | |
| 207 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) | |
| 208 else: | |
| 209 self._image_filename_re = None | |
| 210 self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) | |
| 211 self._from_trybot = from_trybot | |
| 212 | |
| 213 # Executes subprocess.call(cmd). | |
| 214 # Raises an Exception if the command fails. | |
| 215 def _Call(self, cmd): | |
| 216 if subprocess.call(cmd) != 0: | |
| 217 raise _InternalException('error running command: ' + ' '.join(cmd)) | |
| 218 | |
| 219 # Returns the full contents of filepath, as a single string. | |
| 220 # If filepath looks like a URL, try to read it that way instead of as | |
| 221 # a path on local storage. | |
| 222 # | |
| 223 # Raises _InternalException if there is a problem. | |
| 224 def _GetFileContents(self, filepath): | |
| 225 if filepath.startswith('http:') or filepath.startswith('https:'): | |
| 226 try: | |
| 227 return urllib2.urlopen(filepath).read() | |
| 228 except urllib2.HTTPError as e: | |
| 229 raise _InternalException('unable to read URL %s: %s' % ( | |
| 230 filepath, e)) | |
| 231 else: | |
| 232 return open(filepath, 'r').read() | |
| 233 | |
| 234 # Returns a dictionary of actual results from actual-results.json file. | |
| 235 # | |
| 236 # The dictionary returned has this format: | |
| 237 # { | |
| 238 # u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322], | |
| 239 # u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152], | |
| 240 # u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716] | |
| 241 # } | |
| 242 # | |
| 243 # If the JSON actual result summary file cannot be loaded, logs a warning | |
| 244 # message and returns None. | |
| 245 # If the JSON actual result summary file can be loaded, but we have | |
| 246 # trouble parsing it, raises an Exception. | |
| 247 # | |
| 248 # params: | |
| 249 # json_url: URL pointing to a JSON actual result summary file | |
| 250 # sections: a list of section names to include in the results, e.g. | |
| 251 # [gm_json.JSONKEY_ACTUALRESULTS_FAILED, | |
| 252 # gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ; | |
| 253 # if None, then include ALL sections. | |
| 254 def _GetActualResults(self, json_url, sections=None): | |
| 255 try: | |
| 256 json_contents = self._GetFileContents(json_url) | |
| 257 except _InternalException: | |
| 258 print >> sys.stderr, ( | |
| 259 'could not read json_url %s ; skipping this platform.' % | |
| 260 json_url) | |
| 261 return None | |
| 262 json_dict = gm_json.LoadFromString(json_contents) | |
| 263 results_to_return = {} | |
| 264 actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] | |
| 265 if not sections: | |
| 266 sections = actual_results.keys() | |
| 267 for section in sections: | |
| 268 section_results = actual_results[section] | |
| 269 if section_results: | |
| 270 results_to_return.update(section_results) | |
| 271 return results_to_return | |
| 272 | |
| 273 # Rebaseline all tests/types we specified in the constructor, | |
| 274 # within this builder's subdirectory in expectations/gm . | |
| 275 # | |
| 276 # params: | |
| 277 # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' | |
| 278 def RebaselineSubdir(self, builder): | |
| 279 # Read in the actual result summary, and extract all the tests whose | |
| 280 # results we need to update. | |
| 281 results_builder = str(builder) | |
| 282 if self._from_trybot: | |
| 283 results_builder = results_builder + TRYBOT_SUFFIX | |
| 284 actuals_url = '/'.join([self._actuals_base_url, results_builder, | |
| 285 self._actuals_filename]) | |
| 286 # Only update results for tests that are currently failing. | |
| 287 # We don't want to rewrite results for tests that are already succeeding, | |
| 288 # because we don't want to add annotation fields (such as | |
| 289 # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we | |
| 290 # are actually modifying. | |
| 291 sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED] | |
| 292 if self._add_new: | |
| 293 sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) | |
| 294 if self._add_ignored: | |
| 295 sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED) | |
| 296 results_to_update = self._GetActualResults(json_url=actuals_url, | |
| 297 sections=sections) | |
| 298 | |
| 299 # Read in current expectations. | |
| 300 expectations_input_filepath = os.path.join( | |
| 301 self._expectations_root, builder, self._expectations_input_filename) | |
| 302 expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) | |
| 303 expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) | |
| 304 if not expected_results: | |
| 305 expected_results = {} | |
| 306 expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results | |
| 307 | |
| 308 # Update the expectations in memory, skipping any tests/configs that | |
| 309 # the caller asked to exclude. | |
| 310 skipped_images = [] | |
| 311 if results_to_update: | |
| 312 for (image_name, image_results) in results_to_update.iteritems(): | |
| 313 if self._image_filename_re: | |
| 314 (test, config) = self._image_filename_re.match(image_name).groups() | |
| 315 if self._tests: | |
| 316 if test not in self._tests: | |
| 317 skipped_images.append(image_name) | |
| 318 continue | |
| 319 if self._configs: | |
| 320 if config not in self._configs: | |
| 321 skipped_images.append(image_name) | |
| 322 continue | |
| 323 if not expected_results.get(image_name): | |
| 324 expected_results[image_name] = {} | |
| 325 expected_results[image_name]\ | |
| 326 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\ | |
| 327 = [image_results] | |
| 328 if self._mark_unreviewed: | |
| 329 expected_results[image_name]\ | |
| 330 [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\ | |
| 331 = False | |
| 332 if self._mark_ignore_failure: | |
| 333 expected_results[image_name]\ | |
| 334 [gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE]\ | |
| 335 = True | |
| 336 if self._bugs: | |
| 337 expected_results[image_name]\ | |
| 338 [gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\ | |
| 339 = self._bugs | |
| 340 if self._notes: | |
| 341 expected_results[image_name]\ | |
| 342 [gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\ | |
| 343 = self._notes | |
| 344 | |
| 345 # Write out updated expectations. | |
| 346 expectations_output_filepath = os.path.join( | |
| 347 self._expectations_root, builder, self._expectations_output_filename) | |
| 348 gm_json.WriteToFile(expectations_dict, expectations_output_filepath) | |
| 349 | |
| 350 # Mark the JSON file as plaintext, so text-style diffs can be applied. | |
| 351 # Fixes https://code.google.com/p/skia/issues/detail?id=1442 | |
| 352 if self._using_svn: | |
| 353 self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', | |
| 354 'text/x-json', expectations_output_filepath]) | |
| 355 | |
| 356 # main... | |
| 357 | |
| 358 parser = argparse.ArgumentParser( | |
| 359 formatter_class=argparse.RawDescriptionHelpFormatter, | |
| 360 epilog='Here is the full set of builders we know about:' + | |
| 361 '\n '.join([''] + sorted(TEST_BUILDERS))) | |
| 362 parser.add_argument('--actuals-base-url', | |
| 363 help=('base URL from which to read files containing JSON ' | |
| 364 'summaries of actual GM results; defaults to ' | |
| 365 '%(default)s. To get a specific revision (useful for ' | |
| 366 'trybots) replace "svn" with "svn-history/r123". ' | |
| 367 'If SKIMAGE is True, defaults to ' + | |
| 368 gm_json.SKIMAGE_ACTUALS_BASE_URL), | |
| 369 default='http://skia-autogen.googlecode.com/svn/gm-actual') | |
| 370 parser.add_argument('--actuals-filename', | |
| 371 help=('filename (within builder-specific subdirectories ' | |
| 372 'of ACTUALS_BASE_URL) to read a summary of results ' | |
| 373 'from; defaults to %(default)s'), | |
| 374 default='actual-results.json') | |
| 375 parser.add_argument('--add-new', action='store_true', | |
| 376 help=('in addition to the standard behavior of ' | |
| 377 'updating expectations for failing tests, add ' | |
| 378 'expectations for tests which don\'t have ' | |
| 379 'expectations yet.')) | |
| 380 parser.add_argument('--add-ignored', action='store_true', | |
| 381 help=('in addition to the standard behavior of ' | |
| 382 'updating expectations for failing tests, add ' | |
| 383 'expectations for tests for which failures are ' | |
| 384 'currently ignored.')) | |
| 385 parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+', | |
| 386 help=('Skia bug numbers (under ' | |
| 387 'https://code.google.com/p/skia/issues/list ) which ' | |
| 388 'pertain to this set of rebaselines.')) | |
| 389 parser.add_argument('--builders', metavar='BUILDER', nargs='+', | |
| 390 help=('which platforms to rebaseline; ' | |
| 391 'if unspecified, rebaseline all known platforms ' | |
| 392 '(see below for a list)')) | |
| 393 # TODO(epoger): Add test that exercises --configs argument. | |
| 394 parser.add_argument('--configs', metavar='CONFIG', nargs='+', | |
| 395 help=('which configurations to rebaseline, e.g. ' | |
| 396 '"--configs 565 8888", as a filter over the full set ' | |
| 397 'of results in ACTUALS_FILENAME; if unspecified, ' | |
| 398 'rebaseline *all* configs that are available.')) | |
| 399 parser.add_argument('--deprecated', action='store_true', | |
| 400 help=('run the tool even though it has been deprecated; ' | |
| 401 'see http://tinyurl.com/SkiaRebaselineServer for ' | |
| 402 'the recommended/supported process')) | |
| 403 parser.add_argument('--expectations-filename', | |
| 404 help=('filename (under EXPECTATIONS_ROOT) to read ' | |
| 405 'current expectations from, and to write new ' | |
| 406 'expectations into (unless a separate ' | |
| 407 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' | |
| 408 'defaults to %(default)s'), | |
| 409 default='expected-results.json') | |
| 410 parser.add_argument('--expectations-filename-output', | |
| 411 help=('filename (under EXPECTATIONS_ROOT) to write ' | |
| 412 'updated expectations into; by default, overwrites ' | |
| 413 'the input file (EXPECTATIONS_FILENAME)'), | |
| 414 default='') | |
| 415 parser.add_argument('--expectations-root', | |
| 416 help=('root of expectations directory to update-- should ' | |
| 417 'contain one or more builder subdirectories. ' | |
| 418 'Defaults to %(default)s. If SKIMAGE is set, ' | |
| 419 ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT), | |
| 420 default=os.path.join('expectations', 'gm')) | |
| 421 parser.add_argument('--keep-going-on-failure', action='store_true', | |
| 422 help=('instead of halting at the first error encountered, ' | |
| 423 'keep going and rebaseline as many tests as ' | |
| 424 'possible, and then report the full set of errors ' | |
| 425 'at the end')) | |
| 426 parser.add_argument('--notes', | |
| 427 help=('free-form text notes to add to all updated ' | |
| 428 'expectations')) | |
| 429 # TODO(epoger): Add test that exercises --tests argument. | |
| 430 parser.add_argument('--tests', metavar='TEST', nargs='+', | |
| 431 help=('which tests to rebaseline, e.g. ' | |
| 432 '"--tests aaclip bigmatrix", as a filter over the ' | |
| 433 'full set of results in ACTUALS_FILENAME; if ' | |
| 434 'unspecified, rebaseline *all* tests that are ' | |
| 435 'available.')) | |
| 436 parser.add_argument('--unreviewed', action='store_true', | |
| 437 help=('mark all expectations modified by this run as ' | |
| 438 '"%s": False' % | |
| 439 gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED)) | |
| 440 parser.add_argument('--ignore-failure', action='store_true', | |
| 441 help=('mark all expectations modified by this run as ' | |
| 442 '"%s": True' % | |
| 443 gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)) | |
| 444 parser.add_argument('--from-trybot', action='store_true', | |
| 445 help=('pull the actual-results.json file from the ' | |
| 446 'corresponding trybot, rather than the main builder')) | |
| 447 parser.add_argument('--skimage', action='store_true', | |
| 448 help=('Rebaseline skimage results instead of gm. Defaults ' | |
| 449 'to False. If True, TESTS and CONFIGS are ignored, ' | |
| 450 'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set ' | |
| 451 'to alternate defaults, specific to skimage.')) | |
| 452 args = parser.parse_args() | |
| 453 if not args.deprecated: | |
| 454 raise Exception( | |
| 455 'This tool has been deprecated; see' | |
| 456 ' http://tinyurl.com/SkiaRebaselineServer for the recommended/supported' | |
| 457 ' process, or re-run with the --deprecated option to press on.') | |
| 458 exception_handler = ExceptionHandler( | |
| 459 keep_going_on_failure=args.keep_going_on_failure) | |
| 460 if args.builders: | |
| 461 builders = args.builders | |
| 462 missing_json_is_fatal = True | |
| 463 else: | |
| 464 builders = sorted(TEST_BUILDERS) | |
| 465 missing_json_is_fatal = False | |
| 466 if args.skimage: | |
| 467 # Use a different default if --skimage is specified. | |
| 468 if args.actuals_base_url == parser.get_default('actuals_base_url'): | |
| 469 args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL | |
| 470 if args.expectations_root == parser.get_default('expectations_root'): | |
| 471 args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT | |
| 472 for builder in builders: | |
| 473 if not builder in TEST_BUILDERS: | |
| 474 raise Exception(('unrecognized builder "%s"; ' + | |
| 475 'should be one of %s') % ( | |
| 476 builder, TEST_BUILDERS)) | |
| 477 | |
| 478 expectations_json_file = os.path.join(args.expectations_root, builder, | |
| 479 args.expectations_filename) | |
| 480 if os.path.isfile(expectations_json_file): | |
| 481 rebaseliner = JsonRebaseliner( | |
| 482 expectations_root=args.expectations_root, | |
| 483 expectations_input_filename=args.expectations_filename, | |
| 484 expectations_output_filename=(args.expectations_filename_output or | |
| 485 args.expectations_filename), | |
| 486 tests=args.tests, configs=args.configs, | |
| 487 actuals_base_url=args.actuals_base_url, | |
| 488 actuals_filename=args.actuals_filename, | |
| 489 exception_handler=exception_handler, | |
| 490 add_new=args.add_new, add_ignored=args.add_ignored, | |
| 491 bugs=args.bugs, notes=args.notes, | |
| 492 mark_unreviewed=args.unreviewed, | |
| 493 mark_ignore_failure=args.ignore_failure, | |
| 494 from_trybot=args.from_trybot) | |
| 495 try: | |
| 496 rebaseliner.RebaselineSubdir(builder=builder) | |
| 497 except: | |
| 498 exception_handler.RaiseExceptionOrContinue() | |
| 499 else: | |
| 500 try: | |
| 501 raise _InternalException('expectations_json_file %s not found' % | |
| 502 expectations_json_file) | |
| 503 except: | |
| 504 exception_handler.RaiseExceptionOrContinue() | |
| 505 | |
| 506 exception_handler.ReportAllFailures() | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 320443002:
remove GM result-grooming tools that use skia-autogen (Closed)
Base URL: https://skia.googlesource.com/skia.git@master