OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 ''' | 3 ''' |
4 Copyright 2012 Google Inc. | 4 Copyright 2012 Google Inc. |
5 | 5 |
6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
7 found in the LICENSE file. | 7 found in the LICENSE file. |
8 ''' | 8 ''' |
9 | 9 |
10 ''' | 10 ''' |
(...skipping 142 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
153 # actuals_base_url: base URL from which to read actual-result JSON files | 153 # actuals_base_url: base URL from which to read actual-result JSON files |
154 # actuals_filename: filename (under actuals_base_url) from which to read a | 154 # actuals_filename: filename (under actuals_base_url) from which to read a |
155 # summary of results; typically "actual-results.json" | 155 # summary of results; typically "actual-results.json" |
156 # exception_handler: reference to rebaseline.ExceptionHandler object | 156 # exception_handler: reference to rebaseline.ExceptionHandler object |
157 # tests: list of tests to rebaseline, or None if we should rebaseline | 157 # tests: list of tests to rebaseline, or None if we should rebaseline |
158 # whatever files the JSON results summary file tells us to | 158 # whatever files the JSON results summary file tells us to |
159 # configs: which configs to run for each test, or None if we should | 159 # configs: which configs to run for each test, or None if we should |
160 # rebaseline whatever configs the JSON results summary file tells | 160 # rebaseline whatever configs the JSON results summary file tells |
161 # us to | 161 # us to |
162 # add_new: if True, add expectations for tests which don't have any yet | 162 # add_new: if True, add expectations for tests which don't have any yet |
| 163 # bugs: optional list of bug numbers which pertain to these expectations |
| 164 # notes: free-form text notes to add to all updated expectations |
| 165 # mark_unreviewed: if True, mark these expectations as NOT having been |
| 166 # reviewed by a human; otherwise, leave that field blank. |
| 167 # Currently, there is no way to make this script mark |
| 168 # expectations as reviewed-by-human=True. |
| 169 # TODO(epoger): Add that capability to a review tool. |
163 def __init__(self, expectations_root, expectations_input_filename, | 170 def __init__(self, expectations_root, expectations_input_filename, |
164 expectations_output_filename, actuals_base_url, | 171 expectations_output_filename, actuals_base_url, |
165 actuals_filename, exception_handler, | 172 actuals_filename, exception_handler, |
166 tests=None, configs=None, add_new=False): | 173 tests=None, configs=None, add_new=False, bugs=None, notes=None, |
| 174 mark_unreviewed=None): |
167 self._expectations_root = expectations_root | 175 self._expectations_root = expectations_root |
168 self._expectations_input_filename = expectations_input_filename | 176 self._expectations_input_filename = expectations_input_filename |
169 self._expectations_output_filename = expectations_output_filename | 177 self._expectations_output_filename = expectations_output_filename |
170 self._tests = tests | 178 self._tests = tests |
171 self._configs = configs | 179 self._configs = configs |
172 self._actuals_base_url = actuals_base_url | 180 self._actuals_base_url = actuals_base_url |
173 self._actuals_filename = actuals_filename | 181 self._actuals_filename = actuals_filename |
174 self._exception_handler = exception_handler | 182 self._exception_handler = exception_handler |
175 self._add_new = add_new | 183 self._add_new = add_new |
| 184 self._bugs = bugs |
| 185 self._notes = notes |
| 186 self._mark_unreviewed = mark_unreviewed |
176 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) | 187 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) |
177 self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) | 188 self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) |
178 | 189 |
179 # Executes subprocess.call(cmd). | 190 # Executes subprocess.call(cmd). |
180 # Raises an Exception if the command fails. | 191 # Raises an Exception if the command fails. |
181 def _Call(self, cmd): | 192 def _Call(self, cmd): |
182 if subprocess.call(cmd) != 0: | 193 if subprocess.call(cmd) != 0: |
183 raise _InternalException('error running command: ' + ' '.join(cmd)) | 194 raise _InternalException('error running command: ' + ' '.join(cmd)) |
184 | 195 |
185 # Returns the full contents of filepath, as a single string. | 196 # Returns the full contents of filepath, as a single string. |
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
239 # Rebaseline all tests/types we specified in the constructor, | 250 # Rebaseline all tests/types we specified in the constructor, |
240 # within this builder's subdirectory in expectations/gm . | 251 # within this builder's subdirectory in expectations/gm . |
241 # | 252 # |
242 # params: | 253 # params: |
243 # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' | 254 # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' |
244 def RebaselineSubdir(self, builder): | 255 def RebaselineSubdir(self, builder): |
245 # Read in the actual result summary, and extract all the tests whose | 256 # Read in the actual result summary, and extract all the tests whose |
246 # results we need to update. | 257 # results we need to update. |
247 actuals_url = '/'.join([self._actuals_base_url, | 258 actuals_url = '/'.join([self._actuals_base_url, |
248 builder, self._actuals_filename]) | 259 builder, self._actuals_filename]) |
249 # In most cases, we won't need to re-record results that are already | 260 # Only update results for tests that are currently failing. |
250 # succeeding, but including the SUCCEEDED results will allow us to | 261 # We don't want to rewrite results for tests that are already succeeding, |
251 # re-record expectations if they somehow get out of sync. | 262 # because we don't want to add annotation fields (such as |
252 sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED, | 263 # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we |
253 gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED] | 264 # are actually modifying. |
| 265 sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED] |
254 if self._add_new: | 266 if self._add_new: |
255 sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) | 267 sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) |
256 results_to_update = self._GetActualResults(json_url=actuals_url, | 268 results_to_update = self._GetActualResults(json_url=actuals_url, |
257 sections=sections) | 269 sections=sections) |
258 | 270 |
259 # Read in current expectations. | 271 # Read in current expectations. |
260 expectations_input_filepath = os.path.join( | 272 expectations_input_filepath = os.path.join( |
261 self._expectations_root, builder, self._expectations_input_filename) | 273 self._expectations_root, builder, self._expectations_input_filename) |
262 expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) | 274 expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) |
263 expected_results = expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] | 275 expected_results = expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] |
264 | 276 |
265 # Update the expectations in memory, skipping any tests/configs that | 277 # Update the expectations in memory, skipping any tests/configs that |
266 # the caller asked to exclude. | 278 # the caller asked to exclude. |
267 skipped_images = [] | 279 skipped_images = [] |
268 if results_to_update: | 280 if results_to_update: |
269 for (image_name, image_results) in results_to_update.iteritems(): | 281 for (image_name, image_results) in results_to_update.iteritems(): |
270 (test, config) = self._image_filename_re.match(image_name).groups() | 282 (test, config) = self._image_filename_re.match(image_name).groups() |
271 if self._tests: | 283 if self._tests: |
272 if test not in self._tests: | 284 if test not in self._tests: |
273 skipped_images.append(image_name) | 285 skipped_images.append(image_name) |
274 continue | 286 continue |
275 if self._configs: | 287 if self._configs: |
276 if config not in self._configs: | 288 if config not in self._configs: |
277 skipped_images.append(image_name) | 289 skipped_images.append(image_name) |
278 continue | 290 continue |
279 if not expected_results.get(image_name): | 291 if not expected_results.get(image_name): |
280 expected_results[image_name] = {} | 292 expected_results[image_name] = {} |
281 expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGE
STS] = \ | 293 expected_results[image_name]\ |
282 [image_results] | 294 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\ |
| 295 = [image_results] |
| 296 if self._mark_unreviewed: |
| 297 expected_results[image_name]\ |
| 298 [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\ |
| 299 = False |
| 300 if self._bugs: |
| 301 expected_results[image_name]\ |
| 302 [gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\ |
| 303 = self._bugs |
| 304 if self._notes: |
| 305 expected_results[image_name]\ |
| 306 [gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\ |
| 307 = self._notes |
283 | 308 |
284 # Write out updated expectations. | 309 # Write out updated expectations. |
285 expectations_output_filepath = os.path.join( | 310 expectations_output_filepath = os.path.join( |
286 self._expectations_root, builder, self._expectations_output_filename) | 311 self._expectations_root, builder, self._expectations_output_filename) |
287 gm_json.WriteToFile(expectations_dict, expectations_output_filepath) | 312 gm_json.WriteToFile(expectations_dict, expectations_output_filepath) |
288 | 313 |
289 # Mark the JSON file as plaintext, so text-style diffs can be applied. | 314 # Mark the JSON file as plaintext, so text-style diffs can be applied. |
290 # Fixes https://code.google.com/p/skia/issues/detail?id=1442 | 315 # Fixes https://code.google.com/p/skia/issues/detail?id=1442 |
291 if self._using_svn: | 316 if self._using_svn: |
292 self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', | 317 self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', |
293 'text/x-json', expectations_output_filepath]) | 318 'text/x-json', expectations_output_filepath]) |
294 | 319 |
295 # main... | 320 # main... |
296 | 321 |
297 parser = argparse.ArgumentParser() | 322 parser = argparse.ArgumentParser() |
298 parser.add_argument('--actuals-base-url', | 323 parser.add_argument('--actuals-base-url', |
299 help='base URL from which to read files containing JSON ' + | 324 help=('base URL from which to read files containing JSON ' |
300 'summaries of actual GM results; defaults to %(default)s', | 325 'summaries of actual GM results; defaults to ' |
| 326 '%(default)s'), |
301 default='http://skia-autogen.googlecode.com/svn/gm-actual') | 327 default='http://skia-autogen.googlecode.com/svn/gm-actual') |
302 parser.add_argument('--actuals-filename', | 328 parser.add_argument('--actuals-filename', |
303 help='filename (within builder-specific subdirectories ' + | 329 help=('filename (within builder-specific subdirectories ' |
304 'of ACTUALS_BASE_URL) to read a summary of results from; ' + | 330 'of ACTUALS_BASE_URL) to read a summary of results ' |
305 'defaults to %(default)s', | 331 'from; defaults to %(default)s'), |
306 default='actual-results.json') | 332 default='actual-results.json') |
307 # TODO(epoger): Add test that exercises --add-new argument. | 333 # TODO(epoger): Add test that exercises --add-new argument. |
308 parser.add_argument('--add-new', action='store_true', | 334 parser.add_argument('--add-new', action='store_true', |
309 help='in addition to the standard behavior of ' + | 335 help=('in addition to the standard behavior of ' |
310 'updating expectations for failing tests, add ' + | 336 'updating expectations for failing tests, add ' |
311 'expectations for tests which don\'t have expectations ' + | 337 'expectations for tests which don\'t have ' |
312 'yet.') | 338 'expectations yet.')) |
| 339 parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+', |
| 340 help=('Skia bug numbers (under ' |
| 341 'https://code.google.com/p/skia/issues/list ) which ' |
| 342 'pertain to this set of rebaselines.')) |
313 parser.add_argument('--builders', metavar='BUILDER', nargs='+', | 343 parser.add_argument('--builders', metavar='BUILDER', nargs='+', |
314 help='which platforms to rebaseline; ' + | 344 help=('which platforms to rebaseline; ' |
315 'if unspecified, rebaseline all platforms, same as ' + | 345 'if unspecified, rebaseline all platforms, same as ' |
316 '"--builders %s"' % ' '.join(sorted(TEST_BUILDERS))) | 346 '"--builders %s"' % ' '.join(sorted(TEST_BUILDERS)))) |
317 # TODO(epoger): Add test that exercises --configs argument. | 347 # TODO(epoger): Add test that exercises --configs argument. |
318 parser.add_argument('--configs', metavar='CONFIG', nargs='+', | 348 parser.add_argument('--configs', metavar='CONFIG', nargs='+', |
319 help='which configurations to rebaseline, e.g. ' + | 349 help=('which configurations to rebaseline, e.g. ' |
320 '"--configs 565 8888", as a filter over the full set of ' + | 350 '"--configs 565 8888", as a filter over the full set ' |
321 'results in ACTUALS_FILENAME; if unspecified, rebaseline ' + | 351 'of results in ACTUALS_FILENAME; if unspecified, ' |
322 '*all* configs that are available.') | 352 'rebaseline *all* configs that are available.')) |
323 parser.add_argument('--expectations-filename', | 353 parser.add_argument('--expectations-filename', |
324 help='filename (under EXPECTATIONS_ROOT) to read ' + | 354 help=('filename (under EXPECTATIONS_ROOT) to read ' |
325 'current expectations from, and to write new ' + | 355 'current expectations from, and to write new ' |
326 'expectations into (unless a separate ' + | 356 'expectations into (unless a separate ' |
327 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' + | 357 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' |
328 'defaults to %(default)s', | 358 'defaults to %(default)s'), |
329 default='expected-results.json') | 359 default='expected-results.json') |
330 parser.add_argument('--expectations-filename-output', | 360 parser.add_argument('--expectations-filename-output', |
331 help='filename (under EXPECTATIONS_ROOT) to write ' + | 361 help=('filename (under EXPECTATIONS_ROOT) to write ' |
332 'updated expectations into; by default, overwrites the ' + | 362 'updated expectations into; by default, overwrites ' |
333 'input file (EXPECTATIONS_FILENAME)', | 363 'the input file (EXPECTATIONS_FILENAME)'), |
334 default='') | 364 default='') |
335 parser.add_argument('--expectations-root', | 365 parser.add_argument('--expectations-root', |
336 help='root of expectations directory to update-- should ' + | 366 help=('root of expectations directory to update-- should ' |
337 'contain one or more builder subdirectories. Defaults to ' + | 367 'contain one or more builder subdirectories. ' |
338 '%(default)s', | 368 'Defaults to %(default)s'), |
339 default=os.path.join('expectations', 'gm')) | 369 default=os.path.join('expectations', 'gm')) |
340 parser.add_argument('--keep-going-on-failure', action='store_true', | 370 parser.add_argument('--keep-going-on-failure', action='store_true', |
341 help='instead of halting at the first error encountered, ' + | 371 help=('instead of halting at the first error encountered, ' |
342 'keep going and rebaseline as many tests as possible, ' + | 372 'keep going and rebaseline as many tests as ' |
343 'and then report the full set of errors at the end') | 373 'possible, and then report the full set of errors ' |
| 374 'at the end')) |
| 375 parser.add_argument('--notes', |
| 376 help=('free-form text notes to add to all updated ' |
| 377 'expectations')) |
344 # TODO(epoger): Add test that exercises --tests argument. | 378 # TODO(epoger): Add test that exercises --tests argument. |
345 parser.add_argument('--tests', metavar='TEST', nargs='+', | 379 parser.add_argument('--tests', metavar='TEST', nargs='+', |
346 help='which tests to rebaseline, e.g. ' + | 380 help=('which tests to rebaseline, e.g. ' |
347 '"--tests aaclip bigmatrix", as a filter over the full ' + | 381 '"--tests aaclip bigmatrix", as a filter over the ' |
348 'set of results in ACTUALS_FILENAME; if unspecified, ' + | 382 'full set of results in ACTUALS_FILENAME; if ' |
349 'rebaseline *all* tests that are available.') | 383 'unspecified, rebaseline *all* tests that are ' |
| 384 'available.')) |
| 385 parser.add_argument('--unreviewed', action='store_true', |
| 386 help=('mark all expectations modified by this run as ' |
| 387 '"%s": False' % |
| 388 gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED)) |
350 args = parser.parse_args() | 389 args = parser.parse_args() |
351 exception_handler = ExceptionHandler( | 390 exception_handler = ExceptionHandler( |
352 keep_going_on_failure=args.keep_going_on_failure) | 391 keep_going_on_failure=args.keep_going_on_failure) |
353 if args.builders: | 392 if args.builders: |
354 builders = args.builders | 393 builders = args.builders |
355 missing_json_is_fatal = True | 394 missing_json_is_fatal = True |
356 else: | 395 else: |
357 builders = sorted(TEST_BUILDERS) | 396 builders = sorted(TEST_BUILDERS) |
358 missing_json_is_fatal = False | 397 missing_json_is_fatal = False |
359 for builder in builders: | 398 for builder in builders: |
360 if not builder in TEST_BUILDERS: | 399 if not builder in TEST_BUILDERS: |
361 raise Exception(('unrecognized builder "%s"; ' + | 400 raise Exception(('unrecognized builder "%s"; ' + |
362 'should be one of %s') % ( | 401 'should be one of %s') % ( |
363 builder, TEST_BUILDERS)) | 402 builder, TEST_BUILDERS)) |
364 | 403 |
365 expectations_json_file = os.path.join(args.expectations_root, builder, | 404 expectations_json_file = os.path.join(args.expectations_root, builder, |
366 args.expectations_filename) | 405 args.expectations_filename) |
367 if os.path.isfile(expectations_json_file): | 406 if os.path.isfile(expectations_json_file): |
368 rebaseliner = JsonRebaseliner( | 407 rebaseliner = JsonRebaseliner( |
369 expectations_root=args.expectations_root, | 408 expectations_root=args.expectations_root, |
370 expectations_input_filename=args.expectations_filename, | 409 expectations_input_filename=args.expectations_filename, |
371 expectations_output_filename=(args.expectations_filename_output or | 410 expectations_output_filename=(args.expectations_filename_output or |
372 args.expectations_filename), | 411 args.expectations_filename), |
373 tests=args.tests, configs=args.configs, | 412 tests=args.tests, configs=args.configs, |
374 actuals_base_url=args.actuals_base_url, | 413 actuals_base_url=args.actuals_base_url, |
375 actuals_filename=args.actuals_filename, | 414 actuals_filename=args.actuals_filename, |
376 exception_handler=exception_handler, | 415 exception_handler=exception_handler, |
377 add_new=args.add_new) | 416 add_new=args.add_new, bugs=args.bugs, notes=args.notes, |
| 417 mark_unreviewed=args.unreviewed) |
378 try: | 418 try: |
379 rebaseliner.RebaselineSubdir(builder=builder) | 419 rebaseliner.RebaselineSubdir(builder=builder) |
380 except BaseException as e: | 420 except BaseException as e: |
381 exception_handler.RaiseExceptionOrContinue(e) | 421 exception_handler.RaiseExceptionOrContinue(e) |
382 else: | 422 else: |
383 exception_handler.RaiseExceptionOrContinue(_InternalException( | 423 exception_handler.RaiseExceptionOrContinue(_InternalException( |
384 'expectations_json_file %s not found' % expectations_json_file)) | 424 'expectations_json_file %s not found' % expectations_json_file)) |
385 | 425 |
386 exception_handler.ReportAllFailures() | 426 exception_handler.ReportAllFailures() |
OLD | NEW |