rebaseline.py: add --keep-going-on-failure option, off by default

tools/rebaseline.py

 #!/usr/bin/python
 
 '''
 Copyright 2012 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 '''
 
 '''
@@ skipping 56 matching lines @@
    'base-android-nexus-10':
     'Test-Android-Nexus10-MaliT604-Arm7-Release',
    'base-android-nexus-4':
     'Test-Android-Nexus4-Adreno320-Arm7-Release',
 }
 
 
 class _InternalException(Exception):
     pass
 
+# Object that handles exceptions, either raising them immediately or collecting
+# them to display later on.
+class ExceptionHandler(object):
+
+    # params:
+    #  keep_going_on_failure: if False, report failures and quit right away;
+    #                         if True, collect failures until
+    #                         ReportAllFailures() is called
+    def __init__(self, keep_going_on_failure=False):
+        self._keep_going_on_failure = keep_going_on_failure
+        self._failures_encountered = []
+        self._exiting = False
+
+    # Exit the program with the given status value.
+    def _Exit(self, status=1):
+        self._exiting = True
+        sys.exit(status)
+
+    # We have encountered an exception; either collect the info and keep going,
+    # or exit the program right away.
+    def RaiseExceptionOrContinue(self, e):
+        # If we are already quitting the program, propagate any exceptions
+        # so that the proper exit status will be communicated to the shell.
+        if self._exiting:
+            raise e
+
+        if self._keep_going_on_failure:
+            print >> sys.stderr, 'WARNING: swallowing exception %s' % e
+            self._failures_encountered.append(e)
+        else:
+            print >> sys.stderr, e
+            print >> sys.stderr, (
+                'Halting at first exception; to keep going, re-run ' +
+                'with the --keep-going-on-failure option set.')
+            self._Exit()
+
+    def ReportAllFailures(self):
+        if self._failures_encountered:
+            print >> sys.stderr, ('Encountered %d failures (see above).' %
+                                  len(self._failures_encountered))
+            self._Exit()
+
+
 # Object that rebaselines a JSON expectations file (not individual image files).
 class JsonRebaseliner(object):
 
     # params:
     #  expectations_root: root directory of all expectations JSON files
     #  expectations_filename: filename (under expectations_root) of JSON
     #                         expectations file; typically
     #                         "expected-results.json"
     #  actuals_base_url: base URL from which to read actual-result JSON files
     #  actuals_filename: filename (under actuals_base_url) from which to read a
     #                    summary of results; typically "actual-results.json"
+    #  exception_handler: reference to rebaseline.ExceptionHandler object
     #  tests: list of tests to rebaseline, or None if we should rebaseline
     #         whatever files the JSON results summary file tells us to
     #  configs: which configs to run for each test, or None if we should
     #           rebaseline whatever configs the JSON results summary file tells
     #           us to
     #  add_new: if True, add expectations for tests which don't have any yet
     def __init__(self, expectations_root, expectations_filename,
-                 actuals_base_url, actuals_filename,
+                 actuals_base_url, actuals_filename, exception_handler,
                  tests=None, configs=None, add_new=False):
         self._expectations_root = expectations_root
         self._expectations_filename = expectations_filename
         self._tests = tests
         self._configs = configs
         self._actuals_base_url = actuals_base_url
         self._actuals_filename = actuals_filename
+        self._exception_handler = exception_handler
         self._add_new = add_new
         self._testname_pattern = re.compile('(\S+)_(\S+).png')
 
     # Returns the full contents of filepath, as a single string.
     # If filepath looks like a URL, try to read it that way instead of as
     # a path on local storage.
     #
     # Raises _InternalException if there is a problem.
     def _GetFileContents(self, filepath):
         if filepath.startswith('http:') or filepath.startswith('https:'):
@@ skipping 123 matching lines @@
 parser.add_argument('--expectations-filename',
                     help='filename (under EXPECTATIONS_ROOT) to read ' +
                     'current expectations from, and to write new ' +
                     'expectations into; defaults to %(default)s',
                     default='expected-results.json')
 parser.add_argument('--expectations-root',
                     help='root of expectations directory to update-- should ' +
                     'contain one or more base-* subdirectories. Defaults to ' +
                     '%(default)s',
                     default='.')
+parser.add_argument('--keep-going-on-failure', action='store_true',
+                    help='instead of halting at the first error encountered, ' +
+                    'keep going and rebaseline as many tests as possible, ' +
+                    'and then report the full set of errors at the end')
 parser.add_argument('--subdirs', metavar='SUBDIR', nargs='+',
                     help='which platform subdirectories to rebaseline; ' +
                     'if unspecified, rebaseline all subdirs, same as ' +
                     '"--subdirs %s"' % ' '.join(sorted(SUBDIR_MAPPING.keys())))
 # TODO(epoger): Add test that exercises --tests argument.
 parser.add_argument('--tests', metavar='TEST', nargs='+',
                     help='which tests to rebaseline, e.g. ' +
                     '"--tests aaclip bigmatrix", as a filter over the full ' +
                     'set of results in ACTUALS_FILENAME; if unspecified, ' +
                     'rebaseline *all* tests that are available.')
 args = parser.parse_args()
+exception_handler = ExceptionHandler(
+    keep_going_on_failure=args.keep_going_on_failure)
 if args.subdirs:
     subdirs = args.subdirs
     missing_json_is_fatal = True
 else:
     subdirs = sorted(SUBDIR_MAPPING.keys())
     missing_json_is_fatal = False
 for subdir in subdirs:
     if not subdir in SUBDIR_MAPPING.keys():
         raise Exception(('unrecognized platform subdir "%s"; ' +
                          'should be one of %s') % (
                              subdir, SUBDIR_MAPPING.keys()))
     builder = SUBDIR_MAPPING[subdir]
 
     # We instantiate different Rebaseliner objects depending
     # on whether we are rebaselining an expected-results.json file, or
     # individual image files.  Different gm-expected subdirectories may move
     # from individual image files to JSON-format expectations at different
     # times, so we need to make this determination per subdirectory.
     #
     # See https://goto.google.com/ChecksumTransitionDetail
     expectations_json_file = os.path.join(args.expectations_root, subdir,
                                           args.expectations_filename)
     if os.path.isfile(expectations_json_file):
         rebaseliner = JsonRebaseliner(
             expectations_root=args.expectations_root,
             expectations_filename=args.expectations_filename,
             tests=args.tests, configs=args.configs,
             actuals_base_url=args.actuals_base_url,
             actuals_filename=args.actuals_filename,
+            exception_handler=exception_handler,
             add_new=args.add_new)
     else:
         # TODO(epoger): When we get rid of the ImageRebaseliner implementation,
         # we should raise an Exception in this case (no JSON expectations file
         # found to update), to prevent a recurrence of
         # https://code.google.com/p/skia/issues/detail?id=1403 ('rebaseline.py
         # script fails with misleading output when run outside of gm-expected
         # dir')
         rebaseliner = rebaseline_imagefiles.ImageRebaseliner(
             expectations_root=args.expectations_root,
             tests=args.tests, configs=args.configs,
             dry_run=args.dry_run,
             json_base_url=args.actuals_base_url,
             json_filename=args.actuals_filename,
+            exception_handler=exception_handler,
             add_new=args.add_new,
             missing_json_is_fatal=missing_json_is_fatal)
+
     try:
         rebaseliner.RebaselineSubdir(subdir=subdir, builder=builder)
     except BaseException as e:
-        print >> sys.stderr, e
-        sys.exit(1)
+        exception_handler.RaiseExceptionOrContinue(e)
+
+exception_handler.ReportAllFailures()