Add ability to rebaseline skimage to rebaseline.py

tools/rebaseline.py

 #!/usr/bin/python
 
 '''
 Copyright 2012 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 '''
 
 '''
@@ skipping 141 matching lines @@
   #                                which updated expectations should be
   #                                written; typically the same as
   #                                expectations_input_filename, to overwrite
   #                                the old content
   #  actuals_base_url: base URL from which to read actual-result JSON files
   #  actuals_filename: filename (under actuals_base_url) from which to read a
   #                    summary of results; typically "actual-results.json"
   #  exception_handler: reference to rebaseline.ExceptionHandler object
   #  tests: list of tests to rebaseline, or None if we should rebaseline
   #         whatever files the JSON results summary file tells us to
+  #         Ignored if 'skip_pattern_matching' is set to True
   #  configs: which configs to run for each test, or None if we should
   #           rebaseline whatever configs the JSON results summary file tells
   #           us to
+  #           Ignored if 'skip_pattern_matching' is set to True
   #  add_new: if True, add expectations for tests which don't have any yet
   #  bugs: optional list of bug numbers which pertain to these expectations
   #  notes: free-form text notes to add to all updated expectations
   #  mark_unreviewed: if True, mark these expectations as NOT having been
   #                   reviewed by a human; otherwise, leave that field blank.
   #                   Currently, there is no way to make this script mark
   #                   expectations as reviewed-by-human=True.
   #                   TODO(epoger): Add that capability to a review tool.
   #  mark_ignore_failure: if True, mark failures of a given test as being
   #                       ignored.
+  #  skip_pattern_matching: if True, do not attempt to skip tests/configs
+  #                         matching input parameters 'tests'/'configs'.
   def __init__(self, expectations_root, expectations_input_filename,
                expectations_output_filename, actuals_base_url,
                actuals_filename, exception_handler,
                tests=None, configs=None, add_new=False, bugs=None, notes=None,
                mark_unreviewed=None, mark_ignore_failure=False,
-               from_trybot=False):
+               from_trybot=False, skip_pattern_matching=False):

[epoger, 2013/10/10 19:51:39]

Please add from_trybot to the docstring (I know you didn't add the param, but
while you're in there...)

[scroggo, 2013/10/10 20:18:58]

Done.

     self._expectations_root = expectations_root
     self._expectations_input_filename = expectations_input_filename
     self._expectations_output_filename = expectations_output_filename
     self._tests = tests
     self._configs = configs
     self._actuals_base_url = actuals_base_url
     self._actuals_filename = actuals_filename
     self._exception_handler = exception_handler
     self._add_new = add_new
     self._bugs = bugs
     self._notes = notes
     self._mark_unreviewed = mark_unreviewed
     self._mark_ignore_failure = mark_ignore_failure;
-    self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
+    # TODO(scroggo): This is a hack. Since the filenames will not match
+    # the pattern in skimage, we skip the pattern matching entirely.
+    if skip_pattern_matching:

[epoger, 2013/10/10 19:51:39]

Maybe, instead of adding the new parameter that can conflict with the two
existing ones...

if self._tests or self._configs:
  self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
else:
  self._image_filename_re = None

If you do this, you'll have to make sure _tests and _configs are None when you
construct the Rebaseliner object.  (You could do that within your "if
args.skimage" block)

[scroggo, 2013/10/10 20:18:58]

Done. I like this much better!

+      self._image_filename_re = None
+      if tests or configs:
+        raise _InternalException('"skip_pattern_matching" is incompatible with '
+                                 '"tests" and "configs"')
+    else:
+      self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
     self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn'))
     self._from_trybot = from_trybot
 
   # Executes subprocess.call(cmd).
   # Raises an Exception if the command fails.
   def _Call(self, cmd):
     if subprocess.call(cmd) != 0:
       raise _InternalException('error running command: ' + ' '.join(cmd))
 
   # Returns the full contents of filepath, as a single string.
@@ skipping 81 matching lines @@
     expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
     if not expected_results:
       expected_results = {}
       expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results
 
     # Update the expectations in memory, skipping any tests/configs that
     # the caller asked to exclude.
     skipped_images = []
     if results_to_update:
       for (image_name, image_results) in results_to_update.iteritems():
-        (test, config) = self._image_filename_re.match(image_name).groups()
-        if self._tests:
-          if test not in self._tests:
-            skipped_images.append(image_name)
-            continue
-        if self._configs:
-          if config not in self._configs:
-            skipped_images.append(image_name)
-            continue
+        if self._image_filename_re:
+          (test, config) = self._image_filename_re.match(image_name).groups()
+          if self._tests:
+            if test not in self._tests:
+              skipped_images.append(image_name)
+              continue
+          if self._configs:
+            if config not in self._configs:
+              skipped_images.append(image_name)
+              continue
         if not expected_results.get(image_name):
           expected_results[image_name] = {}
         expected_results[image_name]\
                         [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\
                         = [image_results]
         if self._mark_unreviewed:
           expected_results[image_name]\
                           [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\
                           = False
         if self._mark_ignore_failure:
@@ skipping 12 matching lines @@
     # Write out updated expectations.
     expectations_output_filepath = os.path.join(
         self._expectations_root, builder, self._expectations_output_filename)
     gm_json.WriteToFile(expectations_dict, expectations_output_filepath)
 
     # Mark the JSON file as plaintext, so text-style diffs can be applied.
     # Fixes https://code.google.com/p/skia/issues/detail?id=1442
     if self._using_svn:
       self._Call(['svn', 'propset', '--quiet', 'svn:mime-type',
                   'text/x-json', expectations_output_filepath])
-
 # main...
 
 parser = argparse.ArgumentParser(
     formatter_class=argparse.RawDescriptionHelpFormatter,
     epilog='Here is the full set of builders we know about:' +
            '\n    '.join([''] + sorted(TEST_BUILDERS)))
 parser.add_argument('--actuals-base-url',
                     help=('base URL from which to read files containing JSON '
                           'summaries of actual GM results; defaults to '
-                          '%(default)s. To get a specific revision (useful for'
-                          'trybots) replace "svn" with "svn-history/r123".'),
+                          '%(default)s. To get a specific revision (useful for '
+                          'trybots) replace "svn" with "svn-history/r123". '
+                          'If SKIMAGE is True, defaults to ' +
+                          gm_json.SKIMAGE_ACTUALS_BASE_URL),
                     default='http://skia-autogen.googlecode.com/svn/gm-actual')
 parser.add_argument('--actuals-filename',
                     help=('filename (within builder-specific subdirectories '
                           'of ACTUALS_BASE_URL) to read a summary of results '
                           'from; defaults to %(default)s'),
                     default='actual-results.json')
 parser.add_argument('--add-new', action='store_true',
                     help=('in addition to the standard behavior of '
                           'updating expectations for failing tests, add '
                           'expectations for tests which don\'t have '
                           'expectations yet.'))
 parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+',
                     help=('Skia bug numbers (under '
                           'https://code.google.com/p/skia/issues/list ) which '
                           'pertain to this set of rebaselines.'))
 parser.add_argument('--builders', metavar='BUILDER', nargs='+',
                     help=('which platforms to rebaseline; '
                           'if unspecified, rebaseline all known platforms '
                           '(see below for a list)'))
 # TODO(epoger): Add test that exercises --configs argument.
 parser.add_argument('--configs', metavar='CONFIG', nargs='+',
                     help=('which configurations to rebaseline, e.g. '
                           '"--configs 565 8888", as a filter over the full set '
                           'of results in ACTUALS_FILENAME; if unspecified, '
-                          'rebaseline *all* configs that are available.'))
+                          'rebaseline *all* configs that are available. '
+                          'Ignored if SKIMAGE is True.'))
 parser.add_argument('--expectations-filename',
                     help=('filename (under EXPECTATIONS_ROOT) to read '
                           'current expectations from, and to write new '
                           'expectations into (unless a separate '
                           'EXPECTATIONS_FILENAME_OUTPUT has been specified); '
                           'defaults to %(default)s'),
                     default='expected-results.json')
 parser.add_argument('--expectations-filename-output',
                     help=('filename (under EXPECTATIONS_ROOT) to write '
                           'updated expectations into; by default, overwrites '
                           'the input file (EXPECTATIONS_FILENAME)'),
                     default='')
 parser.add_argument('--expectations-root',
                     help=('root of expectations directory to update-- should '
                           'contain one or more builder subdirectories. '
-                          'Defaults to %(default)s'),
+                          'Defaults to %(default)s. If SKIMAGE is set, '
+                          ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT),
                     default=os.path.join('expectations', 'gm'))
 parser.add_argument('--keep-going-on-failure', action='store_true',
                     help=('instead of halting at the first error encountered, '
                           'keep going and rebaseline as many tests as '
                           'possible, and then report the full set of errors '
                           'at the end'))
 parser.add_argument('--notes',
                     help=('free-form text notes to add to all updated '
                           'expectations'))
 # TODO(epoger): Add test that exercises --tests argument.
 parser.add_argument('--tests', metavar='TEST', nargs='+',
                     help=('which tests to rebaseline, e.g. '
                           '"--tests aaclip bigmatrix", as a filter over the '
                           'full set of results in ACTUALS_FILENAME; if '
                           'unspecified, rebaseline *all* tests that are '
-                          'available.'))
+                          'available. Ignored if SKIMAGE is True.'))
 parser.add_argument('--unreviewed', action='store_true',
                     help=('mark all expectations modified by this run as '
                           '"%s": False' %
                           gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED))
 parser.add_argument('--ignore-failure', action='store_true',
                     help=('mark all expectations modified by this run as '
                           '"%s": True' %
                           gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED))
 parser.add_argument('--from-trybot', action='store_true',
                     help=('pull the actual-results.json file from the '
                           'corresponding trybot, rather than the main builder'))
+parser.add_argument('--skimage', action='store_true',
+                    help=('Rebaseline skimage results instead of gm. Defaults '
+                          'to False. If True, TESTS and CONFIGS are ignored, '
+                          'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set '
+                          'to alternate defaults, specific to skimage.'))
 args = parser.parse_args()
 exception_handler = ExceptionHandler(
     keep_going_on_failure=args.keep_going_on_failure)
 if args.builders:
   builders = args.builders
   missing_json_is_fatal = True
 else:
   builders = sorted(TEST_BUILDERS)
   missing_json_is_fatal = False
+if args.skimage:
+  # Use a different default if --skimage is specified.
+  if args.actuals_base_url == parser.get_default('actuals_base_url'):
+    args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL
+  if args.expectations_root == parser.get_default('expectations_root'):
+    args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT
 for builder in builders:
   if not builder in TEST_BUILDERS:
     raise Exception(('unrecognized builder "%s"; ' +
                      'should be one of %s') % (
                          builder, TEST_BUILDERS))
 
   expectations_json_file = os.path.join(args.expectations_root, builder,
                                         args.expectations_filename)
   if os.path.isfile(expectations_json_file):
     rebaseliner = JsonRebaseliner(
         expectations_root=args.expectations_root,
         expectations_input_filename=args.expectations_filename,
         expectations_output_filename=(args.expectations_filename_output or
                                       args.expectations_filename),
         tests=args.tests, configs=args.configs,
         actuals_base_url=args.actuals_base_url,
         actuals_filename=args.actuals_filename,
         exception_handler=exception_handler,
         add_new=args.add_new, bugs=args.bugs, notes=args.notes,
         mark_unreviewed=args.unreviewed,
         mark_ignore_failure=args.ignore_failure,
-        from_trybot=args.from_trybot)
+        from_trybot=args.from_trybot,
+        skip_pattern_matching=args.skimage)
     try:
       rebaseliner.RebaselineSubdir(builder=builder)
     except:
       exception_handler.RaiseExceptionOrContinue()
   else:
     try:
       raise _InternalException('expectations_json_file %s not found' %
                                expectations_json_file)
     except:
       exception_handler.RaiseExceptionOrContinue()
 
 exception_handler.ReportAllFailures()