Remove base-* directories from gm expected/actual paths; just use platform names

trunk/tools/rebaseline.py

 #!/usr/bin/python
 
 '''
 Copyright 2012 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 '''
 
 '''
@@ skipping 18 matching lines @@
 #
 # This assumes that the 'gm' directory has been checked out as a sibling of
 # the 'tools' directory containing this script, which will be the case if
 # 'trunk' was checked out as a single unit.
 GM_DIRECTORY = os.path.realpath(
     os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
 if GM_DIRECTORY not in sys.path:
   sys.path.append(GM_DIRECTORY)
 import gm_json
 
-# Mapping of expectations/gm subdir (under
-# https://skia.googlecode.com/svn/trunk/expectations/gm/ )
-# to builder name (see list at http://108.170.217.252:10117/builders )
-SUBDIR_MAPPING = {
-   'base-shuttle-win7-intel-float':
+# Full list of builder names (under
+# https://skia.googlecode.com/svn/trunk/expectations/gm/ ,
+# which should match the list at http://108.170.217.252:10117/builders )
+#
+# TODO(epoger): Generate this list automatically.

[rmistry, 2013/08/14 12:28:05]

This should be very easy to do (you could add it to this CL):

master_url = get from global_variables.json
# Below step will take a few seconds to run.
all_builders = list(json.load(urllib2.urlopen(master_url + 'json/builders')))
test_builders = filter(lambda x: 'Trybot' not in x and 'Test' in x,
all_builders)

Also, lets remove the '_LIST' from 'BUILDERS_LIST'.

[borenet, 2013/08/14 12:57:48]

+1.  We need to get rid of the handwritten list ASAP.

[epoger, 2013/08/16 20:16:25]

Wow, that's pretty cool.  Thanks!  Done.  (You'll see that I added a helper file
to do this for both rebaseline.py and submit_try .)

+BUILDERS_LIST = [
+    'Test-Android-GalaxyNexus-SGX540-Arm7-Debug',

[epoger, 2013/08/13 19:59:37]

Patchset 1 updates rebaseline.py to use builder names, rather than subdir names.

+    'Test-Android-GalaxyNexus-SGX540-Arm7-Release',
+    'Test-Android-Nexus10-MaliT604-Arm7-Debug',
+    'Test-Android-Nexus10-MaliT604-Arm7-Release',
+    'Test-Android-Nexus4-Adreno320-Arm7-Debug',
+    'Test-Android-Nexus4-Adreno320-Arm7-Release',
+    'Test-Android-Nexus7-Tegra3-Arm7-Debug',
+    'Test-Android-Nexus7-Tegra3-Arm7-Release',
+    'Test-Android-NexusS-SGX540-Arm7-Debug',
+    'Test-Android-NexusS-SGX540-Arm7-Release',
+    'Test-Android-RazrI-SGX540-x86-Debug',
+    'Test-Android-RazrI-SGX540-x86-Release',
+    'Test-Android-Xoom-Tegra2-Arm7-Debug',
+    'Test-Android-Xoom-Tegra2-Arm7-Release',
+    'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug',
+    'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release',
+    'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
+    'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release',
+    'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug',
+    'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release',
+    'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug',
+    'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release',
+    'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug',
+    'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release',
+    'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug',
+    'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release',
+    'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Debug',
+    'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release',
+    'Test-Ubuntu12-ShuttleA-ATI5770-x86-Debug',
+    'Test-Ubuntu12-ShuttleA-ATI5770-x86-Release',
+    'Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug',
+    'Test-Win7-ShuttleA-HD2000-x86_64-Debug',
+    'Test-Win7-ShuttleA-HD2000-x86_64-Release',
+    'Test-Win7-ShuttleA-HD2000-x86-Debug',
+    'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE',
+    'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite',
     'Test-Win7-ShuttleA-HD2000-x86-Release',
-   'base-shuttle-win7-intel-angle':
     'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE',
-   'base-shuttle-win7-intel-directwrite':
     'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite',
-   'base-shuttle_ubuntu12_ati5770':
-    'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release',
-   'base-macmini':
-    'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release',
-   'base-macmini-lion-float':
-    'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release',
-   'base-android-galaxy-nexus':
-    'Test-Android-GalaxyNexus-SGX540-Arm7-Debug',
-   'base-android-nexus-7':
-    'Test-Android-Nexus7-Tegra3-Arm7-Release',
-   'base-android-nexus-s':
-    'Test-Android-NexusS-SGX540-Arm7-Release',
-   'base-android-xoom':
-    'Test-Android-Xoom-Tegra2-Arm7-Release',
-   'base-android-nexus-10':
-    'Test-Android-Nexus10-MaliT604-Arm7-Release',
-   'base-android-nexus-4':
-    'Test-Android-Nexus4-Adreno320-Arm7-Release',
-}
+]
 
 
 class _InternalException(Exception):
   pass
 
 # Object that handles exceptions, either raising them immediately or collecting
 # them to display later on.
 class ExceptionHandler(object):
 
   # params:
@@ skipping 128 matching lines @@
     actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
     if not sections:
       sections = actual_results.keys()
     for section in sections:
       section_results = actual_results[section]
       if section_results:
         results_to_return.update(section_results)
     return results_to_return
 
   # Rebaseline all tests/types we specified in the constructor,
-  # within this expectations/gm subdir.
+  # within this builder's subdirectory in expectations/gm .
   #
   # params:
-  #  subdir : e.g. 'base-shuttle-win7-intel-float'
   #  builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release'
-  def RebaselineSubdir(self, subdir, builder):
+  def RebaselineSubdir(self, builder):
     # Read in the actual result summary, and extract all the tests whose
     # results we need to update.
     actuals_url = '/'.join([self._actuals_base_url,
-                            subdir, builder, subdir,
-                            self._actuals_filename])
+                            builder, self._actuals_filename])
     # In most cases, we won't need to re-record results that are already
     # succeeding, but including the SUCCEEDED results will allow us to
     # re-record expectations if they somehow get out of sync.
     sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED,
                 gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED]
     if self._add_new:
       sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON)
     results_to_update = self._GetActualResults(json_url=actuals_url,
                                                sections=sections)
 
     # Read in current expectations.
     expectations_input_filepath = os.path.join(
-        self._expectations_root, subdir, self._expectations_input_filename)
+        self._expectations_root, builder, self._expectations_input_filename)
     expectations_dict = gm_json.LoadFromFile(expectations_input_filepath)
     expected_results = expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS]
 
     # Update the expectations in memory, skipping any tests/configs that
     # the caller asked to exclude.
     skipped_images = []
     if results_to_update:
       for (image_name, image_results) in results_to_update.iteritems():
         (test, config) = self._image_filename_re.match(image_name).groups()
         if self._tests:
           if test not in self._tests:
             skipped_images.append(image_name)
             continue
         if self._configs:
           if config not in self._configs:
             skipped_images.append(image_name)
             continue
         if not expected_results.get(image_name):
           expected_results[image_name] = {}
         expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] = \
                         [image_results]
 
     # Write out updated expectations.
     expectations_output_filepath = os.path.join(
-        self._expectations_root, subdir, self._expectations_output_filename)
+        self._expectations_root, builder, self._expectations_output_filename)
     gm_json.WriteToFile(expectations_dict, expectations_output_filepath)
 
     # Mark the JSON file as plaintext, so text-style diffs can be applied.
     # Fixes https://code.google.com/p/skia/issues/detail?id=1442
     if self._using_svn:
       self._Call(['svn', 'propset', '--quiet', 'svn:mime-type',
                   'text/x-json', expectations_output_filepath])
 
 # main...
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--actuals-base-url',
                     help='base URL from which to read files containing JSON ' +
                     'summaries of actual GM results; defaults to %(default)s',
                     default='http://skia-autogen.googlecode.com/svn/gm-actual')
 parser.add_argument('--actuals-filename',
-                    help='filename (within platform-specific subdirectories ' +
+                    help='filename (within builder-specific subdirectories ' +
                     'of ACTUALS_BASE_URL) to read a summary of results from; ' +
                     'defaults to %(default)s',
                     default='actual-results.json')
 # TODO(epoger): Add test that exercises --add-new argument.
 parser.add_argument('--add-new', action='store_true',
                     help='in addition to the standard behavior of ' +
                     'updating expectations for failing tests, add ' +
                     'expectations for tests which don\'t have expectations ' +
                     'yet.')
+parser.add_argument('--builders', metavar='BUILDER', nargs='+',
+                    help='which platforms to rebaseline; ' +
+                    'if unspecified, rebaseline all platforms, same as ' +
+                    '"--builders %s"' % ' '.join(sorted(BUILDERS_LIST)))
 # TODO(epoger): Add test that exercises --configs argument.
 parser.add_argument('--configs', metavar='CONFIG', nargs='+',
                     help='which configurations to rebaseline, e.g. ' +
                     '"--configs 565 8888", as a filter over the full set of ' +
                     'results in ACTUALS_FILENAME; if unspecified, rebaseline ' +
                     '*all* configs that are available.')
 parser.add_argument('--expectations-filename',
                     help='filename (under EXPECTATIONS_ROOT) to read ' +
                     'current expectations from, and to write new ' +
                     'expectations into (unless a separate ' +
                     'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' +
                     'defaults to %(default)s',
                     default='expected-results.json')
 parser.add_argument('--expectations-filename-output',
                     help='filename (under EXPECTATIONS_ROOT) to write ' +
                     'updated expectations into; by default, overwrites the ' +
                     'input file (EXPECTATIONS_FILENAME)',
                     default='')
 parser.add_argument('--expectations-root',
                     help='root of expectations directory to update-- should ' +
-                    'contain one or more base-* subdirectories. Defaults to ' +
+                    'contain one or more builder subdirectories. Defaults to ' +
                     '%(default)s',
                     default=os.path.join('expectations', 'gm'))
 parser.add_argument('--keep-going-on-failure', action='store_true',
                     help='instead of halting at the first error encountered, ' +
                     'keep going and rebaseline as many tests as possible, ' +
                     'and then report the full set of errors at the end')
-parser.add_argument('--subdirs', metavar='SUBDIR', nargs='+',
-                    help='which platform subdirectories to rebaseline; ' +
-                    'if unspecified, rebaseline all subdirs, same as ' +
-                    '"--subdirs %s"' % ' '.join(sorted(SUBDIR_MAPPING.keys())))
 # TODO(epoger): Add test that exercises --tests argument.
 parser.add_argument('--tests', metavar='TEST', nargs='+',
                     help='which tests to rebaseline, e.g. ' +
                     '"--tests aaclip bigmatrix", as a filter over the full ' +
                     'set of results in ACTUALS_FILENAME; if unspecified, ' +
                     'rebaseline *all* tests that are available.')
 args = parser.parse_args()
 exception_handler = ExceptionHandler(
     keep_going_on_failure=args.keep_going_on_failure)
-if args.subdirs:
-  subdirs = args.subdirs
+if args.builders:
+  builders = args.builders
   missing_json_is_fatal = True
 else:
-  subdirs = sorted(SUBDIR_MAPPING.keys())
+  builders = sorted(BUILDERS_LIST)
   missing_json_is_fatal = False
-for subdir in subdirs:
-  if not subdir in SUBDIR_MAPPING.keys():
-    raise Exception(('unrecognized platform subdir "%s"; ' +
+for builder in builders:
+  if not builder in BUILDERS_LIST:
+    raise Exception(('unrecognized builder "%s"; ' +
                      'should be one of %s') % (
-                         subdir, SUBDIR_MAPPING.keys()))
-  builder = SUBDIR_MAPPING[subdir]
+                         builder, BUILDERS_LIST))
 
-  # We instantiate different Rebaseliner objects depending
-  # on whether we are rebaselining an expected-results.json file, or
-  # individual image files.  Different expectations/gm subdirectories may move
-  # from individual image files to JSON-format expectations at different
-  # times, so we need to make this determination per subdirectory.
-  #
-  # See https://goto.google.com/ChecksumTransitionDetail
-  expectations_json_file = os.path.join(args.expectations_root, subdir,
+  expectations_json_file = os.path.join(args.expectations_root, builder,
                                         args.expectations_filename)
   if os.path.isfile(expectations_json_file):
     rebaseliner = JsonRebaseliner(
         expectations_root=args.expectations_root,
         expectations_input_filename=args.expectations_filename,
         expectations_output_filename=(args.expectations_filename_output or
                                       args.expectations_filename),
         tests=args.tests, configs=args.configs,
         actuals_base_url=args.actuals_base_url,
         actuals_filename=args.actuals_filename,
         exception_handler=exception_handler,
         add_new=args.add_new)
     try:
-      rebaseliner.RebaselineSubdir(subdir=subdir, builder=builder)
+      rebaseliner.RebaselineSubdir(builder=builder)
     except BaseException as e:
       exception_handler.RaiseExceptionOrContinue(e)
   else:
     exception_handler.RaiseExceptionOrContinue(_InternalException(
         'expectations_json_file %s not found' % expectations_json_file))
 
 exception_handler.ReportAllFailures()