rebaseline.py: if --tests is not specified, get test list from actual-results.json

tools/rebaseline.py

 #!/usr/bin/python

[epoger, 2013/06/05 03:16:20]

Toplevel comment: This change is part of "Step 1" in
https://goto.google.com/ChecksumTransitionDetail .

With this CL we can get the list of images that need rebaselining from
actual-results.json, and the typical rebaselining use case becomes simply this:

python .../tools/rebaseline.py

In an upcoming CL, I will complete "Step 1" by using actual-results.json to also
give me the actual image checksums, which tells me where they might be available
in Google Storage.

 
 '''
 Copyright 2012 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 '''
 
 '''
 Rebaselines the given GM tests, on all bots and all configurations.
 Must be run from the gm-expected directory.  If run from a git or SVN
 checkout, the files will be added to the staging area for commit.
 '''
 
+# System-level imports
 import argparse
 import os
 import subprocess
 import sys
+import urllib2
+
+# Imports from within Skia
+#
+# Make sure that they are in the PYTHONPATH, but add them at the *end*
+# so any that are already in the PYTHONPATH will be preferred.
+GM_DIRECTORY = os.path.realpath(
+    os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
+if GM_DIRECTORY not in sys.path:
+    sys.path.append(GM_DIRECTORY)
+import gm_json
+
 
 # Mapping of gm-expectations subdir (under
 # https://skia.googlecode.com/svn/gm-expected/ )
 # to builder name (see list at http://108.170.217.252:10117/builders )
 SUBDIR_MAPPING = {
    'base-shuttle-win7-intel-float':
     'Test-Win7-ShuttleA-HD2000-x86-Release',
    'base-shuttle-win7-intel-angle':
     'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE',
    'base-shuttle-win7-intel-directwrite':
@@ skipping 16 matching lines @@
     'Test-Android-Nexus10-MaliT604-Arm7-Release',
 }
 
 
 class CommandFailedException(Exception):
     pass
 
 class Rebaseliner(object):
 
     # params:
-    #  tests: list of tests to rebaseline
+    #  tests: list of tests to rebaseline, or None if we should rebaseline
+    #         whatever tests the actual-results.json file tells us to
+    #  json_base_url: base URL from which to read json_filename
+    #  json_filename: filename (under json_base_url) from which to read a
+    #                 summary of results
     #  configs: which configs to run for each test
     #  subdirs: which platform subdirectories to rebaseline; if an empty list,
     #           rebaseline all platform subdirectories
     #  dry_run: if True, instead of actually downloading files or adding
     #           files to checkout, display a list of operations that
     #           we would normally perform
-    def __init__(self, tests, configs=[], subdirs=[], dry_run=False):
-        if not tests:
-            raise Exception('at least one test must be specified')
+    def __init__(self, tests, json_base_url, json_filename,
+                 configs=[], subdirs=[], dry_run=False):
         self._tests = tests
         self._configs = configs
         if not subdirs:
             self._subdirs = sorted(SUBDIR_MAPPING.keys())
         else:
             self._subdirs = subdirs
+        self._json_base_url = json_base_url
+        self._json_filename = json_filename
         self._dry_run = dry_run
         self._is_svn_checkout = (
             os.path.exists('.svn') or
             os.path.exists(os.path.join(os.pardir, '.svn')))
         self._is_git_checkout = (
             os.path.exists('.git') or
             os.path.exists(os.path.join(os.pardir, '.git')))
 
     # If dry_run is False, execute subprocess.call(cmd).
     # If dry_run is True, print the command we would have otherwise run.
@@ skipping 12 matching lines @@
         # so that we don't corrupt the existing file if it fails midway thru.
         temp_filename = os.path.join(os.path.dirname(dest_filename),
                                      '.temp-' + os.path.basename(dest_filename))
 
         # TODO(epoger): Replace calls to "curl"/"mv" (which will only work on
         # Unix) with a Python HTTP library (which should work cross-platform)
         self._Call([ 'curl', '--fail', '--silent', source_url,
                      '--output', temp_filename ])
         self._Call([ 'mv', temp_filename, dest_filename ])
 
+    # Returns the full contents of a URL, as a single string.
+    #
+    # Unlike standard URL handling, we allow relative "file:" URLs;
+    # for example, "file:one/two" resolves to the file ./one/two
+    # (relative to current working dir)
+    def _GetContentsOfUrl(self, url):
+        file_prefix = 'file:'
+        if url.startswith(file_prefix):
+            filename = url[len(file_prefix):]
+            return open(filename, 'r').read()
+        else:
+            return urllib2.urlopen(url).read()
+
+    # Returns a list of files that require rebaselining.
+    #
+    # Note that this returns a list of FILES, like this:
+    #  ['imageblur_565.png', 'xfermodes_pdf.png']
+    # rather than a list of TESTS, like this:
+    #  ['imageblur', 'xfermodes']
+    #
+    # params:
+    #  json_url: URL pointing to a JSON actual result summary file
+    #
+    # TODO(epoger): add a parameter indicating whether "no-comparison"
+    # results (those for which we don't have any expectations yet)
+    # should be rebaselined.  For now, we only return failed expectations.
+    def _GetFilesToRebaseline(self, json_url):
+        print ('# Getting tests to rebaseline from JSON summary URL %s ...'
+               % json_url)
+        json_contents = self._GetContentsOfUrl(json_url)
+        json_dict = gm_json.LoadFromString(json_contents)
+        actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
+
+        tests_to_rebaseline = []
+        failed_results = actual_results[gm_json.JSONKEY_ACTUALRESULTS_FAILED]
+        if failed_results:
+            tests_to_rebaseline.extend(failed_results.keys())
+
+        print '# ... found tests_to_rebaseline %s' % tests_to_rebaseline
+        return tests_to_rebaseline
+
     # Rebaseline a single file.
     def _RebaselineOneFile(self, expectations_subdir, builder_name,
                            infilename, outfilename):
+        print '# ' + infilename
         url = ('http://skia-autogen.googlecode.com/svn/gm-actual/' +
                expectations_subdir + '/' + builder_name + '/' +
                expectations_subdir + '/' + infilename)
 
         # Try to download this file, but if that fails, keep going...
         #
         # This not treated as a fatal failure because not all
         # platforms generate all configs (e.g., Android does not
         # generate PDF).
         #
@@ skipping 31 matching lines @@
             configs = self._configs
         else:
             if (expectations_subdir == 'base-shuttle-win7-intel-angle'):
                 configs = [ 'angle', 'anglemsaa16' ]
             else:
                 configs = [ '565', '8888', 'gpu', 'pdf', 'mesa', 'msaa16',
                             'msaa4' ]
         print '# ' + expectations_subdir + ':'
         for config in configs:
             infilename = test + '_' + config + '.png'
-            print '# ' + infilename
             outfilename = os.path.join(expectations_subdir, infilename);
             self._RebaselineOneFile(expectations_subdir=expectations_subdir,
                                     builder_name=builder_name,
                                     infilename=infilename,
                                     outfilename=outfilename)
 
     # Rebaseline all platforms/tests/types we specified in the constructor.
     def RebaselineAll(self):
-        for test in self._tests:
-            for subdir in self._subdirs:
-                if not subdir in SUBDIR_MAPPING.keys():
-                    raise Exception(('unrecognized platform subdir "%s"; ' +
-                                     'should be one of %s') % (
-                                         subdir, SUBDIR_MAPPING.keys()))
-                builder_name = SUBDIR_MAPPING[subdir]
-                self._RebaselineOneTest(expectations_subdir=subdir,
-                                        builder_name=builder_name,
-                                        test=test)
-
+        for subdir in self._subdirs:
+            if not subdir in SUBDIR_MAPPING.keys():
+                raise Exception(('unrecognized platform subdir "%s"; ' +
+                                 'should be one of %s') % (
+                                     subdir, SUBDIR_MAPPING.keys()))
+            builder_name = SUBDIR_MAPPING[subdir]
+            if self._tests:
+                for test in self._tests:
+                    self._RebaselineOneTest(expectations_subdir=subdir,
+                                            builder_name=builder_name,
+                                            test=test)
+            else:  # get the raw list of files that need rebaselining from JSON
+                json_url = '/'.join([self._json_base_url,
+                                     subdir, builder_name, subdir,
+                                     self._json_filename])
+                filenames = self._GetFilesToRebaseline(json_url=json_url)
+                for filename in filenames:
+                    outfilename = os.path.join(subdir, filename);
+                    self._RebaselineOneFile(expectations_subdir=subdir,

[epoger, 2013/06/05 03:16:20]

Patchset 5 fixes the JSON case: when rebaselining based on actual-results.json,
we don't need to multiply the test cases by config types.  The JSON file gives
us the full set of files that need rebaselining.

+                                            builder_name=builder_name,
+                                            infilename=filename,
+                                            outfilename=outfilename)
 
 # main...
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--configs', metavar='CONFIG', nargs='+',
                     help='which configurations to rebaseline, e.g. ' +
                     '"--configs 565 8888"; if unspecified, run a default ' +
                     'set of configs')
 parser.add_argument('--dry_run', action='store_true',
                     help='instead of actually downloading files or adding ' +
                     'files to checkout, display a list of operations that ' +
                     'we would normally perform')
+parser.add_argument('--json_base_url',
+                    help='base URL from which to read JSON_FILENAME ' +
+                    'files; defaults to %(default)s',
+                    default='http://skia-autogen.googlecode.com/svn/gm-actual')
+parser.add_argument('--json_filename',
+                    help='filename (under JSON_BASE_URL) to read a summary ' +
+                    'of results from; defaults to %(default)s',
+                    default='actual-results.json')
 parser.add_argument('--subdirs', metavar='SUBDIR', nargs='+',
                     help='which platform subdirectories to rebaseline; ' +
                     'if unspecified, rebaseline all subdirs, same as ' +
                     '"--subdirs %s"' % ' '.join(sorted(SUBDIR_MAPPING.keys())))
-parser.add_argument('--tests', metavar='TEST', nargs='+', required=True,
+parser.add_argument('--tests', metavar='TEST', nargs='+',
                     help='which tests to rebaseline, e.g. ' +
-                    '"--tests aaclip bigmatrix"')
+                    '"--tests aaclip bigmatrix"; if unspecified, then all ' +
+                    'failing tests (according to the actual-results.json ' +
+                    'file) will be rebaselined.')
 args = parser.parse_args()
 rebaseliner = Rebaseliner(tests=args.tests, configs=args.configs,
-                          subdirs=args.subdirs, dry_run=args.dry_run)
+                          subdirs=args.subdirs, dry_run=args.dry_run,
+                          json_base_url=args.json_base_url,
+                          json_filename=args.json_filename)
 rebaseliner.RebaselineAll()