rebaseline.py: if --tests is not specified, get test list from actual-results.json

tools/rebaseline.py

 #!/usr/bin/python
 
 '''
 Copyright 2012 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 '''
 
 '''
 Rebaselines the given GM tests, on all bots and all configurations.
 Must be run from the gm-expected directory.  If run from a git or SVN
 checkout, the files will be added to the staging area for commit.
 '''
 
+# System-level imports
 import argparse
 import os
 import subprocess
 import sys
+import urllib2
+
+# Imports from within Skia
+#
+# Make sure that they are in the PYTHONPATH, but add them at the *end*
+# so any that are already in the PYTHONPATH will be preferred.
+GM_DIRECTORY = os.path.realpath(
+    os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
+if GM_DIRECTORY not in sys.path:
+    sys.path.append(GM_DIRECTORY)
+import gm_json
+
 
 # Mapping of gm-expectations subdir (under
 # https://skia.googlecode.com/svn/gm-expected/ )
 # to builder name (see list at http://108.170.217.252:10117/builders )
 SUBDIR_MAPPING = {
    'base-shuttle-win7-intel-float':
     'Test-Win7-ShuttleA-HD2000-x86-Release',
    'base-shuttle-win7-intel-angle':
     'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE',
    'base-shuttle-win7-intel-directwrite':
@@ skipping 16 matching lines @@
     'Test-Android-Nexus10-MaliT604-Arm7-Release',
 }
 
 
 class CommandFailedException(Exception):
     pass
 
 class Rebaseliner(object):
 
     # params:
-    #  tests: list of tests to rebaseline
+    #  tests: list of tests to rebaseline, or None if we should rebaseline
+    #         whatever tests the actual-results.json file tells us to
+    #  json_base_url: base URL from which to read json_filename
+    #  json_filename: filename (under json_base_url) from which to read a
+    #                 summary of results
     #  configs: which configs to run for each test
     #  subdirs: which platform subdirectories to rebaseline; if an empty list,
     #           rebaseline all platform subdirectories
     #  dry_run: if True, instead of actually downloading files or adding
     #           files to checkout, display a list of operations that
     #           we would normally perform
-    def __init__(self, tests, configs=[], subdirs=[], dry_run=False):
-        if not tests:
-            raise Exception('at least one test must be specified')
+    def __init__(self, tests, json_base_url, json_filename,
+                 configs=[], subdirs=[], dry_run=False):
         self._tests = tests
         self._configs = configs
         if not subdirs:
             self._subdirs = sorted(SUBDIR_MAPPING.keys())
         else:
             self._subdirs = subdirs
+        self._json_base_url = json_base_url
+        self._json_filename = json_filename
         self._dry_run = dry_run
         self._is_svn_checkout = (
             os.path.exists('.svn') or
             os.path.exists(os.path.join(os.pardir, '.svn')))
         self._is_git_checkout = (
             os.path.exists('.git') or
             os.path.exists(os.path.join(os.pardir, '.git')))
 
     # If dry_run is False, execute subprocess.call(cmd).
     # If dry_run is True, print the command we would have otherwise run.
@@ skipping 12 matching lines @@
         # so that we don't corrupt the existing file if it fails midway thru.
         temp_filename = os.path.join(os.path.dirname(dest_filename),
                                      '.temp-' + os.path.basename(dest_filename))
 
         # TODO(epoger): Replace calls to "curl"/"mv" (which will only work on
         # Unix) with a Python HTTP library (which should work cross-platform)
         self._Call([ 'curl', '--fail', '--silent', source_url,
                      '--output', temp_filename ])
         self._Call([ 'mv', temp_filename, dest_filename ])
 
+    # Returns the full contents of a URL, as a single string.
+    #
+    # Unlike standard URL handling, we allow relative "file:" URLs;
+    # for example, "file:one/two" resolves to the file ./one/two
+    # (relative to current working dir)
+    def _GetContentsOfUrl(self, url):
+        file_prefix = 'file:'
+        if url.startswith(file_prefix):
+            filename = url[len(file_prefix):]
+            return open(filename, 'r').read()
+        else:
+            return urllib2.urlopen(url).read()
+
+    # Returns a list of tests that require rebaselining.
+    #
+    # params:
+    #  json_url: URL pointing to a JSON actual result summary file
+    #
+    # TODO(epoger): add a parameter indicating whether "no-comparison"
+    # results (those for which we don't have any expectations yet)
+    # should be rebaselined.  For now, we only return failed expectations.
+    def _GetTestsToRebaseline(self, json_url):
+        print ('# Getting tests to rebaseline from JSON summary file %s ...'
+               % json_url)
+        json_contents = self._GetContentsOfUrl(json_url)
+        json_dict = gm_json.LoadFromString(json_contents)
+        actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
+        failed_results = actual_results[gm_json.JSONKEY_ACTUALRESULTS_FAILED]
+        tests_to_rebaseline = failed_results.keys()
+        print '# ... found tests_to_rebaseline %s' % tests_to_rebaseline
+        return tests_to_rebaseline
+
     # Rebaseline a single file.
     def _RebaselineOneFile(self, expectations_subdir, builder_name,
                            infilename, outfilename):
         url = ('http://skia-autogen.googlecode.com/svn/gm-actual/' +
                expectations_subdir + '/' + builder_name + '/' +
                expectations_subdir + '/' + infilename)
 
         # Try to download this file, but if that fails, keep going...
         #
         # This not treated as a fatal failure because not all
@@ skipping 43 matching lines @@
             infilename = test + '_' + config + '.png'
             print '# ' + infilename
             outfilename = os.path.join(expectations_subdir, infilename);
             self._RebaselineOneFile(expectations_subdir=expectations_subdir,
                                     builder_name=builder_name,
                                     infilename=infilename,
                                     outfilename=outfilename)
 
     # Rebaseline all platforms/tests/types we specified in the constructor.
     def RebaselineAll(self):
-        for test in self._tests:
-            for subdir in self._subdirs:
-                if not subdir in SUBDIR_MAPPING.keys():
-                    raise Exception(('unrecognized platform subdir "%s"; ' +
-                                     'should be one of %s') % (
-                                         subdir, SUBDIR_MAPPING.keys()))
-                builder_name = SUBDIR_MAPPING[subdir]
+        for subdir in self._subdirs:
+            if not subdir in SUBDIR_MAPPING.keys():
+                raise Exception(('unrecognized platform subdir "%s"; ' +
+                                 'should be one of %s') % (
+                                     subdir, SUBDIR_MAPPING.keys()))
+            builder_name = SUBDIR_MAPPING[subdir]
+            if self._tests:

[epoger, 2013/06/04 18:52:23]

Patchset 3: If --tests has been specified, just use that list... but if not,
download the latest actual-results.json file for the given platform, and
rebaseline all tests marked as failing there.

The fact that the existing tests still pass indicates that the "--tests have
been specified" path still works as before.  I will add more self-tests to
exercise the JSON path...

+                tests = self._tests
+            else:
+                json_url = '/'.join([self._json_base_url,
+                                     subdir, builder_name, subdir,
+                                     self._json_filename])
+                tests = self._GetTestsToRebaseline(json_url=json_url)
+            for test in tests:
                 self._RebaselineOneTest(expectations_subdir=subdir,
                                         builder_name=builder_name,
                                         test=test)
 
 
 # main...
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--configs', metavar='CONFIG', nargs='+',
                     help='which configurations to rebaseline, e.g. ' +
                     '"--configs 565 8888"; if unspecified, run a default ' +
                     'set of configs')
 parser.add_argument('--dry_run', action='store_true',
                     help='instead of actually downloading files or adding ' +
                     'files to checkout, display a list of operations that ' +
                     'we would normally perform')
+parser.add_argument('--json_base_url',
+                    help='base URL from which to read JSON_FILENAME ' +
+                    'files; defaults to %(default)s',
+                    default='http://skia-autogen.googlecode.com/svn/gm-actual')
+parser.add_argument('--json_filename',
+                    help='filename (under JSON_BASE_URL) to read a summary ' +
+                    'of results from; defaults to %(default)s',
+                    default='actual-results.json')
 parser.add_argument('--subdirs', metavar='SUBDIR', nargs='+',
                     help='which platform subdirectories to rebaseline; ' +
                     'if unspecified, rebaseline all subdirs, same as ' +
                     '"--subdirs %s"' % ' '.join(sorted(SUBDIR_MAPPING.keys())))
-parser.add_argument('--tests', metavar='TEST', nargs='+', required=True,
+parser.add_argument('--tests', metavar='TEST', nargs='+',
                     help='which tests to rebaseline, e.g. ' +
-                    '"--tests aaclip bigmatrix"')
+                    '"--tests aaclip bigmatrix"; if unspecified, then all ' +
+                    'failing tests (according to the actual-results.json ' +
+                    'file) will be rebaselined.')
 args = parser.parse_args()
 rebaseliner = Rebaseliner(tests=args.tests, configs=args.configs,
-                          subdirs=args.subdirs, dry_run=args.dry_run)
+                          subdirs=args.subdirs, dry_run=args.dry_run,
+                          json_base_url=args.json_base_url,
+                          json_filename=args.json_filename)
 rebaseliner.RebaselineAll()