rebaseline_server: add --compare-configs option

gm/rebaseline_server/server.py

 #!/usr/bin/python
 
 """
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 HTTP server for our HTML rebaseline viewer.
 """
@@ skipping 10 matching lines @@
 import socket
 import subprocess
 import sys
 import thread
 import threading
 import time
 import urlparse
 
 # Imports from within Skia
 #
-# We need to add the 'tools' directory, so that we can import svn.py within
+# We need to add the 'tools' directory for svn.py, and the 'gm' directory for
+# gm_json.py .
 # that directory.
 # Make sure that the 'tools' dir is in the PYTHONPATH, but add it at the *end*
 # so any dirs that are already in the PYTHONPATH will be preferred.
 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
-TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(PARENT_DIRECTORY))
+GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
+TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
 TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools')
 if TOOLS_DIRECTORY not in sys.path:
   sys.path.append(TOOLS_DIRECTORY)
 import svn
+if GM_DIRECTORY not in sys.path:
+  sys.path.append(GM_DIRECTORY)
+import gm_json
 
 # Imports from local dir
 #
 # Note: we import results under a different name, to avoid confusion with the
 # Server.results() property. See discussion at
 # https://codereview.chromium.org/195943004/diff/1/gm/rebaseline_server/server.py#newcode44
+import compare_configs
 import compare_to_expectations
 import imagepairset
 import results as results_mod
 
 PATHSPLIT_RE = re.compile('/([^/]+)/(.+)')
 
 # A simple dictionary of file name extensions to MIME types. The empty string
 # entry is used as the default when no extension was given or if the extension
 # has no entry in this dictionary.
 MIME_TYPE_MAP = {'': 'application/octet-stream',
                  'html': 'text/html',
                  'css': 'text/css',
                  'png': 'image/png',
                  'js': 'application/javascript',
                  'json': 'application/json'
                  }
 
 # Keys that server.py uses to create the toplevel content header.
 # NOTE: Keep these in sync with static/constants.js
 KEY__EDITS__MODIFICATIONS = 'modifications'
 KEY__EDITS__OLD_RESULTS_HASH = 'oldResultsHash'
 KEY__EDITS__OLD_RESULTS_TYPE = 'oldResultsType'
 
-DEFAULT_ACTUALS_DIR = compare_to_expectations.DEFAULT_ACTUALS_DIR
+DEFAULT_ACTUALS_DIR = results_mod.DEFAULT_ACTUALS_DIR
 DEFAULT_ACTUALS_REPO_REVISION = 'HEAD'
 DEFAULT_ACTUALS_REPO_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
 DEFAULT_PORT = 8888
 
-# Directory within which the server will serve out static files.
-STATIC_CONTENTS_SUBDIR = 'static'  # within PARENT_DIR
-GENERATED_IMAGES_SUBDIR = 'generated-images'  # within STATIC_CONTENTS_SUBDIR
+# Directory, relative to PARENT_DIRECTORY, within which the server will serve
+# out live results (not static files).
+RESULTS_SUBDIR = 'results'
+# Directory, relative to PARENT_DIRECTORY, within which the server will serve
+# out static files.
+STATIC_CONTENTS_SUBDIR = 'static'
+# All of the GENERATED_*_SUBDIRS are relative to STATIC_CONTENTS_SUBDIR
+GENERATED_HTML_SUBDIR = 'generated-html'
+GENERATED_IMAGES_SUBDIR = 'generated-images'
+GENERATED_JSON_SUBDIR = 'generated-json'
 
 # How often (in seconds) clients should reload while waiting for initial
 # results to load.
 RELOAD_INTERVAL_UNTIL_READY = 10
 
+SUMMARY_TYPES = [
+    results_mod.KEY__HEADER__RESULTS_FAILURES,
+    results_mod.KEY__HEADER__RESULTS_ALL,
+]
+# If --compare-configs is specified, compare these configs.
+CONFIG_PAIRS_TO_COMPARE = [('8888', 'gpu')]
+
 _HTTP_HEADER_CONTENT_LENGTH = 'Content-Length'
 _HTTP_HEADER_CONTENT_TYPE = 'Content-Type'
 
 _SERVER = None   # This gets filled in by main()
 
 
 def _run_command(args, directory):
   """Runs a command and returns stdout as a single string.
 
   Args:
@@ skipping 37 matching lines @@
               checkout already exists)
   Returns: an svn.Svn object referring to the local checkout.
   """
   local_checkout = svn.Svn(dir_path)
   if not os.path.isdir(dir_path):
     os.makedirs(dir_path)
     local_checkout.Checkout(repo_url, '.')
   return local_checkout
 
 
+def _create_index(file_path, config_pairs):
+  """Creates an index file linking to all results available from this server.
+
+  Prior to https://codereview.chromium.org/215503002 , we had a static
+  index.html within our repo.  But now that the results may or may not include
+  config comparisons, index.html needs to be generated differently depending
+  on which results are included.
+
+  TODO(epoger): Instead of including raw HTML within the Python code,
+  consider restoring the index.html file as a template and using django (or
+  similar) to fill in dynamic content.
+
+  Args:
+    file_path: path on local disk to write index to; any directory components
+               of this path that do not already exist will be created
+    config_pairs: what pairs of configs (if any) we compare actual results of
+  """
+  dir_path = os.path.dirname(file_path)
+  if not os.path.isdir(dir_path):
+    os.makedirs(dir_path)
+  with open(file_path, 'w') as file_handle:
+    file_handle.write(
+        '<!DOCTYPE html><html>'
+        '<head><title>rebaseline_server</title></head>'
+        '<body><ul>')
+    if SUMMARY_TYPES:
+      file_handle.write('<li>Expectations vs Actuals</li><ul>')
+      for summary_type in SUMMARY_TYPES:
+        file_handle.write(
+            '<li>'
+            '<a href="/%s/view.html#/view.html?resultsToLoad=/%s/%s">'
+            '%s</a></li>' % (
+                STATIC_CONTENTS_SUBDIR, RESULTS_SUBDIR,
+                summary_type, summary_type))
+      file_handle.write('</ul>')
+    if config_pairs:
+      file_handle.write('<li>Comparing configs within actual results</li><ul>')
+      for config_pair in config_pairs:
+        file_handle.write('<li>%s vs %s:' % config_pair)
+        for summary_type in SUMMARY_TYPES:
+          file_handle.write(
+              ' <a href="/%s/view.html#/view.html?'
+              'resultsToLoad=/%s/%s/%s-vs-%s_%s.json">%s</a>' % (
+                  STATIC_CONTENTS_SUBDIR, STATIC_CONTENTS_SUBDIR,
+                  GENERATED_JSON_SUBDIR, config_pair[0], config_pair[1],
+                  summary_type, summary_type))
+        file_handle.write('</li>')
+      file_handle.write('</ul>')
+    file_handle.write('</ul></body></html>')
+
+
 class Server(object):
   """ HTTP server for our HTML rebaseline viewer. """
 
   def __init__(self,
                actuals_dir=DEFAULT_ACTUALS_DIR,
                actuals_repo_revision=DEFAULT_ACTUALS_REPO_REVISION,
                actuals_repo_url=DEFAULT_ACTUALS_REPO_URL,
                port=DEFAULT_PORT, export=False, editable=True,
-               reload_seconds=0):
+               reload_seconds=0, config_pairs=None):
     """
     Args:
       actuals_dir: directory under which we will check out the latest actual
           GM results
       actuals_repo_revision: revision of actual-results.json files to process
       actuals_repo_url: SVN repo to download actual-results.json files from;
           if None or '', don't fetch new actual-results files at all,
           just compare to whatever files are already in actuals_dir
       port: which TCP port to listen on for HTTP requests
       export: whether to allow HTTP clients on other hosts to access this server
       editable: whether HTTP clients are allowed to submit new baselines
       reload_seconds: polling interval with which to check for new results;
           if 0, don't check for new results at all
+      config_pairs: List of (string, string) tuples; for each tuple, compare
+          actual results of these two configs.  If None or empty,
+          don't compare configs at all.
     """
     self._actuals_dir = actuals_dir
     self._actuals_repo_revision = actuals_repo_revision
     self._actuals_repo_url = actuals_repo_url
     self._port = port
     self._export = export
     self._editable = editable
     self._reload_seconds = reload_seconds
+    self._config_pairs = config_pairs or []
+    _create_index(
+        file_path=os.path.join(
+            PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR,
+            "index.html"),
+        config_pairs=config_pairs)
     if actuals_repo_url:
       self._actuals_repo = _create_svn_checkout(
           dir_path=actuals_dir, repo_url=actuals_repo_url)
 
     # Reentrant lock that must be held whenever updating EITHER of:
     # 1. self._results
     # 2. the expected or actual results on local disk
     self.results_rlock = threading.RLock()
     # self._results will be filled in by calls to update_results()
     self._results = None
@@ skipping 56 matching lines @@
       #
       # Because Skia uses depot_tools, we have to update using "gclient sync"
       # instead of raw git (or SVN) update.  Happily, this will work whether
       # the checkout was created using git or SVN.
       if self._reload_seconds:
         logging.info(
             'Updating expected GM results in %s by syncing Skia repo ...' %
             compare_to_expectations.DEFAULT_EXPECTATIONS_DIR)
         _run_command(['gclient', 'sync'], TRUNK_DIRECTORY)
 
-      self._results = compare_to_expectations.Results(
+      self._results = compare_to_expectations.ExpectationComparisons(
           actuals_root=self._actuals_dir,
           generated_images_root=os.path.join(
               PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
               GENERATED_IMAGES_SUBDIR),
           diff_base_url=posixpath.join(
               os.pardir, STATIC_CONTENTS_SUBDIR, GENERATED_IMAGES_SUBDIR))
 
+      json_dir = os.path.join(
+          PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_JSON_SUBDIR)
+      if not os.path.isdir(json_dir):
+         os.makedirs(json_dir)
+
+      for config_pair in self._config_pairs:
+        config_comparisons = compare_configs.ConfigComparisons(
+            configs=config_pair,
+            actuals_root=self._actuals_dir,
+            generated_images_root=os.path.join(
+                PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
+                GENERATED_IMAGES_SUBDIR),
+            diff_base_url=posixpath.join(
+                os.pardir, GENERATED_IMAGES_SUBDIR))
+        for summary_type in SUMMARY_TYPES:
+          gm_json.WriteToFile(
+              config_comparisons.get_packaged_results_of_type(
+                  results_type=summary_type),
+              os.path.join(
+                  json_dir, '%s-vs-%s_%s.json' % (
+                      config_pair[0], config_pair[1], summary_type)))
+
   def _result_loader(self, reload_seconds=0):
     """ Call self.update_results(), either once or periodically.
 
     Params:
       reload_seconds: integer; if nonzero, reload results at this interval
           (in which case, this method will never return!)
     """
     self.update_results()
     logging.info('Initial results loaded. Ready for requests on %s' % self._url)
     if reload_seconds:
@@ skipping 29 matching lines @@
   def do_GET(self):
     """
     Handles all GET requests, forwarding them to the appropriate
     do_GET_* dispatcher.
 
     If we see any Exceptions, return a 404.  This fixes http://skbug.com/2147
     """
     try:
       logging.debug('do_GET: path="%s"' % self.path)
       if self.path == '' or self.path == '/' or self.path == '/index.html' :
-        self.redirect_to('/%s/index.html' % STATIC_CONTENTS_SUBDIR)
+        self.redirect_to('/%s/%s/index.html' % (
+            STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR))
         return
       if self.path == '/favicon.ico' :
         self.redirect_to('/%s/favicon.ico' % STATIC_CONTENTS_SUBDIR)
         return
 
       # All requests must be of this form:
       #   /dispatcher/remainder
       # where 'dispatcher' indicates which do_GET_* dispatcher to run
       # and 'remainder' is the remaining path sent to that dispatcher.
       normpath = posixpath.normpath(self.path)
       (dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups()
       dispatchers = {
-          'results': self.do_GET_results,
-           STATIC_CONTENTS_SUBDIR: self.do_GET_static,
+          RESULTS_SUBDIR: self.do_GET_results,
+          STATIC_CONTENTS_SUBDIR: self.do_GET_static,
       }
       dispatcher = dispatchers[dispatcher_name]
       dispatcher(remainder)
     except:
       self.send_error(404)
       raise
 
   def do_GET_results(self, results_type):
     """ Handle a GET request for GM results.
 
     Args:
       results_type: string indicating which set of results to return;
             must be one of the results_mod.RESULTS_* constants
     """
     logging.debug('do_GET_results: sending results of type "%s"' % results_type)
-    # Since we must make multiple calls to the Results object, grab a
-    # reference to it in case it is updated to point at a new Results
-    # object within another thread.
+    # Since we must make multiple calls to the ExpectationComparisons object,
+    # grab a reference to it in case it is updated to point at a new
+    # ExpectationComparisons object within another thread.
     #
     # TODO(epoger): Rather than using a global variable for the handler
     # to refer to the Server object, make Server a subclass of
     # HTTPServer, and then it could be available to the handler via
     # the handler's .server instance variable.
     results_obj = _SERVER.results
     if results_obj:
       response_dict = results_obj.get_packaged_results_of_type(
           results_type=results_type, reload_seconds=_SERVER.reload_seconds,
           is_editable=_SERVER.is_editable, is_exported=_SERVER.is_exported)
@@ skipping 171 matching lines @@
                           'files from. Defaults to %(default)s ; if set to '
                           'empty string, just compare to actual-results '
                           'already found in ACTUALS_DIR.'),
                     default=DEFAULT_ACTUALS_REPO_URL)
   parser.add_argument('--actuals-revision',
                     help=('revision of actual-results.json files to process. '
                           'Defaults to %(default)s .  Beware of setting this '
                           'argument in conjunction with --editable; you '
                           'probably only want to edit results at HEAD.'),
                     default=DEFAULT_ACTUALS_REPO_REVISION)
+  parser.add_argument('--compare-configs', action='store_true',
+                      help=('In addition to generating differences between '
+                            'expectations and actuals, also generate '
+                            'differences between these config pairs: '
+                            + str(CONFIG_PAIRS_TO_COMPARE)))
   parser.add_argument('--editable', action='store_true',
                       help=('Allow HTTP clients to submit new baselines.'))
   parser.add_argument('--export', action='store_true',
                       help=('Instead of only allowing access from HTTP clients '
                             'on localhost, allow HTTP clients on other hosts '
                             'to access this server.  WARNING: doing so will '
                             'allow users on other hosts to modify your '
                             'GM expectations, if combined with --editable.'))
   parser.add_argument('--port', type=int,
                       help=('Which TCP port to listen on for HTTP requests; '
                             'defaults to %(default)s'),
                       default=DEFAULT_PORT)
   parser.add_argument('--reload', type=int,
                       help=('How often (a period in seconds) to update the '
                             'results.  If specified, both expected and actual '
                             'results will be updated by running "gclient sync" '
                             'on your Skia checkout as a whole.  '
                             'By default, we do not reload at all, and you '
                             'must restart the server to pick up new data.'),
                       default=0)
   args = parser.parse_args()
+  if args.compare_configs:
+    config_pairs = CONFIG_PAIRS_TO_COMPARE
+  else:
+    config_pairs = None
+
   global _SERVER
   _SERVER = Server(actuals_dir=args.actuals_dir,
                    actuals_repo_revision=args.actuals_revision,
                    actuals_repo_url=args.actuals_repo,
                    port=args.port, export=args.export, editable=args.editable,
-                   reload_seconds=args.reload)
+                   reload_seconds=args.reload, config_pairs=config_pairs)
   _SERVER.run()
 
 
 if __name__ == '__main__':
   main()