teach rebaseline_server to generate diffs of rendered SKPs

gm/rebaseline_server/server.py

Index: gm/rebaseline_server/server.py
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py
index 50367f6f35eab16fc045b4d4433db01e6adb145b..540c7b0de628ccef6293faba970971835b191d09 100755
--- a/gm/rebaseline_server/server.py
+++ b/gm/rebaseline_server/server.py
@@ -40,6 +40,7 @@ import gm_json
 # https://codereview.chromium.org/195943004/diff/1/gm/rebaseline_server/server.py#newcode44
 # pylint: enable=C0301
 import compare_configs
+import compare_rendered_pictures
 import compare_to_expectations
 import download_actuals
 import imagediffdb
@@ -73,8 +74,11 @@ DEFAULT_PORT = 8888
 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(PARENT_DIRECTORY))
 # Directory, relative to PARENT_DIRECTORY, within which the server will serve
-# out live results (not static files).
-RESULTS_SUBDIR = 'results'
+# out image diff data from within the precomputed _SERVER.results .
+PRECOMPUTED_RESULTS_SUBDIR = 'results'
+# Directory, relative to PARENT_DIRECTORY, within which the server will serve
+# out live-generated image diff data.
+LIVE_RESULTS_SUBDIR = 'live-results'
 # Directory, relative to PARENT_DIRECTORY, within which the server will serve
 # out static files.
 STATIC_CONTENTS_SUBDIR = 'static'
@@ -165,7 +169,7 @@ def _create_index(file_path, config_pairs):
             '<li><a href="/{static_subdir}/view.html#/view.html?'
             'resultsToLoad=/{results_subdir}/{summary_type}">'
             '{summary_type}</a></li>'.format(
-                results_subdir=RESULTS_SUBDIR,
+                results_subdir=PRECOMPUTED_RESULTS_SUBDIR,
                 static_subdir=STATIC_CONTENTS_SUBDIR,
                 summary_type=summary_type))
       file_handle.write('</ul>')
@@ -193,7 +197,8 @@ class Server(object):
                json_filename=DEFAULT_JSON_FILENAME,
                gm_summaries_bucket=DEFAULT_GM_SUMMARIES_BUCKET,
                port=DEFAULT_PORT, export=False, editable=True,
-               reload_seconds=0, config_pairs=None, builder_regex_list=None):
+               reload_seconds=0, config_pairs=None, builder_regex_list=None,
+               boto_file_path=None):
     """
     Args:
       actuals_dir: directory under which we will check out the latest actual
@@ -212,6 +217,9 @@ class Server(object):
           don't compare configs at all.
       builder_regex_list: List of regular expressions specifying which builders
           we will process. If None, process all builders.
+      boto_file_path: Path to .boto file giving us credentials to access
+          Google Storage buckets; if None, we will only be able to access
+          public GS buckets.
     """
     self._actuals_dir = actuals_dir
     self._json_filename = json_filename
@@ -222,7 +230,12 @@ class Server(object):
     self._reload_seconds = reload_seconds
     self._config_pairs = config_pairs or []
     self._builder_regex_list = builder_regex_list
-    self._gs = gs_utils.GSUtils()
+
+    if boto_file_path:
+      self._gs = gs_utils.GSUtils(boto_file_path=boto_file_path)
+    else:
+      self._gs = gs_utils.GSUtils()
+
     _create_index(
         file_path=os.path.join(
             PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR,
@@ -234,9 +247,15 @@ class Server(object):
     # 2. the expected or actual results on local disk
     self.results_rlock = threading.RLock()
 
-    # These will be filled in by calls to update_results()
+    # Create a single ImageDiffDB instance that is used by all our differs.
+    self._image_diff_db = imagediffdb.ImageDiffDB(
+        gs=self._gs,
+        storage_root=os.path.join(
+            PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
+            GENERATED_IMAGES_SUBDIR))
+
+    # This will be filled in by calls to update_results()
     self._results = None
-    self._image_diff_db = None
 
   @property
   def results(self):
@@ -343,12 +362,6 @@ class Server(object):
             compare_to_expectations.DEFAULT_EXPECTATIONS_DIR)
         _run_command(['gclient', 'sync'], TRUNK_DIRECTORY)
 
-      if not self._image_diff_db:
-        self._image_diff_db = imagediffdb.ImageDiffDB(
-            storage_root=os.path.join(
-              PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
-              GENERATED_IMAGES_SUBDIR))
-
       self._results = compare_to_expectations.ExpectationComparisons(
           image_diff_db=self._image_diff_db,
           actuals_root=self._actuals_dir,
@@ -443,7 +456,8 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       normpath = posixpath.normpath(self.path)
       (dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups()
       dispatchers = {
-          RESULTS_SUBDIR: self.do_GET_results,
+          PRECOMPUTED_RESULTS_SUBDIR: self.do_GET_precomputed_results,
+          LIVE_RESULTS_SUBDIR: self.do_GET_live_results,
           STATIC_CONTENTS_SUBDIR: self.do_GET_static,
       }
       dispatcher = dispatchers[dispatcher_name]
@@ -452,14 +466,15 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       self.send_error(404)
       raise
 
-  def do_GET_results(self, results_type):
-    """ Handle a GET request for GM results.
+  def do_GET_precomputed_results(self, results_type):

[epoger, 2014/07/31 15:18:31]

For now, leaving this one in place to fetch the latest GM results (which are
pregenerated every 5 minutes using the --reload option, for fastest user
response).

Eventually, I hope we can get rid of this and use live result generation for all
rebaseline_server activity.  That depends on its performance being reasonable...

+    """ Handle a GET request for part of the precomputed _SERVER.results object.
 
     Args:
       results_type: string indicating which set of results to return;
             must be one of the results_mod.RESULTS_* constants
     """
-    logging.debug('do_GET_results: sending results of type "%s"' % results_type)
+    logging.debug('do_GET_precomputed_results: sending results of type "%s"' %
+                  results_type)
     # Since we must make multiple calls to the ExpectationComparisons object,
     # grab a reference to it in case it is updated to point at a new
     # ExpectationComparisons object within another thread.
@@ -487,6 +502,26 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       }
     self.send_json_dict(response_dict)
 
+  def do_GET_live_results(self, url_remainder):
+    """ Handle a GET request for live-generated image diff data.
+
+    Args:
+      url_remainder: string indicating which image diffs to generate
+    """
+    # EPOGER: move these definitions to the top of the file?
+    PARAM_ACTUALS_DIR = 'actualsDir'
+    PARAM_EXPECTATIONS_DIR = 'expectationsDir'
+
+    logging.debug('do_GET_live_results: url_remainder="%s"' % url_remainder)
+    param_dict = urlparse.parse_qs(url_remainder)
+    results_obj = compare_rendered_pictures.RenderedPicturesComparisons(
+        actuals_dirs=param_dict[PARAM_ACTUALS_DIR],

[epoger, 2014/07/31 15:18:31]

The live results are nice and flexible, because the caller can pass in arbitrary
actual/expectations directories.  We use the long-lived image_diff_db instance,
so any image diffs which have already been calculated don't need to be
calculated again.

But the first time a bunch of image pairs are diffed, this will take some
time...

+        expectations_dirs=param_dict[PARAM_EXPECTATIONS_DIR],
+        image_diff_db=_SERVER._image_diff_db,
+        diff_base_url='/static/generated-images')
+    self.send_json_dict(results_obj.get_packaged_results_of_type(
+        results_mod.KEY__HEADER__RESULTS_ALL))
+
   def do_GET_static(self, path):
     """ Handle a GET request for a file under STATIC_CONTENTS_SUBDIR .
     Only allow serving of files within STATIC_CONTENTS_SUBDIR that is a
@@ -643,6 +678,12 @@ def main():
                           'actual GM results. If this directory does not '
                           'exist, it will be created. Defaults to %(default)s'),
                     default=DEFAULT_ACTUALS_DIR)
+  parser.add_argument('--boto',
+                    help=('Path to .boto file giving us credentials to access '
+                          'Google Storage buckets. If not specified, we will '
+                          'only be able to access public GS buckets (and thus '
+                          'won\'t be able to download SKP images).'),
+                    default='')
   # TODO(epoger): Before https://codereview.chromium.org/310093003 ,
   # when this tool downloaded the JSON summaries from skia-autogen,
   # it had an --actuals-revision the caller could specify to download
@@ -700,7 +741,7 @@ def main():
                    gm_summaries_bucket=args.gm_summaries_bucket,
                    port=args.port, export=args.export, editable=args.editable,
                    reload_seconds=args.reload, config_pairs=config_pairs,
-                   builder_regex_list=args.builders)
+                   builder_regex_list=args.builders, boto_file_path=args.boto)
   _SERVER.run()