teach rebaseline_server to generate diffs of rendered SKPs

gm/rebaseline_server/server.py

Index: gm/rebaseline_server/server.py
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py
index 50367f6f35eab16fc045b4d4433db01e6adb145b..6271f97dd2fa97dbd0da68456b24f5f1fd255e57 100755
--- a/gm/rebaseline_server/server.py
+++ b/gm/rebaseline_server/server.py
@@ -40,6 +40,7 @@ import gm_json
 # https://codereview.chromium.org/195943004/diff/1/gm/rebaseline_server/server.py#newcode44
 # pylint: enable=C0301
 import compare_configs
+import compare_rendered_pictures
 import compare_to_expectations
 import download_actuals
 import imagediffdb
@@ -73,8 +74,11 @@ DEFAULT_PORT = 8888
 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(PARENT_DIRECTORY))
 # Directory, relative to PARENT_DIRECTORY, within which the server will serve
-# out live results (not static files).
-RESULTS_SUBDIR = 'results'
+# out image diff data from within the precomputed _SERVER.results .
+PRECOMPUTED_RESULTS_SUBDIR = 'results'
+# Directory, relative to PARENT_DIRECTORY, within which the server will serve
+# out live-generated image diff data.
+LIVE_RESULTS_SUBDIR = 'live-results'
 # Directory, relative to PARENT_DIRECTORY, within which the server will serve
 # out static files.
 STATIC_CONTENTS_SUBDIR = 'static'
@@ -83,6 +87,10 @@ GENERATED_HTML_SUBDIR = 'generated-html'
 GENERATED_IMAGES_SUBDIR = 'generated-images'
 GENERATED_JSON_SUBDIR = 'generated-json'
 
+# Parameters we use within do_GET_live_results()
+LIVE_PARAM__SET_A_DIR = 'setADir'
+LIVE_PARAM__SET_B_DIR = 'setBDir'
+
 # How often (in seconds) clients should reload while waiting for initial
 # results to load.
 RELOAD_INTERVAL_UNTIL_READY = 10
@@ -165,7 +173,7 @@ def _create_index(file_path, config_pairs):
             '<li><a href="/{static_subdir}/view.html#/view.html?'
             'resultsToLoad=/{results_subdir}/{summary_type}">'
             '{summary_type}</a></li>'.format(
-                results_subdir=RESULTS_SUBDIR,
+                results_subdir=PRECOMPUTED_RESULTS_SUBDIR,
                 static_subdir=STATIC_CONTENTS_SUBDIR,
                 summary_type=summary_type))
       file_handle.write('</ul>')
@@ -193,7 +201,9 @@ class Server(object):
                json_filename=DEFAULT_JSON_FILENAME,
                gm_summaries_bucket=DEFAULT_GM_SUMMARIES_BUCKET,
                port=DEFAULT_PORT, export=False, editable=True,
-               reload_seconds=0, config_pairs=None, builder_regex_list=None):
+               reload_seconds=0, config_pairs=None, builder_regex_list=None,
+               boto_file_path=None,
+               imagediffdb_threads=imagediffdb.DEFAULT_NUM_WORKER_THREADS):
     """
     Args:
       actuals_dir: directory under which we will check out the latest actual
@@ -212,6 +222,10 @@ class Server(object):
           don't compare configs at all.
       builder_regex_list: List of regular expressions specifying which builders
           we will process. If None, process all builders.
+      boto_file_path: Path to .boto file giving us credentials to access
+          Google Storage buckets; if None, we will only be able to access
+          public GS buckets.
+      imagediffdb_threads: How many threads to spin up within imagediffdb.
     """
     self._actuals_dir = actuals_dir
     self._json_filename = json_filename
@@ -222,7 +236,13 @@ class Server(object):
     self._reload_seconds = reload_seconds
     self._config_pairs = config_pairs or []
     self._builder_regex_list = builder_regex_list
-    self._gs = gs_utils.GSUtils()
+    self.truncate_results = False
+
+    if boto_file_path:
+      self._gs = gs_utils.GSUtils(boto_file_path=boto_file_path)
+    else:
+      self._gs = gs_utils.GSUtils()
+
     _create_index(
         file_path=os.path.join(
             PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR,
@@ -234,9 +254,16 @@ class Server(object):
     # 2. the expected or actual results on local disk
     self.results_rlock = threading.RLock()
 
-    # These will be filled in by calls to update_results()
+    # Create a single ImageDiffDB instance that is used by all our differs.
+    self._image_diff_db = imagediffdb.ImageDiffDB(
+        gs=self._gs,
+        storage_root=os.path.join(
+            PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
+            GENERATED_IMAGES_SUBDIR),
+        num_worker_threads=imagediffdb_threads)
+
+    # This will be filled in by calls to update_results()
     self._results = None
-    self._image_diff_db = None
 
   @property
   def results(self):
@@ -245,6 +272,16 @@ class Server(object):
     return self._results
 
   @property
+  def image_diff_db(self):
+    """ Returns reference to our ImageDiffDB object."""
+    return self._image_diff_db
+
+  @property
+  def gs(self):
+    """ Returns reference to our GSUtils object."""
+    return self._gs
+
+  @property
   def is_exported(self):
     """ Returns true iff HTTP clients on other hosts are allowed to access
     this server. """
@@ -343,12 +380,6 @@ class Server(object):
             compare_to_expectations.DEFAULT_EXPECTATIONS_DIR)
         _run_command(['gclient', 'sync'], TRUNK_DIRECTORY)
 
-      if not self._image_diff_db:
-        self._image_diff_db = imagediffdb.ImageDiffDB(
-            storage_root=os.path.join(
-              PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR,
-              GENERATED_IMAGES_SUBDIR))
-
       self._results = compare_to_expectations.ExpectationComparisons(
           image_diff_db=self._image_diff_db,
           actuals_root=self._actuals_dir,
@@ -443,7 +474,8 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       normpath = posixpath.normpath(self.path)
       (dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups()
       dispatchers = {
-          RESULTS_SUBDIR: self.do_GET_results,
+          PRECOMPUTED_RESULTS_SUBDIR: self.do_GET_precomputed_results,
+          LIVE_RESULTS_SUBDIR: self.do_GET_live_results,
           STATIC_CONTENTS_SUBDIR: self.do_GET_static,
       }
       dispatcher = dispatchers[dispatcher_name]
@@ -452,14 +484,15 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       self.send_error(404)
       raise
 
-  def do_GET_results(self, results_type):
-    """ Handle a GET request for GM results.
+  def do_GET_precomputed_results(self, results_type):
+    """ Handle a GET request for part of the precomputed _SERVER.results object.
 
     Args:
       results_type: string indicating which set of results to return;
             must be one of the results_mod.RESULTS_* constants
     """
-    logging.debug('do_GET_results: sending results of type "%s"' % results_type)
+    logging.debug('do_GET_precomputed_results: sending results of type "%s"' %
+                  results_type)
     # Since we must make multiple calls to the ExpectationComparisons object,
     # grab a reference to it in case it is updated to point at a new
     # ExpectationComparisons object within another thread.
@@ -487,6 +520,23 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
       }
     self.send_json_dict(response_dict)
 
+  def do_GET_live_results(self, url_remainder):
+    """ Handle a GET request for live-generated image diff data.
+
+    Args:
+      url_remainder: string indicating which image diffs to generate
+    """
+    logging.debug('do_GET_live_results: url_remainder="%s"' % url_remainder)
+    param_dict = urlparse.parse_qs(url_remainder)
+    results_obj = compare_rendered_pictures.RenderedPicturesComparisons(
+        setA_dirs=param_dict[LIVE_PARAM__SET_A_DIR],
+        setB_dirs=param_dict[LIVE_PARAM__SET_B_DIR],
+        image_diff_db=_SERVER.image_diff_db,
+        diff_base_url='/static/generated-images',
+        gs=_SERVER.gs, truncate_results=_SERVER.truncate_results)
+    self.send_json_dict(results_obj.get_packaged_results_of_type(
+        results_mod.KEY__HEADER__RESULTS_ALL))
+
   def do_GET_static(self, path):
     """ Handle a GET request for a file under STATIC_CONTENTS_SUBDIR .
     Only allow serving of files within STATIC_CONTENTS_SUBDIR that is a
@@ -643,6 +693,12 @@ def main():
                           'actual GM results. If this directory does not '
                           'exist, it will be created. Defaults to %(default)s'),
                     default=DEFAULT_ACTUALS_DIR)
+  parser.add_argument('--boto',
+                    help=('Path to .boto file giving us credentials to access '
+                          'Google Storage buckets. If not specified, we will '
+                          'only be able to access public GS buckets (and thus '
+                          'won\'t be able to download SKP images).'),
+                    default='')
   # TODO(epoger): Before https://codereview.chromium.org/310093003 ,
   # when this tool downloaded the JSON summaries from skia-autogen,
   # it had an --actuals-revision the caller could specify to download
@@ -688,6 +744,14 @@ def main():
                             'By default, we do not reload at all, and you '
                             'must restart the server to pick up new data.'),
                       default=0)
+  parser.add_argument('--threads', type=int,
+                      help=('How many parallel threads we use to download '
+                            'images and generate diffs; defaults to '
+                            '%(default)s'),
+                      default=imagediffdb.DEFAULT_NUM_WORKER_THREADS)
+  parser.add_argument('--truncate', action='store_true',
+                      help=('FOR TESTING ONLY: truncate the set of images we '
+                            'process, to speed up testing.'))
   args = parser.parse_args()
   if args.compare_configs:
     config_pairs = CONFIG_PAIRS_TO_COMPARE
@@ -700,7 +764,10 @@ def main():
                    gm_summaries_bucket=args.gm_summaries_bucket,
                    port=args.port, export=args.export, editable=args.editable,
                    reload_seconds=args.reload, config_pairs=config_pairs,
-                   builder_regex_list=args.builders)
+                   builder_regex_list=args.builders, boto_file_path=args.boto,
+                   imagediffdb_threads=args.threads)
+  if args.truncate:
+    _SERVER.truncate_results = True
   _SERVER.run()