Add support for rebaselining from trybots.

gm/rebaseline_server/server.py

 #!/usr/bin/python
 
 """
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 HTTP server for our HTML rebaseline viewer.
 """
@@ skipping 242 matching lines @@
           _SKP_PLATFORMS[0], _SKP_PLATFORMS[1]))
       file_handle.write('</li>')
 
     file_handle.write('\n</ul></body></html>')
 
 
 class Server(object):
   """ HTTP server for our HTML rebaseline viewer. """
 
   def __init__(self,
+               actuals_source,
                actuals_dir=DEFAULT_ACTUALS_DIR,
                json_filename=DEFAULT_JSON_FILENAME,
-               gm_summaries_bucket=DEFAULT_GM_SUMMARIES_BUCKET,
                port=DEFAULT_PORT, export=False, editable=True,
                reload_seconds=0, config_pairs=None, builder_regex_list=None,
                boto_file_path=None,
                imagediffdb_threads=imagediffdb.DEFAULT_NUM_WORKER_THREADS):
     """
     Args:
+      actuals_source: actuals_source.get_builders() ->
+          {builder:string -> [ bucket:string, path:string, generation:string ]}
+          If None, don't fetch new actual-results files
+          at all, just compare to whatever files are already in actuals_dir
       actuals_dir: directory under which we will check out the latest actual
           GM results
       json_filename: basename of the JSON summary file to load for each builder
-      gm_summaries_bucket: Google Storage bucket to download json_filename
-          files from; if None or '', don't fetch new actual-results files
-          at all, just compare to whatever files are already in actuals_dir
       port: which TCP port to listen on for HTTP requests
       export: whether to allow HTTP clients on other hosts to access this server
       editable: whether HTTP clients are allowed to submit new GM baselines
           (SKP baseline modifications are performed using an entirely different
           mechanism, not affected by this parameter)
       reload_seconds: polling interval with which to check for new results;
           if 0, don't check for new results at all
       config_pairs: List of (string, string) tuples; for each tuple, compare
           actual results of these two configs.  If None or empty,
           don't compare configs at all.
       builder_regex_list: List of regular expressions specifying which builders
           we will process. If None, process all builders.
       boto_file_path: Path to .boto file giving us credentials to access
           Google Storage buckets; if None, we will only be able to access
           public GS buckets.
       imagediffdb_threads: How many threads to spin up within imagediffdb.
     """
+    self._actuals_source = actuals_source
     self._actuals_dir = actuals_dir
     self._json_filename = json_filename
-    self._gm_summaries_bucket = gm_summaries_bucket
     self._port = port
     self._export = export
     self._editable = editable
     self._reload_seconds = reload_seconds
     self._config_pairs = config_pairs or []
     self._builder_regex_list = builder_regex_list
     self.truncate_results = False
 
     if boto_file_path:
       self._gs = gs_utils.GSUtils(boto_file_path=boto_file_path)
@@ skipping 70 matching lines @@
     the same time.
 
     Args:
       invalidate: if True, invalidate self._results immediately upon entry;
                   otherwise, we will let readers see those results until we
                   replace them
     """
     with self.results_rlock:
       if invalidate:
         self._results = None
-      if self._gm_summaries_bucket:
+
+      if self._actuals_source:
         logging.info(
-            'Updating GM result summaries in %s from gm_summaries_bucket %s ...'
-            % (self._actuals_dir, self._gm_summaries_bucket))
+            'Updating GM result summaries in %s from %s ...'
+            % (self._actuals_dir, self._actuals_source.description()))
 
         # Clean out actuals_dir first, in case some builders have gone away
         # since we last ran.
         if os.path.isdir(self._actuals_dir):
           shutil.rmtree(self._actuals_dir)
 
-        # Get the list of builders we care about.
-        all_builders = download_actuals.get_builders_list(
-            summaries_bucket=self._gm_summaries_bucket)
+        # Get the list of actuals we care about.
+        all_actuals = self._actuals_source.get_builders()
+
         if self._builder_regex_list:
           matching_builders = []
-          for builder in all_builders:
+          for builder in all_actuals:
             for regex in self._builder_regex_list:
               if re.match(regex, builder):
                 matching_builders.append(builder)
                 break  # go on to the next builder, no need to try more regexes
         else:
-          matching_builders = all_builders
+          matching_builders = all_actuals.keys()
 
         # Download the JSON file for each builder we care about.
         #
         # TODO(epoger): When this is a large number of builders, we would be
         # better off downloading them in parallel!
         for builder in matching_builders:
           self._gs.download_file(
-              source_bucket=self._gm_summaries_bucket,
-              source_path=posixpath.join(builder, self._json_filename),
+              source_bucket=all_actuals[builder].bucket,
+              source_path=all_actuals[builder].path,
+              source_generation=all_actuals[builder].generation,
               dest_path=os.path.join(self._actuals_dir, builder,
                                      self._json_filename),
               create_subdirs_if_needed=True)
 
       # We only update the expectations dir if the server was run with a
       # nonzero --reload argument; otherwise, we expect the user to maintain
       # her own expectations as she sees fit.
       #
       # Because the Skia repo is hosted using git, and git does not
       # support updating a single directory tree, we have to update the entire
@@ skipping 463 matching lines @@
   parser.add_argument('--editable', action='store_true',
                       help=('Allow HTTP clients to submit new GM baselines; '
                             'SKP baselines can be edited regardless of this '
                             'setting.'))
   parser.add_argument('--export', action='store_true',
                       help=('Instead of only allowing access from HTTP clients '
                             'on localhost, allow HTTP clients on other hosts '
                             'to access this server.  WARNING: doing so will '
                             'allow users on other hosts to modify your '
                             'GM expectations, if combined with --editable.'))
+  parser.add_argument('--rietveld-issue',
+                      help=('Download json_filename files from latest trybot'
+                            'runs on this codereview.chromium.org issue.'
+                            'Overrides --gm-summaries-bucket.'))
   parser.add_argument('--gm-summaries-bucket',
                     help=('Google Cloud Storage bucket to download '
                           'JSON_FILENAME files from. '
                           'Defaults to %(default)s ; if set to '
                           'empty string, just compare to actual-results '
                           'already found in ACTUALS_DIR.'),
                     default=DEFAULT_GM_SUMMARIES_BUCKET)
   parser.add_argument('--json-filename',
                     help=('JSON summary filename to read for each builder; '
                           'defaults to %(default)s.'),
@@ skipping 17 matching lines @@
                       default=imagediffdb.DEFAULT_NUM_WORKER_THREADS)
   parser.add_argument('--truncate', action='store_true',
                       help=('FOR TESTING ONLY: truncate the set of images we '
                             'process, to speed up testing.'))
   args = parser.parse_args()
   if args.compare_configs:
     config_pairs = CONFIG_PAIRS_TO_COMPARE
   else:
     config_pairs = None
 
+  if args.rietveld_issue:
+    actuals_source = download_actuals.RietveldIssueActuals(args.rietveld_issue,
+                                                           args.json_filename)
+  else:
+    actuals_source = download_actuals.TipOfTreeActuals(args.gm_summaries_bucket,
+                                                       args.json_filename)
+
   global _SERVER
-  _SERVER = Server(actuals_dir=args.actuals_dir,
+  _SERVER = Server(actuals_source,
+                   actuals_dir=args.actuals_dir,
                    json_filename=args.json_filename,
-                   gm_summaries_bucket=args.gm_summaries_bucket,
                    port=args.port, export=args.export, editable=args.editable,
                    reload_seconds=args.reload, config_pairs=config_pairs,
                    builder_regex_list=args.builders, boto_file_path=args.boto,
                    imagediffdb_threads=args.threads)
   if args.truncate:
     _SERVER.truncate_results = True
   _SERVER.run()
 
 
 if __name__ == '__main__':
   main()