rebaseline_server: download actual-results.json files from Google Storage, not skia-autogen

gm/rebaseline_server/server.py

 #!/usr/bin/python
 
 """
 Copyright 2013 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 HTTP server for our HTML rebaseline viewer.
 """
@@ skipping 10 matching lines @@
 import socket
 import subprocess
 import sys
 import thread
 import threading
 import time
 import urlparse
 
 # Imports from within Skia
 #
-# We need to add the 'tools' directory for svn.py, and the 'gm' directory for
-# gm_json.py .
-# that directory.
-# Make sure that the 'tools' dir is in the PYTHONPATH, but add it at the *end*
+# We need to add the 'gm' directory for gm_json.py .
+# Make sure it is in the PYTHONPATH, but add it at the *end*
 # so any dirs that are already in the PYTHONPATH will be preferred.
 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
-TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools')
-if TOOLS_DIRECTORY not in sys.path:
-  sys.path.append(TOOLS_DIRECTORY)
-import svn
 if GM_DIRECTORY not in sys.path:
   sys.path.append(GM_DIRECTORY)
 import gm_json
 
 # Imports from local dir
 #
 # Note: we import results under a different name, to avoid confusion with the
 # Server.results() property. See discussion at
 # https://codereview.chromium.org/195943004/diff/1/gm/rebaseline_server/server.py#newcode44
 import compare_configs
@@ skipping 14 matching lines @@
                  'json': 'application/json'
                  }
 
 # Keys that server.py uses to create the toplevel content header.
 # NOTE: Keep these in sync with static/constants.js
 KEY__EDITS__MODIFICATIONS = 'modifications'
 KEY__EDITS__OLD_RESULTS_HASH = 'oldResultsHash'
 KEY__EDITS__OLD_RESULTS_TYPE = 'oldResultsType'
 
 DEFAULT_ACTUALS_DIR = results_mod.DEFAULT_ACTUALS_DIR
-DEFAULT_ACTUALS_REPO_REVISION = 'HEAD'
-DEFAULT_ACTUALS_REPO_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
+DEFAULT_ACTUALS_GS_ROOT = 'gs://chromium-skia-gm-summaries'

[rmistry, 2014/05/30 14:15:28]

Should we add this to global_variables.json? I believe gs://chromium-skia-gm is
there as well.

 DEFAULT_PORT = 8888
 
 # Directory, relative to PARENT_DIRECTORY, within which the server will serve
 # out live results (not static files).
 RESULTS_SUBDIR = 'results'
 # Directory, relative to PARENT_DIRECTORY, within which the server will serve
 # out static files.
 STATIC_CONTENTS_SUBDIR = 'static'
 # All of the GENERATED_*_SUBDIRS are relative to STATIC_CONTENTS_SUBDIR
 GENERATED_HTML_SUBDIR = 'generated-html'
@@ skipping 43 matching lines @@
   """Returns routable IP address of this host (the IP address of its network
      interface that would be used for most traffic, not its localhost
      interface).  See http://stackoverflow.com/a/166589 """
   sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
   sock.connect(('8.8.8.8', 80))
   host = sock.getsockname()[0]
   sock.close()
   return host
 
 
-def _create_svn_checkout(dir_path, repo_url):
-  """Creates local checkout of an SVN repository at the specified directory
-  path, returning an svn.Svn object referring to the local checkout.
-
-  Args:
-    dir_path: path to the local checkout; if this directory does not yet exist,
-              it will be created and the repo will be checked out into it
-    repo_url: URL of SVN repo to check out into dir_path (unless the local
-              checkout already exists)
-  Returns: an svn.Svn object referring to the local checkout.
-  """
-  local_checkout = svn.Svn(dir_path)
-  if not os.path.isdir(dir_path):
-    os.makedirs(dir_path)
-    local_checkout.Checkout(repo_url, '.')
-  return local_checkout
-
-
 def _create_index(file_path, config_pairs):
   """Creates an index file linking to all results available from this server.
 
   Prior to https://codereview.chromium.org/215503002 , we had a static
   index.html within our repo.  But now that the results may or may not include
   config comparisons, index.html needs to be generated differently depending
   on which results are included.
 
   TODO(epoger): Instead of including raw HTML within the Python code,
   consider restoring the index.html file as a template and using django (or
@@ skipping 36 matching lines @@
         file_handle.write('</li>')
       file_handle.write('</ul>')
     file_handle.write('</ul></body></html>')
 
 
 class Server(object):
   """ HTTP server for our HTML rebaseline viewer. """
 
   def __init__(self,
                actuals_dir=DEFAULT_ACTUALS_DIR,
-               actuals_repo_revision=DEFAULT_ACTUALS_REPO_REVISION,
-               actuals_repo_url=DEFAULT_ACTUALS_REPO_URL,
+               actuals_gs_root=DEFAULT_ACTUALS_GS_ROOT,
                port=DEFAULT_PORT, export=False, editable=True,
                reload_seconds=0, config_pairs=None, builder_regex_list=None):
     """
     Args:
-      actuals_dir: directory under which we will check out the latest actual
+      actuals_dir: directory into which we will download the latest actual

[rmistry, 2014/05/30 14:15:28]

[optional] since we are not going to be dealing with remove GS directories as
well should we should specifying "local" to differenciate (atleast in the param
description):

actuals_dir: local directory into which we will download the latest actual GM
results

[rmistry, 2014/05/30 14:16:27]

So many typos:
"since we are going to be dealing with remote GS directories as well"

           GM results
-      actuals_repo_revision: revision of actual-results.json files to process
-      actuals_repo_url: SVN repo to download actual-results.json files from;
-          if None or '', don't fetch new actual-results files at all,
+      actuals_gs_root: Google Storage root to download actual-results.json files
+          from; if None or '', don't fetch new actual-results files at all,
           just compare to whatever files are already in actuals_dir
       port: which TCP port to listen on for HTTP requests
       export: whether to allow HTTP clients on other hosts to access this server
       editable: whether HTTP clients are allowed to submit new baselines
       reload_seconds: polling interval with which to check for new results;
           if 0, don't check for new results at all
       config_pairs: List of (string, string) tuples; for each tuple, compare
           actual results of these two configs.  If None or empty,
           don't compare configs at all.
       builder_regex_list: List of regular expressions specifying which builders
           we will process. If None, process all builders.
     """
     self._actuals_dir = actuals_dir
-    self._actuals_repo_revision = actuals_repo_revision
-    self._actuals_repo_url = actuals_repo_url
+    self._actuals_gs_root = actuals_gs_root
     self._port = port
     self._export = export
     self._editable = editable
     self._reload_seconds = reload_seconds
     self._config_pairs = config_pairs or []
     self._builder_regex_list = builder_regex_list
     _create_index(
         file_path=os.path.join(
             PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR,
             "index.html"),
         config_pairs=config_pairs)
-    if actuals_repo_url:
-      self._actuals_repo = _create_svn_checkout(
-          dir_path=actuals_dir, repo_url=actuals_repo_url)
 
     # Reentrant lock that must be held whenever updating EITHER of:
     # 1. self._results
     # 2. the expected or actual results on local disk
     self.results_rlock = threading.RLock()
     # self._results will be filled in by calls to update_results()
     self._results = None
 
   @property
   def results(self):
@@ skipping 27 matching lines @@
     the same time.
 
     Args:
       invalidate: if True, invalidate self._results immediately upon entry;
                   otherwise, we will let readers see those results until we
                   replace them
     """
     with self.results_rlock:
       if invalidate:
         self._results = None
-      if self._actuals_repo_url:
-        logging.info(
-            'Updating actual GM results in %s to revision %s from repo %s ...'
-            % (
-                self._actuals_dir, self._actuals_repo_revision,
-                self._actuals_repo_url))
-        self._actuals_repo.Update(
-            path='.', revision=self._actuals_repo_revision)
+      if self._actuals_gs_root:
+        logging.info('Updating actual GM results in %s from %s ...' % (
+            self._actuals_dir, self._actuals_gs_root))
+        if os.path.isdir(self._actuals_dir):
+          shutil.rmtree(self._actuals_dir)
+        os.makedirs(self._actuals_dir)
+        # EPOGER: what if the user does not have gsutil installed in her $PATH?

[epoger, 2014/05/29 21:19:57]

Any thoughts on what to do about this?

I had originally intended to download the JSON files via HTTP (like how we
download the images).  The problem: I am not aware of a simple way to download a
whole directory full of files via HTTP.  You have to know the exact filenames
you are looking for, and download each one by its full URL.

Maybe there is some way to download all of gs://chromium-skia-gm-summaries via
HTTP?

Or maybe we should add gsutil to our DEPS?

[rmistry, 2014/05/30 14:15:28]

Maybe add a check at the top to make sure that there is a gsutil in the PATH, if
not then we can echo instructions on downloading and setting up gsutil.
Also, having gsutil in PATH will not garantee that appropriate permissions
exist, might want to do a basic check after finding gsutil in PATH to ensure
that the right credentials / permissions exist.
No idea about this. But I am not surprised that there is no way to download a
folder via HTTP.
Even if we did this the user would still need an appropriate .boto file or would
need to be authenticated via gsutil.

+        _run_command(
+            args=['gsutil', '-m', 'cp', '-R',
+                  posixpath.join(self._actuals_gs_root, '*'), '.'],
+            directory=self._actuals_dir)
 
       # We only update the expectations dir if the server was run with a
       # nonzero --reload argument; otherwise, we expect the user to maintain
       # her own expectations as she sees fit.
       #
       # Because the Skia repo is moving from SVN to git, and git does not
       # support updating a single directory tree, we have to update the entire
       # repo checkout.
       #
       # Because Skia uses depot_tools, we have to update using "gclient sync"
@@ skipping 289 matching lines @@
     self.end_headers()
     json.dump(json_dict, self.wfile)
 
 
 def main():
   logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                       datefmt='%m/%d/%Y %H:%M:%S',
                       level=logging.INFO)
   parser = argparse.ArgumentParser()
   parser.add_argument('--actuals-dir',
-                    help=('Directory into which we will check out the latest '
+                    help=('Directory into which we will download the latest '
                           'actual GM results. If this directory does not '
                           'exist, it will be created. Defaults to %(default)s'),
                     default=DEFAULT_ACTUALS_DIR)
-  parser.add_argument('--actuals-repo',
-                    help=('URL of SVN repo to download actual-results.json '
+  parser.add_argument('--actuals-gs-root',
+                    help=('Google Storage root to download actual-results.json '
                           'files from. Defaults to %(default)s ; if set to '
                           'empty string, just compare to actual-results '
                           'already found in ACTUALS_DIR.'),
-                    default=DEFAULT_ACTUALS_REPO_URL)
-  parser.add_argument('--actuals-revision',
-                    help=('revision of actual-results.json files to process. '
-                          'Defaults to %(default)s .  Beware of setting this '
-                          'argument in conjunction with --editable; you '
-                          'probably only want to edit results at HEAD.'),
-                    default=DEFAULT_ACTUALS_REPO_REVISION)
+                    default=DEFAULT_ACTUALS_GS_ROOT)
   parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+',
                       help=('Only process builders matching these regular '
                             'expressions.  If unspecified, process all '
                             'builders.'))
   parser.add_argument('--compare-configs', action='store_true',
                       help=('In addition to generating differences between '
                             'expectations and actuals, also generate '
                             'differences between these config pairs: '
                             + str(CONFIG_PAIRS_TO_COMPARE)))
   parser.add_argument('--editable', action='store_true',
                       help=('Allow HTTP clients to submit new baselines.'))
   parser.add_argument('--export', action='store_true',
                       help=('Instead of only allowing access from HTTP clients '
                             'on localhost, allow HTTP clients on other hosts '
                             'to access this server.  WARNING: doing so will '
                             'allow users on other hosts to modify your '
                             'GM expectations, if combined with --editable.'))
   parser.add_argument('--port', type=int,
                       help=('Which TCP port to listen on for HTTP requests; '
                             'defaults to %(default)s'),
                       default=DEFAULT_PORT)
   parser.add_argument('--reload', type=int,
                       help=('How often (a period in seconds) to update the '
                             'results.  If specified, both expected and actual '
-                            'results will be updated by running "gclient sync" '
-                            'on your Skia checkout as a whole.  '
+                            'results will be updated, and "gclient sync" will '
+                            'be run on your Skia checkout as a whole.  '
                             'By default, we do not reload at all, and you '
                             'must restart the server to pick up new data.'),
                       default=0)
   args = parser.parse_args()
   if args.compare_configs:
     config_pairs = CONFIG_PAIRS_TO_COMPARE
   else:
     config_pairs = None
 
   global _SERVER
   _SERVER = Server(actuals_dir=args.actuals_dir,
-                   actuals_repo_revision=args.actuals_revision,
-                   actuals_repo_url=args.actuals_repo,
+                   actuals_gs_root=args.actuals_gs_root,
                    port=args.port, export=args.export, editable=args.editable,
                    reload_seconds=args.reload, config_pairs=config_pairs,
                    builder_regex_list=args.builders)
   _SERVER.run()
 
 
 if __name__ == '__main__':
   main()