rebaseline_server: download actual-results.json files from Google Storage, not skia-autogen

gm/rebaseline_server/server.py

Index: gm/rebaseline_server/server.py
diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py
index 40874d603f63aa937cbf58a230e648428d243241..9ffc64b266a09095e48150f2250c57bcd276f5f3 100755
--- a/gm/rebaseline_server/server.py
+++ b/gm/rebaseline_server/server.py
@@ -28,18 +28,12 @@ import urlparse
 
 # Imports from within Skia
 #
-# We need to add the 'tools' directory for svn.py, and the 'gm' directory for
-# gm_json.py .
-# that directory.
-# Make sure that the 'tools' dir is in the PYTHONPATH, but add it at the *end*
+# We need to add the 'gm' directory for gm_json.py .
+# Make sure it is in the PYTHONPATH, but add it at the *end*
 # so any dirs that are already in the PYTHONPATH will be preferred.
 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
 TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
-TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools')
-if TOOLS_DIRECTORY not in sys.path:
-  sys.path.append(TOOLS_DIRECTORY)
-import svn
 if GM_DIRECTORY not in sys.path:
   sys.path.append(GM_DIRECTORY)
 import gm_json
@@ -74,8 +68,7 @@ KEY__EDITS__OLD_RESULTS_HASH = 'oldResultsHash'
 KEY__EDITS__OLD_RESULTS_TYPE = 'oldResultsType'
 
 DEFAULT_ACTUALS_DIR = results_mod.DEFAULT_ACTUALS_DIR
-DEFAULT_ACTUALS_REPO_REVISION = 'HEAD'
-DEFAULT_ACTUALS_REPO_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
+DEFAULT_ACTUALS_GS_ROOT = 'gs://chromium-skia-gm-summaries'

[rmistry, 2014/05/30 14:15:28]

Should we add this to global_variables.json? I believe gs://chromium-skia-gm is
there as well.

 DEFAULT_PORT = 8888
 
 # Directory, relative to PARENT_DIRECTORY, within which the server will serve
@@ -139,24 +132,6 @@ def _get_routable_ip_address():
   return host
 
 
-def _create_svn_checkout(dir_path, repo_url):
-  """Creates local checkout of an SVN repository at the specified directory
-  path, returning an svn.Svn object referring to the local checkout.
-
-  Args:
-    dir_path: path to the local checkout; if this directory does not yet exist,
-              it will be created and the repo will be checked out into it
-    repo_url: URL of SVN repo to check out into dir_path (unless the local
-              checkout already exists)
-  Returns: an svn.Svn object referring to the local checkout.
-  """
-  local_checkout = svn.Svn(dir_path)
-  if not os.path.isdir(dir_path):
-    os.makedirs(dir_path)
-    local_checkout.Checkout(repo_url, '.')
-  return local_checkout
-
-
 def _create_index(file_path, config_pairs):
   """Creates an index file linking to all results available from this server.
 
@@ -213,17 +188,15 @@ class Server(object):
 
   def __init__(self,
                actuals_dir=DEFAULT_ACTUALS_DIR,
-               actuals_repo_revision=DEFAULT_ACTUALS_REPO_REVISION,
-               actuals_repo_url=DEFAULT_ACTUALS_REPO_URL,
+               actuals_gs_root=DEFAULT_ACTUALS_GS_ROOT,
                port=DEFAULT_PORT, export=False, editable=True,
                reload_seconds=0, config_pairs=None, builder_regex_list=None):
     """
     Args:
-      actuals_dir: directory under which we will check out the latest actual
+      actuals_dir: directory into which we will download the latest actual

[rmistry, 2014/05/30 14:15:28]

[optional] since we are not going to be dealing with remove GS directories as
well should we should specifying "local" to differenciate (atleast in the param
description):

actuals_dir: local directory into which we will download the latest actual GM
results

[rmistry, 2014/05/30 14:16:27]

So many typos:
"since we are going to be dealing with remote GS directories as well"

           GM results
-      actuals_repo_revision: revision of actual-results.json files to process
-      actuals_repo_url: SVN repo to download actual-results.json files from;
-          if None or '', don't fetch new actual-results files at all,
+      actuals_gs_root: Google Storage root to download actual-results.json files
+          from; if None or '', don't fetch new actual-results files at all,
           just compare to whatever files are already in actuals_dir
       port: which TCP port to listen on for HTTP requests
       export: whether to allow HTTP clients on other hosts to access this server
@@ -237,8 +210,7 @@ class Server(object):
           we will process. If None, process all builders.
     """
     self._actuals_dir = actuals_dir
-    self._actuals_repo_revision = actuals_repo_revision
-    self._actuals_repo_url = actuals_repo_url
+    self._actuals_gs_root = actuals_gs_root
     self._port = port
     self._export = export
     self._editable = editable
@@ -250,9 +222,6 @@ class Server(object):
             PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR,
             "index.html"),
         config_pairs=config_pairs)
-    if actuals_repo_url:
-      self._actuals_repo = _create_svn_checkout(
-          dir_path=actuals_dir, repo_url=actuals_repo_url)
 
     # Reentrant lock that must be held whenever updating EITHER of:
     # 1. self._results
@@ -300,14 +269,17 @@ class Server(object):
     with self.results_rlock:
       if invalidate:
         self._results = None
-      if self._actuals_repo_url:
-        logging.info(
-            'Updating actual GM results in %s to revision %s from repo %s ...'
-            % (
-                self._actuals_dir, self._actuals_repo_revision,
-                self._actuals_repo_url))
-        self._actuals_repo.Update(
-            path='.', revision=self._actuals_repo_revision)
+      if self._actuals_gs_root:
+        logging.info('Updating actual GM results in %s from %s ...' % (
+            self._actuals_dir, self._actuals_gs_root))
+        if os.path.isdir(self._actuals_dir):
+          shutil.rmtree(self._actuals_dir)
+        os.makedirs(self._actuals_dir)
+        # EPOGER: what if the user does not have gsutil installed in her $PATH?

[epoger, 2014/05/29 21:19:57]

Any thoughts on what to do about this?

I had originally intended to download the JSON files via HTTP (like how we
download the images).  The problem: I am not aware of a simple way to download a
whole directory full of files via HTTP.  You have to know the exact filenames
you are looking for, and download each one by its full URL.

Maybe there is some way to download all of gs://chromium-skia-gm-summaries via
HTTP?

Or maybe we should add gsutil to our DEPS?

[rmistry, 2014/05/30 14:15:28]

Maybe add a check at the top to make sure that there is a gsutil in the PATH, if
not then we can echo instructions on downloading and setting up gsutil.
Also, having gsutil in PATH will not garantee that appropriate permissions
exist, might want to do a basic check after finding gsutil in PATH to ensure
that the right credentials / permissions exist.
No idea about this. But I am not surprised that there is no way to download a
folder via HTTP.
Even if we did this the user would still need an appropriate .boto file or would
need to be authenticated via gsutil.

+        _run_command(
+            args=['gsutil', '-m', 'cp', '-R',
+                  posixpath.join(self._actuals_gs_root, '*'), '.'],
+            directory=self._actuals_dir)
 
       # We only update the expectations dir if the server was run with a
       # nonzero --reload argument; otherwise, we expect the user to maintain
@@ -617,22 +589,16 @@ def main():
                       level=logging.INFO)
   parser = argparse.ArgumentParser()
   parser.add_argument('--actuals-dir',
-                    help=('Directory into which we will check out the latest '
+                    help=('Directory into which we will download the latest '
                           'actual GM results. If this directory does not '
                           'exist, it will be created. Defaults to %(default)s'),
                     default=DEFAULT_ACTUALS_DIR)
-  parser.add_argument('--actuals-repo',
-                    help=('URL of SVN repo to download actual-results.json '
+  parser.add_argument('--actuals-gs-root',
+                    help=('Google Storage root to download actual-results.json '
                           'files from. Defaults to %(default)s ; if set to '
                           'empty string, just compare to actual-results '
                           'already found in ACTUALS_DIR.'),
-                    default=DEFAULT_ACTUALS_REPO_URL)
-  parser.add_argument('--actuals-revision',
-                    help=('revision of actual-results.json files to process. '
-                          'Defaults to %(default)s .  Beware of setting this '
-                          'argument in conjunction with --editable; you '
-                          'probably only want to edit results at HEAD.'),
-                    default=DEFAULT_ACTUALS_REPO_REVISION)
+                    default=DEFAULT_ACTUALS_GS_ROOT)
   parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+',
                       help=('Only process builders matching these regular '
                             'expressions.  If unspecified, process all '
@@ -657,8 +623,8 @@ def main():
   parser.add_argument('--reload', type=int,
                       help=('How often (a period in seconds) to update the '
                             'results.  If specified, both expected and actual '
-                            'results will be updated by running "gclient sync" '
-                            'on your Skia checkout as a whole.  '
+                            'results will be updated, and "gclient sync" will '
+                            'be run on your Skia checkout as a whole.  '
                             'By default, we do not reload at all, and you '
                             'must restart the server to pick up new data.'),
                       default=0)
@@ -670,8 +636,7 @@ def main():
 
   global _SERVER
   _SERVER = Server(actuals_dir=args.actuals_dir,
-                   actuals_repo_revision=args.actuals_revision,
-                   actuals_repo_url=args.actuals_repo,
+                   actuals_gs_root=args.actuals_gs_root,
                    port=args.port, export=args.export, editable=args.editable,
                    reload_seconds=args.reload, config_pairs=config_pairs,
                    builder_regex_list=args.builders)