download_actuals.py: download JSON files from Google Storage instead of skia-autogen

gm/rebaseline_server/download_actuals.py

 #!/usr/bin/python
 
 """
 Copyright 2014 Google Inc.
 
 Use of this source code is governed by a BSD-style license that can be
 found in the LICENSE file.
 
 Download actual GM results for a particular builder.
 """
@@ skipping 23 matching lines @@
 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 GM_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'gm')
 TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools')
 if GM_DIRECTORY not in sys.path:
   sys.path.append(GM_DIRECTORY)
 if TOOLS_DIRECTORY not in sys.path:
   sys.path.append(TOOLS_DIRECTORY)
 import buildbot_globals
 import gm_json
 
-DEFAULT_ACTUALS_BASE_URL = posixpath.join(
-    buildbot_globals.Get('autogen_svn_url'), 'gm-actual')
+# Imports from third-party code
+APICLIENT_DIRECTORY = os.path.join(
+    TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client')
+if APICLIENT_DIRECTORY not in sys.path:
+  sys.path.append(APICLIENT_DIRECTORY)
+from googleapiclient.discovery import build as build_service
+
+
+GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket')
+DEFAULT_ACTUALS_BASE_URL = (
+    'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET)
 DEFAULT_JSON_FILENAME = 'actual-results.json'
 
 
 class Download(object):
 
   def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL,
                json_filename=DEFAULT_JSON_FILENAME,
                gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL):
     """
     Args:
@@ skipping 33 matching lines @@
       results_of_this_type = actual_results_dict[result_type]
       if not results_of_this_type:
         continue
       for image_name in sorted(results_of_this_type.keys()):
         (test, config) = self._image_filename_re.match(image_name).groups()
         (hash_type, hash_digest) = results_of_this_type[image_name]
         source_url = gm_json.CreateGmActualUrl(
             test_name=test, hash_type=hash_type, hash_digest=hash_digest,
             gm_actuals_root_url=self._gm_actuals_root_url)
         dest_path = os.path.join(dest_dir, config, test + '.png')
+        # TODO(epoger): To speed this up, we should only download files that
+        # we don't already have on local disk.
         copy_contents(source_url=source_url, dest_path=dest_path,
                       create_subdirs_if_needed=True)
 
 
 def create_filepath_url(filepath):
   """ Returns a file:/// URL pointing at the given filepath on local disk.
 
   For now, this is only used by unittests, but I anticipate it being useful
   in production, as a way for developers to run rebaseline_server over locally
   generated images.
@@ skipping 35 matching lines @@
   """
   if create_subdirs_if_needed:
     dest_dir = os.path.dirname(dest_path)
     if not os.path.exists(dest_dir):
       os.makedirs(dest_dir)
   with contextlib.closing(urllib.urlopen(source_url)) as source_handle:
     with open(dest_path, 'wb') as dest_handle:
       shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle)
 
 
+def gcs_list_bucket_contents(bucket, subdir=None):

[epoger, 2014/05/30 22:11:36]

I plan to use this to replace the use of the gsutil binary in server.py, solving
the problem we discussed in
https://codereview.chromium.org/303223006/diff/1/gm/rebaseline_server/server....
.

The GM summary dirs in Google Storage are publicly readable, so you don't even
need any .boto credentials to use the API.

[benchen, 2014/05/30 23:45:03]

This looks promising. If we use the API for restricted ops, it looks like the
.boto auth will be replaced by a client_secrets.json file
https://developers.google.com/api-client-library/python/samples/samples#Cloud...

On 2014/05/30 22:11:36, epoger wrote:

+  """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple.
+
+  Args:
+    bucket: name of the Google Storage bucket
+    subdir: directory within the bucket to list, or None for root directory
+  """
+  if subdir:
+    if not subdir.endswith('/'):
+      subdir += '/'

[rmistry, 2014/06/03 12:23:39]

Can replace 173 and 174 with:
posixpath.join(subdir, '')
and could do:
subdir_length = len(subdir) if subdir else 0

This changes 6 lines into 3 lines:
if subdir:
  posixpath.join(subdir, '')
subdir_length = len(subdir) if subdir else 0

[epoger, 2014/06/03 15:27:52]

Took half of the suggestion, thanks.  I would prefer to be clear about exactly
what the trailing slash is about, so kept it explicit and added a comment.

+    subdir_length = len(subdir)
+  else:
+    subdir_length = 0
+
+  storage = build_service('storage', 'v1')

[rmistry, 2014/06/03 12:23:39]

google-api-python-client is cool, I wonder who wrote it (though
google-api-java-client is cooler).

[epoger, 2014/06/03 15:27:52]

I heard that the guy who wrote google-api-java-client is a bit unpredictable. 
Rumor has it they won't even let him eat solid food.

+  command = storage.objects().list(
+      bucket=bucket, delimiter='/', fields='items(name),prefixes',

[rmistry, 2014/06/03 12:23:39]

Can also use posixpath.sep instead of '/' though I am not sure which is better.

[epoger, 2014/06/03 15:27:52]

I think it's 50/50, so went with the shorter one.  I am happy to change it if
you prefer.

+      prefix=subdir)
+  results = command.execute()
+  dirs = [item[subdir_length:-1] for item in results.get('prefixes', [])]
+  files = [item['name'][subdir_length:] for item in results.get('items', [])]

[rmistry, 2014/06/03 12:23:39]

Lines 184 and 185 are a little confusing could you add some documentation or
break up the lines to make them more self-explanatory?

[epoger, 2014/06/03 15:27:52]

You're right.  Do you think the comments are sufficient?  Or do you think I
should also break up the list comprehension into more comprehensible code?

+  return (dirs, files)
+
+
 def main():
   parser = optparse.OptionParser()
   required_params = []
   parser.add_option('--actuals-base-url',
                     action='store', type='string',
                     default=DEFAULT_ACTUALS_BASE_URL,
                     help=('Base URL from which to read files containing JSON '
                           'summaries of actual GM results; defaults to '
-                          '"%default". To get a specific revision (useful for '
-                          'trybots) replace "svn" with "svn-history/r123".'))
-  # TODO(epoger): Rather than telling the user to run "svn ls" to get the list
-  # of builders, add a --list-builders option that will print the list.
+                          '"%default".'))

[epoger, 2014/05/30 22:11:36]

Unfortunately, this tool has lost the ability to read old revisions of the JSON
file.  Since we use versioned storage, that information is available for
download... but I haven't figured out how the user should specify the revision
to retrieve.

[benchen, 2014/05/30 23:45:03]

Joe must know the solution in details.
On 2014/05/30 22:11:36, epoger wrote:

[rmistry, 2014/06/03 12:23:39]

Or Joe will know who to ask from the Cloud Storage API team.

[epoger, 2014/06/03 15:27:52]

Sorry, I wasn't clear.  It isn't a Cloud Storage API question.  It's a matter of
figuring out which timestamped version of the actual-results.json file we want
to retrieve from Google Storage.

   required_params.append('builder')
   parser.add_option('--builder',
                     action='store', type='string',
                     help=('REQUIRED: Which builder to download results for. '
-                          'To see a list of builders, run "svn ls %s".' %
-                          DEFAULT_ACTUALS_BASE_URL))
+                          'To see a list of builders, run with the '
+                          '--list-builders option set.'))
   required_params.append('dest_dir')
   parser.add_option('--dest-dir',
                     action='store', type='string',
                     help=('REQUIRED: Directory where all images should be '
                           'written. If this directory does not exist yet, it '
                           'will be created.'))
   parser.add_option('--json-filename',
                     action='store', type='string',
                     default=DEFAULT_JSON_FILENAME,
                     help=('JSON summary filename to read for each builder; '
                           'defaults to "%default".'))
+  parser.add_option('--list-builders', action='store_true',
+                    help=('List all available builders.'))
   (params, remaining_args) = parser.parse_args()
 
+  if params.list_builders:
+    dirs, _ = gcs_list_bucket_contents(bucket=GM_SUMMARIES_BUCKET)
+    print '\n'.join(dirs)
+    return
+
   # Make sure all required options were set,
   # and that there were no items left over in the command line.
   for required_param in required_params:
     if not getattr(params, required_param):
       raise Exception('required option \'%s\' was not set' % required_param)
   if len(remaining_args) is not 0:
     raise Exception('extra items specified in the command line: %s' %
                     remaining_args)
 
   downloader = Download(actuals_base_url=params.actuals_base_url)
   downloader.fetch(builder_name=params.builder,
                    dest_dir=params.dest_dir)
 
 
 
 if __name__ == '__main__':
   main()