Index: gm/rebaseline_server/download_actuals.py |
diff --git a/gm/rebaseline_server/download_actuals.py b/gm/rebaseline_server/download_actuals.py |
index 3f3f640f1216a1a5b041d809a8f316e6e11bec3a..dfaff40d108bc80fec6a91fb7c364a3cf71c3b52 100755 |
--- a/gm/rebaseline_server/download_actuals.py |
+++ b/gm/rebaseline_server/download_actuals.py |
@@ -41,8 +41,17 @@ if TOOLS_DIRECTORY not in sys.path: |
import buildbot_globals |
import gm_json |
-DEFAULT_ACTUALS_BASE_URL = posixpath.join( |
- buildbot_globals.Get('autogen_svn_url'), 'gm-actual') |
+# Imports from third-party code |
+APICLIENT_DIRECTORY = os.path.join( |
+ TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') |
+if APICLIENT_DIRECTORY not in sys.path: |
+ sys.path.append(APICLIENT_DIRECTORY) |
+from googleapiclient.discovery import build as build_service |
+ |
+ |
+GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket') |
+DEFAULT_ACTUALS_BASE_URL = ( |
+ 'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET) |
DEFAULT_JSON_FILENAME = 'actual-results.json' |
@@ -96,6 +105,8 @@ class Download(object): |
test_name=test, hash_type=hash_type, hash_digest=hash_digest, |
gm_actuals_root_url=self._gm_actuals_root_url) |
dest_path = os.path.join(dest_dir, config, test + '.png') |
+ # TODO(epoger): To speed this up, we should only download files that |
+ # we don't already have on local disk. |
copy_contents(source_url=source_url, dest_path=dest_path, |
create_subdirs_if_needed=True) |
@@ -151,6 +162,30 @@ def copy_contents(source_url, dest_path, create_subdirs_if_needed=False): |
shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) |
+def gcs_list_bucket_contents(bucket, subdir=None): |
epoger
2014/05/30 22:11:36
I plan to use this to replace the use of the gsuti
benchen
2014/05/30 23:45:03
This looks promising. If we use the API for restri
|
+ """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. |
+ |
+ Args: |
+ bucket: name of the Google Storage bucket |
+ subdir: directory within the bucket to list, or None for root directory |
+ """ |
+ if subdir: |
+ if not subdir.endswith('/'): |
+ subdir += '/' |
rmistry
2014/06/03 12:23:39
Can replace 173 and 174 with:
posixpath.join(subdi
epoger
2014/06/03 15:27:52
Took half of the suggestion, thanks. I would pref
|
+ subdir_length = len(subdir) |
+ else: |
+ subdir_length = 0 |
+ |
+ storage = build_service('storage', 'v1') |
rmistry
2014/06/03 12:23:39
google-api-python-client is cool, I wonder who wro
epoger
2014/06/03 15:27:52
I heard that the guy who wrote google-api-java-cli
|
+ command = storage.objects().list( |
+ bucket=bucket, delimiter='/', fields='items(name),prefixes', |
rmistry
2014/06/03 12:23:39
Can also use posixpath.sep instead of '/' though I
epoger
2014/06/03 15:27:52
I think it's 50/50, so went with the shorter one.
|
+ prefix=subdir) |
+ results = command.execute() |
+ dirs = [item[subdir_length:-1] for item in results.get('prefixes', [])] |
+ files = [item['name'][subdir_length:] for item in results.get('items', [])] |
rmistry
2014/06/03 12:23:39
Lines 184 and 185 are a little confusing could you
epoger
2014/06/03 15:27:52
You're right. Do you think the comments are suffi
|
+ return (dirs, files) |
+ |
+ |
def main(): |
parser = optparse.OptionParser() |
required_params = [] |
@@ -159,16 +194,13 @@ def main(): |
default=DEFAULT_ACTUALS_BASE_URL, |
help=('Base URL from which to read files containing JSON ' |
'summaries of actual GM results; defaults to ' |
- '"%default". To get a specific revision (useful for ' |
- 'trybots) replace "svn" with "svn-history/r123".')) |
- # TODO(epoger): Rather than telling the user to run "svn ls" to get the list |
- # of builders, add a --list-builders option that will print the list. |
+ '"%default".')) |
epoger
2014/05/30 22:11:36
Unfortunately, this tool has lost the ability to r
benchen
2014/05/30 23:45:03
Joe must know the solution in details.
On 2014/05/
rmistry
2014/06/03 12:23:39
Or Joe will know who to ask from the Cloud Storage
epoger
2014/06/03 15:27:52
Sorry, I wasn't clear. It isn't a Cloud Storage A
|
required_params.append('builder') |
parser.add_option('--builder', |
action='store', type='string', |
help=('REQUIRED: Which builder to download results for. ' |
- 'To see a list of builders, run "svn ls %s".' % |
- DEFAULT_ACTUALS_BASE_URL)) |
+ 'To see a list of builders, run with the ' |
+ '--list-builders option set.')) |
required_params.append('dest_dir') |
parser.add_option('--dest-dir', |
action='store', type='string', |
@@ -180,8 +212,15 @@ def main(): |
default=DEFAULT_JSON_FILENAME, |
help=('JSON summary filename to read for each builder; ' |
'defaults to "%default".')) |
+ parser.add_option('--list-builders', action='store_true', |
+ help=('List all available builders.')) |
(params, remaining_args) = parser.parse_args() |
+ if params.list_builders: |
+ dirs, _ = gcs_list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) |
+ print '\n'.join(dirs) |
+ return |
+ |
# Make sure all required options were set, |
# and that there were no items left over in the command line. |
for required_param in required_params: |