OLD | NEW |
---|---|
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 """ | 3 """ |
4 Copyright 2014 Google Inc. | 4 Copyright 2014 Google Inc. |
5 | 5 |
6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
7 found in the LICENSE file. | 7 found in the LICENSE file. |
8 | 8 |
9 Download actual GM results for a particular builder. | 9 Download actual GM results for a particular builder. |
10 """ | 10 """ |
(...skipping 23 matching lines...) Expand all Loading... | |
34 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) | 34 TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) |
35 GM_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'gm') | 35 GM_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'gm') |
36 TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools') | 36 TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools') |
37 if GM_DIRECTORY not in sys.path: | 37 if GM_DIRECTORY not in sys.path: |
38 sys.path.append(GM_DIRECTORY) | 38 sys.path.append(GM_DIRECTORY) |
39 if TOOLS_DIRECTORY not in sys.path: | 39 if TOOLS_DIRECTORY not in sys.path: |
40 sys.path.append(TOOLS_DIRECTORY) | 40 sys.path.append(TOOLS_DIRECTORY) |
41 import buildbot_globals | 41 import buildbot_globals |
42 import gm_json | 42 import gm_json |
43 | 43 |
44 DEFAULT_ACTUALS_BASE_URL = posixpath.join( | 44 # Imports from third-party code |
45 buildbot_globals.Get('autogen_svn_url'), 'gm-actual') | 45 APICLIENT_DIRECTORY = os.path.join( |
46 TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') | |
47 if APICLIENT_DIRECTORY not in sys.path: | |
48 sys.path.append(APICLIENT_DIRECTORY) | |
49 from googleapiclient.discovery import build as build_service | |
50 | |
51 | |
52 GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket') | |
53 DEFAULT_ACTUALS_BASE_URL = ( | |
54 'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET) | |
46 DEFAULT_JSON_FILENAME = 'actual-results.json' | 55 DEFAULT_JSON_FILENAME = 'actual-results.json' |
47 | 56 |
48 | 57 |
49 class Download(object): | 58 class Download(object): |
50 | 59 |
51 def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL, | 60 def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL, |
52 json_filename=DEFAULT_JSON_FILENAME, | 61 json_filename=DEFAULT_JSON_FILENAME, |
53 gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL): | 62 gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL): |
54 """ | 63 """ |
55 Args: | 64 Args: |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
89 results_of_this_type = actual_results_dict[result_type] | 98 results_of_this_type = actual_results_dict[result_type] |
90 if not results_of_this_type: | 99 if not results_of_this_type: |
91 continue | 100 continue |
92 for image_name in sorted(results_of_this_type.keys()): | 101 for image_name in sorted(results_of_this_type.keys()): |
93 (test, config) = self._image_filename_re.match(image_name).groups() | 102 (test, config) = self._image_filename_re.match(image_name).groups() |
94 (hash_type, hash_digest) = results_of_this_type[image_name] | 103 (hash_type, hash_digest) = results_of_this_type[image_name] |
95 source_url = gm_json.CreateGmActualUrl( | 104 source_url = gm_json.CreateGmActualUrl( |
96 test_name=test, hash_type=hash_type, hash_digest=hash_digest, | 105 test_name=test, hash_type=hash_type, hash_digest=hash_digest, |
97 gm_actuals_root_url=self._gm_actuals_root_url) | 106 gm_actuals_root_url=self._gm_actuals_root_url) |
98 dest_path = os.path.join(dest_dir, config, test + '.png') | 107 dest_path = os.path.join(dest_dir, config, test + '.png') |
108 # TODO(epoger): To speed this up, we should only download files that | |
109 # we don't already have on local disk. | |
99 copy_contents(source_url=source_url, dest_path=dest_path, | 110 copy_contents(source_url=source_url, dest_path=dest_path, |
100 create_subdirs_if_needed=True) | 111 create_subdirs_if_needed=True) |
101 | 112 |
102 | 113 |
103 def create_filepath_url(filepath): | 114 def create_filepath_url(filepath): |
104 """ Returns a file:/// URL pointing at the given filepath on local disk. | 115 """ Returns a file:/// URL pointing at the given filepath on local disk. |
105 | 116 |
106 For now, this is only used by unittests, but I anticipate it being useful | 117 For now, this is only used by unittests, but I anticipate it being useful |
107 in production, as a way for developers to run rebaseline_server over locally | 118 in production, as a way for developers to run rebaseline_server over locally |
108 generated images. | 119 generated images. |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
144 """ | 155 """ |
145 if create_subdirs_if_needed: | 156 if create_subdirs_if_needed: |
146 dest_dir = os.path.dirname(dest_path) | 157 dest_dir = os.path.dirname(dest_path) |
147 if not os.path.exists(dest_dir): | 158 if not os.path.exists(dest_dir): |
148 os.makedirs(dest_dir) | 159 os.makedirs(dest_dir) |
149 with contextlib.closing(urllib.urlopen(source_url)) as source_handle: | 160 with contextlib.closing(urllib.urlopen(source_url)) as source_handle: |
150 with open(dest_path, 'wb') as dest_handle: | 161 with open(dest_path, 'wb') as dest_handle: |
151 shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) | 162 shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) |
152 | 163 |
153 | 164 |
165 def gcs_list_bucket_contents(bucket, subdir=None): | |
epoger
2014/05/30 22:11:36
I plan to use this to replace the use of the gsuti
benchen
2014/05/30 23:45:03
This looks promising. If we use the API for restri
| |
166 """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. | |
167 | |
168 Args: | |
169 bucket: name of the Google Storage bucket | |
170 subdir: directory within the bucket to list, or None for root directory | |
171 """ | |
172 if subdir: | |
173 if not subdir.endswith('/'): | |
174 subdir += '/' | |
rmistry
2014/06/03 12:23:39
Can replace 173 and 174 with:
posixpath.join(subdi
epoger
2014/06/03 15:27:52
Took half of the suggestion, thanks. I would pref
| |
175 subdir_length = len(subdir) | |
176 else: | |
177 subdir_length = 0 | |
178 | |
179 storage = build_service('storage', 'v1') | |
rmistry
2014/06/03 12:23:39
google-api-python-client is cool, I wonder who wro
epoger
2014/06/03 15:27:52
I heard that the guy who wrote google-api-java-cli
| |
180 command = storage.objects().list( | |
181 bucket=bucket, delimiter='/', fields='items(name),prefixes', | |
rmistry
2014/06/03 12:23:39
Can also use posixpath.sep instead of '/' though I
epoger
2014/06/03 15:27:52
I think it's 50/50, so went with the shorter one.
| |
182 prefix=subdir) | |
183 results = command.execute() | |
184 dirs = [item[subdir_length:-1] for item in results.get('prefixes', [])] | |
185 files = [item['name'][subdir_length:] for item in results.get('items', [])] | |
rmistry
2014/06/03 12:23:39
Lines 184 and 185 are a little confusing could you
epoger
2014/06/03 15:27:52
You're right. Do you think the comments are suffi
| |
186 return (dirs, files) | |
187 | |
188 | |
154 def main(): | 189 def main(): |
155 parser = optparse.OptionParser() | 190 parser = optparse.OptionParser() |
156 required_params = [] | 191 required_params = [] |
157 parser.add_option('--actuals-base-url', | 192 parser.add_option('--actuals-base-url', |
158 action='store', type='string', | 193 action='store', type='string', |
159 default=DEFAULT_ACTUALS_BASE_URL, | 194 default=DEFAULT_ACTUALS_BASE_URL, |
160 help=('Base URL from which to read files containing JSON ' | 195 help=('Base URL from which to read files containing JSON ' |
161 'summaries of actual GM results; defaults to ' | 196 'summaries of actual GM results; defaults to ' |
162 '"%default". To get a specific revision (useful for ' | 197 '"%default".')) |
epoger
2014/05/30 22:11:36
Unfortunately, this tool has lost the ability to r
benchen
2014/05/30 23:45:03
Joe must know the solution in details.
On 2014/05/
rmistry
2014/06/03 12:23:39
Or Joe will know who to ask from the Cloud Storage
epoger
2014/06/03 15:27:52
Sorry, I wasn't clear. It isn't a Cloud Storage A
| |
163 'trybots) replace "svn" with "svn-history/r123".')) | |
164 # TODO(epoger): Rather than telling the user to run "svn ls" to get the list | |
165 # of builders, add a --list-builders option that will print the list. | |
166 required_params.append('builder') | 198 required_params.append('builder') |
167 parser.add_option('--builder', | 199 parser.add_option('--builder', |
168 action='store', type='string', | 200 action='store', type='string', |
169 help=('REQUIRED: Which builder to download results for. ' | 201 help=('REQUIRED: Which builder to download results for. ' |
170 'To see a list of builders, run "svn ls %s".' % | 202 'To see a list of builders, run with the ' |
171 DEFAULT_ACTUALS_BASE_URL)) | 203 '--list-builders option set.')) |
172 required_params.append('dest_dir') | 204 required_params.append('dest_dir') |
173 parser.add_option('--dest-dir', | 205 parser.add_option('--dest-dir', |
174 action='store', type='string', | 206 action='store', type='string', |
175 help=('REQUIRED: Directory where all images should be ' | 207 help=('REQUIRED: Directory where all images should be ' |
176 'written. If this directory does not exist yet, it ' | 208 'written. If this directory does not exist yet, it ' |
177 'will be created.')) | 209 'will be created.')) |
178 parser.add_option('--json-filename', | 210 parser.add_option('--json-filename', |
179 action='store', type='string', | 211 action='store', type='string', |
180 default=DEFAULT_JSON_FILENAME, | 212 default=DEFAULT_JSON_FILENAME, |
181 help=('JSON summary filename to read for each builder; ' | 213 help=('JSON summary filename to read for each builder; ' |
182 'defaults to "%default".')) | 214 'defaults to "%default".')) |
215 parser.add_option('--list-builders', action='store_true', | |
216 help=('List all available builders.')) | |
183 (params, remaining_args) = parser.parse_args() | 217 (params, remaining_args) = parser.parse_args() |
184 | 218 |
219 if params.list_builders: | |
220 dirs, _ = gcs_list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) | |
221 print '\n'.join(dirs) | |
222 return | |
223 | |
185 # Make sure all required options were set, | 224 # Make sure all required options were set, |
186 # and that there were no items left over in the command line. | 225 # and that there were no items left over in the command line. |
187 for required_param in required_params: | 226 for required_param in required_params: |
188 if not getattr(params, required_param): | 227 if not getattr(params, required_param): |
189 raise Exception('required option \'%s\' was not set' % required_param) | 228 raise Exception('required option \'%s\' was not set' % required_param) |
190 if len(remaining_args) is not 0: | 229 if len(remaining_args) is not 0: |
191 raise Exception('extra items specified in the command line: %s' % | 230 raise Exception('extra items specified in the command line: %s' % |
192 remaining_args) | 231 remaining_args) |
193 | 232 |
194 downloader = Download(actuals_base_url=params.actuals_base_url) | 233 downloader = Download(actuals_base_url=params.actuals_base_url) |
195 downloader.fetch(builder_name=params.builder, | 234 downloader.fetch(builder_name=params.builder, |
196 dest_dir=params.dest_dir) | 235 dest_dir=params.dest_dir) |
197 | 236 |
198 | 237 |
199 | 238 |
200 if __name__ == '__main__': | 239 if __name__ == '__main__': |
201 main() | 240 main() |
OLD | NEW |