Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(762)

Side by Side Diff: download_from_google_storage.py

Issue 42273002: Add a platform filtering and executable setting abilities to download_from_google_storage. (Closed) Base URL: http://src.chromium.org/svn/trunk/tools/depot_tools/
Patch Set: Created 7 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | upload_to_google_storage.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Download files from Google Storage based on SHA1 sums.""" 6 """Download files from Google Storage based on SHA1 sums."""
7 7
8 8
9 import hashlib 9 import hashlib
10 import optparse 10 import optparse
11 import os 11 import os
12 import Queue 12 import Queue
13 import re 13 import re
14 import stat
14 import sys 15 import sys
15 import threading 16 import threading
16 import time 17 import time
17 18
18 import subprocess2 19 import subprocess2
19 20
20 21
21 GSUTIL_DEFAULT_PATH = os.path.join( 22 GSUTIL_DEFAULT_PATH = os.path.join(
22 os.path.dirname(os.path.abspath(__file__)), 23 os.path.dirname(os.path.abspath(__file__)),
23 'third_party', 'gsutil', 'gsutil') 24 'third_party', 'gsutil', 'gsutil')
(...skipping 154 matching lines...) Expand 10 before | Expand all | Expand 10 after
178 file_url, output_filename))) 179 file_url, output_filename)))
179 continue 180 continue
180 # Fetch the file. 181 # Fetch the file.
181 out_q.put('%d> Downloading %s...' % ( 182 out_q.put('%d> Downloading %s...' % (
182 thread_num, output_filename)) 183 thread_num, output_filename))
183 code, _, err = gsutil.check_call('cp', '-q', file_url, output_filename) 184 code, _, err = gsutil.check_call('cp', '-q', file_url, output_filename)
184 if code != 0: 185 if code != 0:
185 out_q.put('%d> %s' % (thread_num, err)) 186 out_q.put('%d> %s' % (thread_num, err))
186 ret_codes.put((code, err)) 187 ret_codes.put((code, err))
187 188
189 # Mark executable if necessary. We key off of the custom header
190 # "x-goog-meta-executable".
191 #
192 # TODO(hinoka): It is supposedly faster to use "gsutil stat" but that
193 # doesn't appear to be supported by the gsutil currently in our tree. When
194 # we update, this code should use that instead of "gsutil ls -L".
195 if not sys.platform.startswith('win'):
196 code, out, _ = gsutil.check_call('ls', '-L', file_url)
197 if code != 0:
198 out_q.put('%d> %s' % (thread_num, err))
199 ret_codes.put((code, err))
200 elif re.search('x-goog-meta-executable:', out):
201 st = os.stat(output_filename)
202 os.chmod(output_filename, st.st_mode | stat.S_IEXEC)
188 203
189 def printer_worker(output_queue): 204 def printer_worker(output_queue):
190 while True: 205 while True:
191 line = output_queue.get() 206 line = output_queue.get()
192 # Its plausible we want to print empty lines. 207 # Its plausible we want to print empty lines.
193 if line is None: 208 if line is None:
194 break 209 break
195 print line 210 print line
196 211
197 212
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
275 parser.add_option('-d', '--directory', action='store_true', 290 parser.add_option('-d', '--directory', action='store_true',
276 help='The target is a directory. ' 291 help='The target is a directory. '
277 'Cannot be used with -s/--sha1_file.') 292 'Cannot be used with -s/--sha1_file.')
278 parser.add_option('-s', '--sha1_file', action='store_true', 293 parser.add_option('-s', '--sha1_file', action='store_true',
279 help='The target is a file containing a sha1 sum. ' 294 help='The target is a file containing a sha1 sum. '
280 'Cannot be used with -d/--directory.') 295 'Cannot be used with -d/--directory.')
281 parser.add_option('-g', '--config', action='store_true', 296 parser.add_option('-g', '--config', action='store_true',
282 help='Alias for "gsutil config". Run this if you want ' 297 help='Alias for "gsutil config". Run this if you want '
283 'to initialize your saved Google Storage ' 298 'to initialize your saved Google Storage '
284 'credentials.') 299 'credentials.')
300 parser.add_option('-p', '--platform',
301 help='A regular expression that is compared against '
302 'Python\'s sys.platform. If this option is specified, '
303 'the download will happen only if there is a match.')
285 304
286 (options, args) = parser.parse_args() 305 (options, args) = parser.parse_args()
287 # First, make sure we can find a working instance of gsutil. 306
307 # Make sure we should run at all based on platform matching.
308 if options.platform:
309 if not re.match(options.platform, sys.platform):
310 print('The current platform doesn\'t match "%s", skipping.' %
311 options.platform)
312 return 0
313
314 # Make sure we can find a working instance of gsutil.
288 if os.path.exists(GSUTIL_DEFAULT_PATH): 315 if os.path.exists(GSUTIL_DEFAULT_PATH):
289 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) 316 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
290 else: 317 else:
291 gsutil = None 318 gsutil = None
292 for path in os.environ["PATH"].split(os.pathsep): 319 for path in os.environ["PATH"].split(os.pathsep):
293 if os.path.exists(path) and 'gsutil' in os.listdir(path): 320 if os.path.exists(path) and 'gsutil' in os.listdir(path):
294 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) 321 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
295 if not gsutil: 322 if not gsutil:
296 parser.error('gsutil not found in %s, bad depot_tools checkout?' % 323 parser.error('gsutil not found in %s, bad depot_tools checkout?' %
297 GSUTIL_DEFAULT_PATH) 324 GSUTIL_DEFAULT_PATH)
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after
343 return code 370 return code
344 371
345 return download_from_google_storage( 372 return download_from_google_storage(
346 input_filename, base_url, gsutil, options.num_threads, options.directory, 373 input_filename, base_url, gsutil, options.num_threads, options.directory,
347 options.recursive, options.force, options.output, options.ignore_errors, 374 options.recursive, options.force, options.output, options.ignore_errors,
348 options.sha1_file) 375 options.sha1_file)
349 376
350 377
351 if __name__ == '__main__': 378 if __name__ == '__main__':
352 sys.exit(main(sys.argv)) 379 sys.exit(main(sys.argv))
OLDNEW
« no previous file with comments | « no previous file | upload_to_google_storage.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698