Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(151)

Side by Side Diff: download_from_google_storage.py

Issue 76583002: Add no_auth flag to skip auth checking for buckets that don't require it. Also fix tests (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools
Patch Set: Set the boto file to empty if --no_auth is specified Created 7 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | tests/download_from_google_storage_unittests.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Download files from Google Storage based on SHA1 sums.""" 6 """Download files from Google Storage based on SHA1 sums."""
7 7
8 8
9 import hashlib 9 import hashlib
10 import optparse 10 import optparse
(...skipping 27 matching lines...) Expand all
38 and is also immutable.""" 38 and is also immutable."""
39 def __init__(self, path, boto_path, timeout=None): 39 def __init__(self, path, boto_path, timeout=None):
40 if not os.path.exists(path): 40 if not os.path.exists(path):
41 raise FileNotFoundError('GSUtil not found in %s' % path) 41 raise FileNotFoundError('GSUtil not found in %s' % path)
42 self.path = path 42 self.path = path
43 self.timeout = timeout 43 self.timeout = timeout
44 self.boto_path = boto_path 44 self.boto_path = boto_path
45 45
46 def get_sub_env(self): 46 def get_sub_env(self):
47 env = os.environ.copy() 47 env = os.environ.copy()
48 if self.boto_path: 48 if self.boto_path == os.devnull:
49 env['AWS_CREDENTIAL_FILE'] = ''
50 env['BOTO_CONFIG'] = ''
51 elif self.boto_path:
49 env['AWS_CREDENTIAL_FILE'] = self.boto_path 52 env['AWS_CREDENTIAL_FILE'] = self.boto_path
53 env['BOTO_CONFIG'] = self.boto_path
50 else: 54 else:
51 custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools' 55 custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools'
52 custompath = os.path.expanduser(custompath) 56 custompath = os.path.expanduser(custompath)
53 if os.path.exists(custompath): 57 if os.path.exists(custompath):
54 env['AWS_CREDENTIAL_FILE'] = custompath 58 env['AWS_CREDENTIAL_FILE'] = custompath
55 59
56 return env 60 return env
57 61
58 def call(self, *args): 62 def call(self, *args):
59 return subprocess2.call((sys.executable, self.path) + args, 63 return subprocess2.call((sys.executable, self.path) + args,
(...skipping 230 matching lines...) Expand 10 before | Expand all | Expand 10 after
290 parser.add_option('-d', '--directory', action='store_true', 294 parser.add_option('-d', '--directory', action='store_true',
291 help='The target is a directory. ' 295 help='The target is a directory. '
292 'Cannot be used with -s/--sha1_file.') 296 'Cannot be used with -s/--sha1_file.')
293 parser.add_option('-s', '--sha1_file', action='store_true', 297 parser.add_option('-s', '--sha1_file', action='store_true',
294 help='The target is a file containing a sha1 sum. ' 298 help='The target is a file containing a sha1 sum. '
295 'Cannot be used with -d/--directory.') 299 'Cannot be used with -d/--directory.')
296 parser.add_option('-g', '--config', action='store_true', 300 parser.add_option('-g', '--config', action='store_true',
297 help='Alias for "gsutil config". Run this if you want ' 301 help='Alias for "gsutil config". Run this if you want '
298 'to initialize your saved Google Storage ' 302 'to initialize your saved Google Storage '
299 'credentials.') 303 'credentials.')
300 parser.add_option('-p', '--platform', 304 parser.add_option('-n', '--no_auth', action='store_true',
305 help='Skip auth checking. Use if it\'s known that the '
306 'target bucket is a public bucket.')
307 parser.add_option('-p', '--platform',
301 help='A regular expression that is compared against ' 308 help='A regular expression that is compared against '
302 'Python\'s sys.platform. If this option is specified, ' 309 'Python\'s sys.platform. If this option is specified, '
303 'the download will happen only if there is a match.') 310 'the download will happen only if there is a match.')
304 311
305 (options, args) = parser.parse_args() 312 (options, args) = parser.parse_args()
306 313
307 # Make sure we should run at all based on platform matching. 314 # Make sure we should run at all based on platform matching.
308 if options.platform: 315 if options.platform:
309 if not re.match(options.platform, sys.platform): 316 if not re.match(options.platform, sys.platform):
310 print('The current platform doesn\'t match "%s", skipping.' % 317 print('The current platform doesn\'t match "%s", skipping.' %
311 options.platform) 318 options.platform)
312 return 0 319 return 0
313 320
321 # Set the boto file to /dev/null if we don't need auth.
322 if options.no_auth:
323 options.boto = os.devnull
324
314 # Make sure we can find a working instance of gsutil. 325 # Make sure we can find a working instance of gsutil.
315 if os.path.exists(GSUTIL_DEFAULT_PATH): 326 if os.path.exists(GSUTIL_DEFAULT_PATH):
316 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) 327 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
317 else: 328 else:
318 gsutil = None 329 gsutil = None
319 for path in os.environ["PATH"].split(os.pathsep): 330 for path in os.environ["PATH"].split(os.pathsep):
320 if os.path.exists(path) and 'gsutil' in os.listdir(path): 331 if os.path.exists(path) and 'gsutil' in os.listdir(path):
321 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) 332 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
322 if not gsutil: 333 if not gsutil:
323 parser.error('gsutil not found in %s, bad depot_tools checkout?' % 334 parser.error('gsutil not found in %s, bad depot_tools checkout?' %
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after
370 return code 381 return code
371 382
372 return download_from_google_storage( 383 return download_from_google_storage(
373 input_filename, base_url, gsutil, options.num_threads, options.directory, 384 input_filename, base_url, gsutil, options.num_threads, options.directory,
374 options.recursive, options.force, options.output, options.ignore_errors, 385 options.recursive, options.force, options.output, options.ignore_errors,
375 options.sha1_file) 386 options.sha1_file)
376 387
377 388
378 if __name__ == '__main__': 389 if __name__ == '__main__':
379 sys.exit(main(sys.argv)) 390 sys.exit(main(sys.argv))
OLDNEW
« no previous file with comments | « no previous file | tests/download_from_google_storage_unittests.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698