| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Download files from Google Storage based on SHA1 sums.""" | 6 """Download files from Google Storage based on SHA1 sums.""" |
| 7 | 7 |
| 8 | 8 |
| 9 import hashlib | 9 import hashlib |
| 10 import optparse | 10 import optparse |
| (...skipping 27 matching lines...) Expand all Loading... |
| 38 and is also immutable.""" | 38 and is also immutable.""" |
| 39 def __init__(self, path, boto_path, timeout=None): | 39 def __init__(self, path, boto_path, timeout=None): |
| 40 if not os.path.exists(path): | 40 if not os.path.exists(path): |
| 41 raise FileNotFoundError('GSUtil not found in %s' % path) | 41 raise FileNotFoundError('GSUtil not found in %s' % path) |
| 42 self.path = path | 42 self.path = path |
| 43 self.timeout = timeout | 43 self.timeout = timeout |
| 44 self.boto_path = boto_path | 44 self.boto_path = boto_path |
| 45 | 45 |
| 46 def get_sub_env(self): | 46 def get_sub_env(self): |
| 47 env = os.environ.copy() | 47 env = os.environ.copy() |
| 48 if self.boto_path: | 48 if self.boto_path == os.devnull: |
| 49 env['AWS_CREDENTIAL_FILE'] = '' |
| 50 env['BOTO_CONFIG'] = '' |
| 51 elif self.boto_path: |
| 49 env['AWS_CREDENTIAL_FILE'] = self.boto_path | 52 env['AWS_CREDENTIAL_FILE'] = self.boto_path |
| 53 env['BOTO_CONFIG'] = self.boto_path |
| 50 else: | 54 else: |
| 51 custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools' | 55 custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools' |
| 52 custompath = os.path.expanduser(custompath) | 56 custompath = os.path.expanduser(custompath) |
| 53 if os.path.exists(custompath): | 57 if os.path.exists(custompath): |
| 54 env['AWS_CREDENTIAL_FILE'] = custompath | 58 env['AWS_CREDENTIAL_FILE'] = custompath |
| 55 | 59 |
| 56 return env | 60 return env |
| 57 | 61 |
| 58 def call(self, *args): | 62 def call(self, *args): |
| 59 return subprocess2.call((sys.executable, self.path) + args, | 63 return subprocess2.call((sys.executable, self.path) + args, |
| (...skipping 230 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 290 parser.add_option('-d', '--directory', action='store_true', | 294 parser.add_option('-d', '--directory', action='store_true', |
| 291 help='The target is a directory. ' | 295 help='The target is a directory. ' |
| 292 'Cannot be used with -s/--sha1_file.') | 296 'Cannot be used with -s/--sha1_file.') |
| 293 parser.add_option('-s', '--sha1_file', action='store_true', | 297 parser.add_option('-s', '--sha1_file', action='store_true', |
| 294 help='The target is a file containing a sha1 sum. ' | 298 help='The target is a file containing a sha1 sum. ' |
| 295 'Cannot be used with -d/--directory.') | 299 'Cannot be used with -d/--directory.') |
| 296 parser.add_option('-g', '--config', action='store_true', | 300 parser.add_option('-g', '--config', action='store_true', |
| 297 help='Alias for "gsutil config". Run this if you want ' | 301 help='Alias for "gsutil config". Run this if you want ' |
| 298 'to initialize your saved Google Storage ' | 302 'to initialize your saved Google Storage ' |
| 299 'credentials.') | 303 'credentials.') |
| 300 parser.add_option('-p', '--platform', | 304 parser.add_option('-n', '--no_auth', action='store_true', |
| 305 help='Skip auth checking. Use if it\'s known that the ' |
| 306 'target bucket is a public bucket.') |
| 307 parser.add_option('-p', '--platform', |
| 301 help='A regular expression that is compared against ' | 308 help='A regular expression that is compared against ' |
| 302 'Python\'s sys.platform. If this option is specified, ' | 309 'Python\'s sys.platform. If this option is specified, ' |
| 303 'the download will happen only if there is a match.') | 310 'the download will happen only if there is a match.') |
| 304 | 311 |
| 305 (options, args) = parser.parse_args() | 312 (options, args) = parser.parse_args() |
| 306 | 313 |
| 307 # Make sure we should run at all based on platform matching. | 314 # Make sure we should run at all based on platform matching. |
| 308 if options.platform: | 315 if options.platform: |
| 309 if not re.match(options.platform, sys.platform): | 316 if not re.match(options.platform, sys.platform): |
| 310 print('The current platform doesn\'t match "%s", skipping.' % | 317 print('The current platform doesn\'t match "%s", skipping.' % |
| 311 options.platform) | 318 options.platform) |
| 312 return 0 | 319 return 0 |
| 313 | 320 |
| 321 # Set the boto file to /dev/null if we don't need auth. |
| 322 if options.no_auth: |
| 323 options.boto = os.devnull |
| 324 |
| 314 # Make sure we can find a working instance of gsutil. | 325 # Make sure we can find a working instance of gsutil. |
| 315 if os.path.exists(GSUTIL_DEFAULT_PATH): | 326 if os.path.exists(GSUTIL_DEFAULT_PATH): |
| 316 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) | 327 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
| 317 else: | 328 else: |
| 318 gsutil = None | 329 gsutil = None |
| 319 for path in os.environ["PATH"].split(os.pathsep): | 330 for path in os.environ["PATH"].split(os.pathsep): |
| 320 if os.path.exists(path) and 'gsutil' in os.listdir(path): | 331 if os.path.exists(path) and 'gsutil' in os.listdir(path): |
| 321 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | 332 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) |
| 322 if not gsutil: | 333 if not gsutil: |
| 323 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 334 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 370 return code | 381 return code |
| 371 | 382 |
| 372 return download_from_google_storage( | 383 return download_from_google_storage( |
| 373 input_filename, base_url, gsutil, options.num_threads, options.directory, | 384 input_filename, base_url, gsutil, options.num_threads, options.directory, |
| 374 options.recursive, options.force, options.output, options.ignore_errors, | 385 options.recursive, options.force, options.output, options.ignore_errors, |
| 375 options.sha1_file) | 386 options.sha1_file) |
| 376 | 387 |
| 377 | 388 |
| 378 if __name__ == '__main__': | 389 if __name__ == '__main__': |
| 379 sys.exit(main(sys.argv)) | 390 sys.exit(main(sys.argv)) |
| OLD | NEW |