OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Download files from Google Storage based on SHA1 sums.""" | 6 """Download files from Google Storage based on SHA1 sums.""" |
7 | 7 |
8 | 8 |
9 import hashlib | 9 import hashlib |
10 import optparse | 10 import optparse |
11 import os | 11 import os |
12 import Queue | 12 import Queue |
13 import re | 13 import re |
14 import stat | 14 import stat |
15 import sys | 15 import sys |
16 import threading | 16 import threading |
17 import time | 17 import time |
18 | 18 |
19 import subprocess2 | 19 import subprocess2 |
20 | 20 |
21 | 21 |
22 GSUTIL_DEFAULT_PATH = os.path.join( | 22 GSUTIL_DEFAULT_PATH = os.path.join( |
23 os.path.dirname(os.path.abspath(__file__)), | 23 os.path.dirname(os.path.abspath(__file__)), 'gsutil.py') |
24 'third_party', 'gsutil', 'gsutil') | |
25 # Maps sys.platform to what we actually want to call them. | 24 # Maps sys.platform to what we actually want to call them. |
26 PLATFORM_MAPPING = { | 25 PLATFORM_MAPPING = { |
27 'cygwin': 'win', | 26 'cygwin': 'win', |
28 'darwin': 'mac', | 27 'darwin': 'mac', |
29 'linux2': 'linux', | 28 'linux2': 'linux', |
30 'win32': 'win', | 29 'win32': 'win', |
31 } | 30 } |
32 | 31 |
33 | 32 |
34 class FileNotFoundError(IOError): | 33 class FileNotFoundError(IOError): |
(...skipping 13 matching lines...) Expand all Loading... | |
48 Under cygwin, this will always return "win32" like the native Python.""" | 47 Under cygwin, this will always return "win32" like the native Python.""" |
49 if sys.platform == 'cygwin': | 48 if sys.platform == 'cygwin': |
50 return 'win32' | 49 return 'win32' |
51 return sys.platform | 50 return sys.platform |
52 | 51 |
53 | 52 |
54 # Common utilities | 53 # Common utilities |
55 class Gsutil(object): | 54 class Gsutil(object): |
56 """Call gsutil with some predefined settings. This is a convenience object, | 55 """Call gsutil with some predefined settings. This is a convenience object, |
57 and is also immutable.""" | 56 and is also immutable.""" |
58 def __init__(self, path, boto_path, timeout=None, bypass_prodaccess=False): | 57 def __init__(self, path, boto_path, timeout=None): |
59 if not os.path.exists(path): | 58 if not os.path.exists(path): |
60 raise FileNotFoundError('GSUtil not found in %s' % path) | 59 raise FileNotFoundError('GSUtil not found in %s' % path) |
61 self.path = path | 60 self.path = path |
62 self.timeout = timeout | 61 self.timeout = timeout |
63 self.boto_path = boto_path | 62 self.boto_path = boto_path |
64 self.bypass_prodaccess = bypass_prodaccess | 63 self.version = '4.7' |
pgervais
2014/12/17 01:07:07
Keyword parameters are good for default values:
d
hinoka
2014/12/17 01:09:41
Done.
| |
65 | 64 |
66 def get_sub_env(self): | 65 def get_sub_env(self): |
67 env = os.environ.copy() | 66 env = os.environ.copy() |
68 if self.boto_path == os.devnull: | 67 if self.boto_path == os.devnull: |
69 env['AWS_CREDENTIAL_FILE'] = '' | 68 env['AWS_CREDENTIAL_FILE'] = '' |
70 env['BOTO_CONFIG'] = '' | 69 env['BOTO_CONFIG'] = '' |
71 elif self.boto_path: | 70 elif self.boto_path: |
72 env['AWS_CREDENTIAL_FILE'] = self.boto_path | 71 env['AWS_CREDENTIAL_FILE'] = self.boto_path |
73 env['BOTO_CONFIG'] = self.boto_path | 72 env['BOTO_CONFIG'] = self.boto_path |
74 else: | 73 else: |
75 custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools' | 74 custompath = env.get('AWS_CREDENTIAL_FILE', '~/.boto') + '.depot_tools' |
76 custompath = os.path.expanduser(custompath) | 75 custompath = os.path.expanduser(custompath) |
77 if os.path.exists(custompath): | 76 if os.path.exists(custompath): |
78 env['AWS_CREDENTIAL_FILE'] = custompath | 77 env['AWS_CREDENTIAL_FILE'] = custompath |
79 | 78 |
80 return env | 79 return env |
81 | 80 |
82 def call(self, *args): | 81 def call(self, *args): |
83 cmd = [sys.executable, self.path] | 82 cmd = [sys.executable, self.path, '--force-version', self.version] |
84 if self.bypass_prodaccess: | |
85 cmd.append('--bypass_prodaccess') | |
86 cmd.extend(args) | 83 cmd.extend(args) |
87 return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout) | 84 return subprocess2.call(cmd, env=self.get_sub_env(), timeout=self.timeout) |
88 | 85 |
89 def check_call(self, *args): | 86 def check_call(self, *args): |
90 cmd = [sys.executable, self.path] | 87 cmd = [sys.executable, self.path, '--force-version', self.version] |
91 if self.bypass_prodaccess: | |
92 cmd.append('--bypass_prodaccess') | |
93 cmd.extend(args) | 88 cmd.extend(args) |
94 ((out, err), code) = subprocess2.communicate( | 89 ((out, err), code) = subprocess2.communicate( |
95 cmd, | 90 cmd, |
96 stdout=subprocess2.PIPE, | 91 stdout=subprocess2.PIPE, |
97 stderr=subprocess2.PIPE, | 92 stderr=subprocess2.PIPE, |
98 env=self.get_sub_env(), | 93 env=self.get_sub_env(), |
99 timeout=self.timeout) | 94 timeout=self.timeout) |
100 | 95 |
101 # Parse output. | 96 # Parse output. |
102 status_code_match = re.search('status=([0-9]+)', err) | 97 status_code_match = re.search('status=([0-9]+)', err) |
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
230 file_url, output_filename))) | 225 file_url, output_filename))) |
231 continue | 226 continue |
232 # Fetch the file. | 227 # Fetch the file. |
233 out_q.put('%d> Downloading %s...' % (thread_num, output_filename)) | 228 out_q.put('%d> Downloading %s...' % (thread_num, output_filename)) |
234 try: | 229 try: |
235 os.remove(output_filename) # Delete the file if it exists already. | 230 os.remove(output_filename) # Delete the file if it exists already. |
236 except OSError: | 231 except OSError: |
237 if os.path.exists(output_filename): | 232 if os.path.exists(output_filename): |
238 out_q.put('%d> Warning: deleting %s failed.' % ( | 233 out_q.put('%d> Warning: deleting %s failed.' % ( |
239 thread_num, output_filename)) | 234 thread_num, output_filename)) |
240 code, _, err = gsutil.check_call('cp', '-q', file_url, output_filename) | 235 code, _, err = gsutil.check_call('cp', file_url, output_filename) |
241 if code != 0: | 236 if code != 0: |
242 out_q.put('%d> %s' % (thread_num, err)) | 237 out_q.put('%d> %s' % (thread_num, err)) |
243 ret_codes.put((code, err)) | 238 ret_codes.put((code, err)) |
244 | 239 |
245 # Set executable bit. | 240 # Set executable bit. |
246 if sys.platform == 'cygwin': | 241 if sys.platform == 'cygwin': |
247 # Under cygwin, mark all files as executable. The executable flag in | 242 # Under cygwin, mark all files as executable. The executable flag in |
248 # Google Storage will not be set when uploading from Windows, so if | 243 # Google Storage will not be set when uploading from Windows, so if |
249 # this script is running under cygwin and we're downloading an | 244 # this script is running under cygwin and we're downloading an |
250 # executable, it will be unrunnable from inside cygwin without this. | 245 # executable, it will be unrunnable from inside cygwin without this. |
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
391 options.platform) | 386 options.platform) |
392 return 0 | 387 return 0 |
393 | 388 |
394 # Set the boto file to /dev/null if we don't need auth. | 389 # Set the boto file to /dev/null if we don't need auth. |
395 if options.no_auth: | 390 if options.no_auth: |
396 options.boto = os.devnull | 391 options.boto = os.devnull |
397 | 392 |
398 # Make sure gsutil exists where we expect it to. | 393 # Make sure gsutil exists where we expect it to. |
399 if os.path.exists(GSUTIL_DEFAULT_PATH): | 394 if os.path.exists(GSUTIL_DEFAULT_PATH): |
400 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, | 395 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, |
401 boto_path=options.boto, | 396 boto_path=options.boto) |
402 bypass_prodaccess=options.no_auth) | |
403 else: | 397 else: |
404 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 398 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
405 GSUTIL_DEFAULT_PATH) | 399 GSUTIL_DEFAULT_PATH) |
406 | 400 |
407 # Passing in -g/--config will run our copy of GSUtil, then quit. | 401 # Passing in -g/--config will run our copy of GSUtil, then quit. |
408 if options.config: | 402 if options.config: |
409 return gsutil.call('config', '-r', '-o', | 403 return gsutil.call('config', '-r', '-o', |
410 os.path.expanduser('~/.boto.depot_tools')) | 404 os.path.expanduser('~/.boto.depot_tools')) |
411 | 405 |
412 if not args: | 406 if not args: |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
460 return code | 454 return code |
461 | 455 |
462 return download_from_google_storage( | 456 return download_from_google_storage( |
463 input_filename, base_url, gsutil, options.num_threads, options.directory, | 457 input_filename, base_url, gsutil, options.num_threads, options.directory, |
464 options.recursive, options.force, options.output, options.ignore_errors, | 458 options.recursive, options.force, options.output, options.ignore_errors, |
465 options.sha1_file, options.verbose, options.auto_platform) | 459 options.sha1_file, options.verbose, options.auto_platform) |
466 | 460 |
467 | 461 |
468 if __name__ == '__main__': | 462 if __name__ == '__main__': |
469 sys.exit(main(sys.argv)) | 463 sys.exit(main(sys.argv)) |
OLD | NEW |