| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Download files from Google Storage based on SHA1 sums.""" | 6 """Download files from Google Storage based on SHA1 sums.""" |
| 7 | 7 |
| 8 | 8 |
| 9 import hashlib | 9 import hashlib |
| 10 import optparse | 10 import optparse |
| (...skipping 14 matching lines...) Expand all Loading... |
| 25 | 25 |
| 26 | 26 |
| 27 class FileNotFoundError(IOError): | 27 class FileNotFoundError(IOError): |
| 28 pass | 28 pass |
| 29 | 29 |
| 30 | 30 |
| 31 class InvalidFileError(IOError): | 31 class InvalidFileError(IOError): |
| 32 pass | 32 pass |
| 33 | 33 |
| 34 | 34 |
| 35 def GetNormalizedPlatform(): |
| 36 """Returns the result of sys.platform accounting for cygwin. |
| 37 Under cygwin, this will always return "win32" like the native Python.""" |
| 38 if sys.platform == 'cygwin': |
| 39 return 'win32' |
| 40 return sys.platform |
| 41 |
| 42 |
| 35 # Common utilities | 43 # Common utilities |
| 36 class Gsutil(object): | 44 class Gsutil(object): |
| 37 """Call gsutil with some predefined settings. This is a convenience object, | 45 """Call gsutil with some predefined settings. This is a convenience object, |
| 38 and is also immutable.""" | 46 and is also immutable.""" |
| 39 def __init__(self, path, boto_path, timeout=None): | 47 def __init__(self, path, boto_path, timeout=None): |
| 40 if not os.path.exists(path): | 48 if not os.path.exists(path): |
| 41 raise FileNotFoundError('GSUtil not found in %s' % path) | 49 raise FileNotFoundError('GSUtil not found in %s' % path) |
| 42 self.path = path | 50 self.path = path |
| 43 self.timeout = timeout | 51 self.timeout = timeout |
| 44 self.boto_path = boto_path | 52 self.boto_path = boto_path |
| (...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 190 if code != 0: | 198 if code != 0: |
| 191 out_q.put('%d> %s' % (thread_num, err)) | 199 out_q.put('%d> %s' % (thread_num, err)) |
| 192 ret_codes.put((code, err)) | 200 ret_codes.put((code, err)) |
| 193 | 201 |
| 194 # Mark executable if necessary. We key off of the custom header | 202 # Mark executable if necessary. We key off of the custom header |
| 195 # "x-goog-meta-executable". | 203 # "x-goog-meta-executable". |
| 196 # | 204 # |
| 197 # TODO(hinoka): It is supposedly faster to use "gsutil stat" but that | 205 # TODO(hinoka): It is supposedly faster to use "gsutil stat" but that |
| 198 # doesn't appear to be supported by the gsutil currently in our tree. When | 206 # doesn't appear to be supported by the gsutil currently in our tree. When |
| 199 # we update, this code should use that instead of "gsutil ls -L". | 207 # we update, this code should use that instead of "gsutil ls -L". |
| 200 if not sys.platform.startswith('win'): | 208 if sys.platform != 'win32': |
| 201 code, out, _ = gsutil.check_call('ls', '-L', file_url) | 209 code, out, _ = gsutil.check_call('ls', '-L', file_url) |
| 202 if code != 0: | 210 if code != 0: |
| 203 out_q.put('%d> %s' % (thread_num, err)) | 211 out_q.put('%d> %s' % (thread_num, err)) |
| 204 ret_codes.put((code, err)) | 212 ret_codes.put((code, err)) |
| 205 elif re.search('x-goog-meta-executable:', out): | 213 elif re.search('x-goog-meta-executable:', out): |
| 206 st = os.stat(output_filename) | 214 st = os.stat(output_filename) |
| 207 os.chmod(output_filename, st.st_mode | stat.S_IEXEC) | 215 os.chmod(output_filename, st.st_mode | stat.S_IEXEC) |
| 208 | 216 |
| 209 def printer_worker(output_queue): | 217 def printer_worker(output_queue): |
| 210 while True: | 218 while True: |
| (...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 386 return code | 394 return code |
| 387 | 395 |
| 388 return download_from_google_storage( | 396 return download_from_google_storage( |
| 389 input_filename, base_url, gsutil, options.num_threads, options.directory, | 397 input_filename, base_url, gsutil, options.num_threads, options.directory, |
| 390 options.recursive, options.force, options.output, options.ignore_errors, | 398 options.recursive, options.force, options.output, options.ignore_errors, |
| 391 options.sha1_file, options.verbose) | 399 options.sha1_file, options.verbose) |
| 392 | 400 |
| 393 | 401 |
| 394 if __name__ == '__main__': | 402 if __name__ == '__main__': |
| 395 sys.exit(main(sys.argv)) | 403 sys.exit(main(sys.argv)) |
| OLD | NEW |