| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # pylint: disable=C0301 | 3 # pylint: disable=C0301 |
| 4 """ | 4 """ |
| 5 Copyright 2014 Google Inc. | 5 Copyright 2014 Google Inc. |
| 6 | 6 |
| 7 Use of this source code is governed by a BSD-style license that can be | 7 Use of this source code is governed by a BSD-style license that can be |
| 8 found in the LICENSE file. | 8 found in the LICENSE file. |
| 9 | 9 |
| 10 Utilities for accessing Google Cloud Storage, using the boto library (wrapper | 10 Utilities for accessing Google Cloud Storage, using the boto library (wrapper |
| (...skipping 395 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 406 self.upload_file( | 406 self.upload_file( |
| 407 source_path=os.path.join(source_dir, rel_path), | 407 source_path=os.path.join(source_dir, rel_path), |
| 408 dest_bucket=b, | 408 dest_bucket=b, |
| 409 dest_path=posixpath.join(dest_dir, rel_path), | 409 dest_path=posixpath.join(dest_dir, rel_path), |
| 410 upload_if=self.UploadIf.ALWAYS, | 410 upload_if=self.UploadIf.ALWAYS, |
| 411 **kwargs) | 411 **kwargs) |
| 412 q.task_done() | 412 q.task_done() |
| 413 break | 413 break |
| 414 except Exception as error: | 414 except Exception as error: |
| 415 if retry < retries - 1: | 415 if retry < retries - 1: |
| 416 print ' Retrying upload, attempt #%d' % retry + 1 | 416 print ' Retrying upload, attempt #%d' % (retry + 1) |
| 417 time.sleep(2 ** retry) | 417 time.sleep(2 ** retry) |
| 418 else: | 418 else: |
| 419 err[rel_path] = error | 419 err[rel_path] = error |
| 420 | 420 |
| 421 for _ in range(num_threads): | 421 for _ in range(num_threads): |
| 422 t = threading.Thread(target=worker) | 422 t = threading.Thread(target=worker) |
| 423 t.daemon = True | 423 t.daemon = True |
| 424 t.start() | 424 t.start() |
| 425 | 425 |
| 426 # Block until all files have been uploaded and all workers have exited. | 426 # Block until all files have been uploaded and all workers have exited. |
| (...skipping 299 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 726 | 726 |
| 727 def _get_local_md5(path): | 727 def _get_local_md5(path): |
| 728 """Returns the MD5 hash of a file on local disk.""" | 728 """Returns the MD5 hash of a file on local disk.""" |
| 729 hasher = hashlib.md5() | 729 hasher = hashlib.md5() |
| 730 with open(path, 'rb') as f: | 730 with open(path, 'rb') as f: |
| 731 while True: | 731 while True: |
| 732 data = f.read(64*1024) | 732 data = f.read(64*1024) |
| 733 if not data: | 733 if not data: |
| 734 return hasher.hexdigest() | 734 return hasher.hexdigest() |
| 735 hasher.update(data) | 735 hasher.update(data) |
| OLD | NEW |