OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 # pylint: disable=C0301 | 3 # pylint: disable=C0301 |
4 """ | 4 """ |
5 Copyright 2014 Google Inc. | 5 Copyright 2014 Google Inc. |
6 | 6 |
7 Use of this source code is governed by a BSD-style license that can be | 7 Use of this source code is governed by a BSD-style license that can be |
8 found in the LICENSE file. | 8 found in the LICENSE file. |
9 | 9 |
10 Utilities for accessing Google Cloud Storage, using the boto library (wrapper | 10 Utilities for accessing Google Cloud Storage, using the boto library (wrapper |
(...skipping 395 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
406 self.upload_file( | 406 self.upload_file( |
407 source_path=os.path.join(source_dir, rel_path), | 407 source_path=os.path.join(source_dir, rel_path), |
408 dest_bucket=b, | 408 dest_bucket=b, |
409 dest_path=posixpath.join(dest_dir, rel_path), | 409 dest_path=posixpath.join(dest_dir, rel_path), |
410 upload_if=self.UploadIf.ALWAYS, | 410 upload_if=self.UploadIf.ALWAYS, |
411 **kwargs) | 411 **kwargs) |
412 q.task_done() | 412 q.task_done() |
413 break | 413 break |
414 except Exception as error: | 414 except Exception as error: |
415 if retry < retries - 1: | 415 if retry < retries - 1: |
416 print ' Retrying upload, attempt #%d' % retry + 1 | 416 print ' Retrying upload, attempt #%d' % (retry + 1) |
417 time.sleep(2 ** retry) | 417 time.sleep(2 ** retry) |
418 else: | 418 else: |
419 err[rel_path] = error | 419 err[rel_path] = error |
420 | 420 |
421 for _ in range(num_threads): | 421 for _ in range(num_threads): |
422 t = threading.Thread(target=worker) | 422 t = threading.Thread(target=worker) |
423 t.daemon = True | 423 t.daemon = True |
424 t.start() | 424 t.start() |
425 | 425 |
426 # Block until all files have been uploaded and all workers have exited. | 426 # Block until all files have been uploaded and all workers have exited. |
(...skipping 299 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
726 | 726 |
727 def _get_local_md5(path): | 727 def _get_local_md5(path): |
728 """Returns the MD5 hash of a file on local disk.""" | 728 """Returns the MD5 hash of a file on local disk.""" |
729 hasher = hashlib.md5() | 729 hasher = hashlib.md5() |
730 with open(path, 'rb') as f: | 730 with open(path, 'rb') as f: |
731 while True: | 731 while True: |
732 data = f.read(64*1024) | 732 data = f.read(64*1024) |
733 if not data: | 733 if not data: |
734 return hasher.hexdigest() | 734 return hasher.hexdigest() |
735 hasher.update(data) | 735 hasher.update(data) |
OLD | NEW |