| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Uploads files to Google Storage content addressed.""" | 6 """Uploads files to Google Storage content addressed.""" |
| 7 | 7 |
| 8 import hashlib | 8 import hashlib |
| 9 import optparse | 9 import optparse |
| 10 import os | 10 import os |
| 11 import Queue | 11 import Queue |
| 12 import re | 12 import re |
| 13 import stat | 13 import stat |
| 14 import sys | 14 import sys |
| 15 import threading | 15 import threading |
| 16 import time | 16 import time |
| 17 | 17 |
| 18 from download_from_google_storage import check_bucket_permissions | 18 from download_from_google_storage import check_bucket_permissions |
| 19 from download_from_google_storage import get_sha1 | 19 from download_from_google_storage import get_sha1 |
| 20 from download_from_google_storage import Gsutil | 20 from download_from_google_storage import Gsutil |
| 21 from download_from_google_storage import printer_worker | 21 from download_from_google_storage import printer_worker |
| 22 | 22 from download_from_google_storage import GSUTIL_DEFAULT_PATH |
| 23 GSUTIL_DEFAULT_PATH = os.path.join( | |
| 24 os.path.dirname(os.path.abspath(__file__)), | |
| 25 'third_party', 'gsutil', 'gsutil') | |
| 26 | 23 |
| 27 USAGE_STRING = """%prog [options] target [target2 ...]. | 24 USAGE_STRING = """%prog [options] target [target2 ...]. |
| 28 Target is the file intended to be uploaded to Google Storage. | 25 Target is the file intended to be uploaded to Google Storage. |
| 29 If target is "-", then a list of files will be taken from standard input | 26 If target is "-", then a list of files will be taken from standard input |
| 30 | 27 |
| 31 This script will generate a file (original filename).sha1 containing the | 28 This script will generate a file (original filename).sha1 containing the |
| 32 sha1 sum of the uploaded file. | 29 sha1 sum of the uploaded file. |
| 33 It is recommended that the .sha1 file is checked into the repository, | 30 It is recommended that the .sha1 file is checked into the repository, |
| 34 the original file removed from the repository, and a hook added to the | 31 the original file removed from the repository, and a hook added to the |
| 35 DEPS file to call download_from_google_storage.py. | 32 DEPS file to call download_from_google_storage.py. |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 89 local_md5 = get_md5_cached(filename) | 86 local_md5 = get_md5_cached(filename) |
| 90 else: | 87 else: |
| 91 local_md5 = get_md5(filename) | 88 local_md5 = get_md5(filename) |
| 92 if local_md5 == remote_md5: | 89 if local_md5 == remote_md5: |
| 93 stdout_queue.put( | 90 stdout_queue.put( |
| 94 '%d> File %s already exists and MD5 matches, upload skipped' % | 91 '%d> File %s already exists and MD5 matches, upload skipped' % |
| 95 (thread_num, filename)) | 92 (thread_num, filename)) |
| 96 continue | 93 continue |
| 97 stdout_queue.put('%d> Uploading %s...' % ( | 94 stdout_queue.put('%d> Uploading %s...' % ( |
| 98 thread_num, filename)) | 95 thread_num, filename)) |
| 99 code, _, err = gsutil.check_call('cp', '-q', filename, file_url) | 96 code, _, err = gsutil.check_call('cp', filename, file_url) |
| 100 if code != 0: | 97 if code != 0: |
| 101 ret_codes.put( | 98 ret_codes.put( |
| 102 (code, | 99 (code, |
| 103 'Encountered error on uploading %s to %s\n%s' % | 100 'Encountered error on uploading %s to %s\n%s' % |
| 104 (filename, file_url, err))) | 101 (filename, file_url, err))) |
| 105 continue | 102 continue |
| 106 | 103 |
| 107 # Mark executable files with the header "x-goog-meta-executable: 1" which | 104 # Mark executable files with the header "x-goog-meta-executable: 1" which |
| 108 # the download script will check for to preserve the executable bit. | 105 # the download script will check for to preserve the executable bit. |
| 109 if not sys.platform.startswith('win'): | 106 if not sys.platform.startswith('win'): |
| (...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 227 help='Use \\0 instead of \\n when parsing ' | 224 help='Use \\0 instead of \\n when parsing ' |
| 228 'the file list from stdin. This is useful if the input ' | 225 'the file list from stdin. This is useful if the input ' |
| 229 'is coming from "find ... -print0".') | 226 'is coming from "find ... -print0".') |
| 230 (options, args) = parser.parse_args() | 227 (options, args) = parser.parse_args() |
| 231 | 228 |
| 232 # Enumerate our inputs. | 229 # Enumerate our inputs. |
| 233 input_filenames = get_targets(args, parser, options.use_null_terminator) | 230 input_filenames = get_targets(args, parser, options.use_null_terminator) |
| 234 | 231 |
| 235 # Make sure we can find a working instance of gsutil. | 232 # Make sure we can find a working instance of gsutil. |
| 236 if os.path.exists(GSUTIL_DEFAULT_PATH): | 233 if os.path.exists(GSUTIL_DEFAULT_PATH): |
| 237 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto, | 234 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
| 238 bypass_prodaccess=True) | |
| 239 else: | 235 else: |
| 240 gsutil = None | 236 gsutil = None |
| 241 for path in os.environ["PATH"].split(os.pathsep): | 237 for path in os.environ["PATH"].split(os.pathsep): |
| 242 if os.path.exists(path) and 'gsutil' in os.listdir(path): | 238 if os.path.exists(path) and 'gsutil' in os.listdir(path): |
| 243 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | 239 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) |
| 244 if not gsutil: | 240 if not gsutil: |
| 245 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 241 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
| 246 GSUTIL_DEFAULT_PATH) | 242 GSUTIL_DEFAULT_PATH) |
| 247 | 243 |
| 248 base_url = 'gs://%s' % options.bucket | 244 base_url = 'gs://%s' % options.bucket |
| 249 | 245 |
| 250 # Check we have a valid bucket with valid permissions. | 246 # Check we have a valid bucket with valid permissions. |
| 251 code = check_bucket_permissions(base_url, gsutil) | 247 code = check_bucket_permissions(base_url, gsutil) |
| 252 if code: | 248 if code: |
| 253 return code | 249 return code |
| 254 | 250 |
| 255 return upload_to_google_storage( | 251 return upload_to_google_storage( |
| 256 input_filenames, base_url, gsutil, options.force, options.use_md5, | 252 input_filenames, base_url, gsutil, options.force, options.use_md5, |
| 257 options.num_threads, options.skip_hashing) | 253 options.num_threads, options.skip_hashing) |
| 258 | 254 |
| 259 | 255 |
| 260 if __name__ == '__main__': | 256 if __name__ == '__main__': |
| 261 sys.exit(main(sys.argv)) | 257 sys.exit(main(sys.argv)) |
| OLD | NEW |