OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Uploads files to Google Storage content addressed.""" | 6 """Uploads files to Google Storage content addressed.""" |
7 | 7 |
8 import hashlib | 8 import hashlib |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import re | 12 import re |
13 import stat | 13 import stat |
14 import sys | 14 import sys |
15 import threading | 15 import threading |
16 import time | 16 import time |
17 | 17 |
18 from download_from_google_storage import check_bucket_permissions | 18 from download_from_google_storage import check_bucket_permissions |
19 from download_from_google_storage import get_sha1 | 19 from download_from_google_storage import get_sha1 |
20 from download_from_google_storage import Gsutil | 20 from download_from_google_storage import Gsutil |
21 from download_from_google_storage import printer_worker | 21 from download_from_google_storage import printer_worker |
22 | 22 from download_from_google_storage import GSUTIL_DEFAULT_PATH |
23 GSUTIL_DEFAULT_PATH = os.path.join( | |
24 os.path.dirname(os.path.abspath(__file__)), | |
25 'third_party', 'gsutil', 'gsutil') | |
26 | 23 |
27 USAGE_STRING = """%prog [options] target [target2 ...]. | 24 USAGE_STRING = """%prog [options] target [target2 ...]. |
28 Target is the file intended to be uploaded to Google Storage. | 25 Target is the file intended to be uploaded to Google Storage. |
29 If target is "-", then a list of files will be taken from standard input | 26 If target is "-", then a list of files will be taken from standard input |
30 | 27 |
31 This script will generate a file (original filename).sha1 containing the | 28 This script will generate a file (original filename).sha1 containing the |
32 sha1 sum of the uploaded file. | 29 sha1 sum of the uploaded file. |
33 It is recommended that the .sha1 file is checked into the repository, | 30 It is recommended that the .sha1 file is checked into the repository, |
34 the original file removed from the repository, and a hook added to the | 31 the original file removed from the repository, and a hook added to the |
35 DEPS file to call download_from_google_storage.py. | 32 DEPS file to call download_from_google_storage.py. |
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
89 local_md5 = get_md5_cached(filename) | 86 local_md5 = get_md5_cached(filename) |
90 else: | 87 else: |
91 local_md5 = get_md5(filename) | 88 local_md5 = get_md5(filename) |
92 if local_md5 == remote_md5: | 89 if local_md5 == remote_md5: |
93 stdout_queue.put( | 90 stdout_queue.put( |
94 '%d> File %s already exists and MD5 matches, upload skipped' % | 91 '%d> File %s already exists and MD5 matches, upload skipped' % |
95 (thread_num, filename)) | 92 (thread_num, filename)) |
96 continue | 93 continue |
97 stdout_queue.put('%d> Uploading %s...' % ( | 94 stdout_queue.put('%d> Uploading %s...' % ( |
98 thread_num, filename)) | 95 thread_num, filename)) |
99 code, _, err = gsutil.check_call('cp', '-q', filename, file_url) | 96 code, _, err = gsutil.check_call('cp', filename, file_url) |
100 if code != 0: | 97 if code != 0: |
101 ret_codes.put( | 98 ret_codes.put( |
102 (code, | 99 (code, |
103 'Encountered error on uploading %s to %s\n%s' % | 100 'Encountered error on uploading %s to %s\n%s' % |
104 (filename, file_url, err))) | 101 (filename, file_url, err))) |
105 continue | 102 continue |
106 | 103 |
107 # Mark executable files with the header "x-goog-meta-executable: 1" which | 104 # Mark executable files with the header "x-goog-meta-executable: 1" which |
108 # the download script will check for to preserve the executable bit. | 105 # the download script will check for to preserve the executable bit. |
109 if not sys.platform.startswith('win'): | 106 if not sys.platform.startswith('win'): |
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
227 help='Use \\0 instead of \\n when parsing ' | 224 help='Use \\0 instead of \\n when parsing ' |
228 'the file list from stdin. This is useful if the input ' | 225 'the file list from stdin. This is useful if the input ' |
229 'is coming from "find ... -print0".') | 226 'is coming from "find ... -print0".') |
230 (options, args) = parser.parse_args() | 227 (options, args) = parser.parse_args() |
231 | 228 |
232 # Enumerate our inputs. | 229 # Enumerate our inputs. |
233 input_filenames = get_targets(args, parser, options.use_null_terminator) | 230 input_filenames = get_targets(args, parser, options.use_null_terminator) |
234 | 231 |
235 # Make sure we can find a working instance of gsutil. | 232 # Make sure we can find a working instance of gsutil. |
236 if os.path.exists(GSUTIL_DEFAULT_PATH): | 233 if os.path.exists(GSUTIL_DEFAULT_PATH): |
237 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto, | 234 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
238 bypass_prodaccess=True) | |
239 else: | 235 else: |
240 gsutil = None | 236 gsutil = None |
241 for path in os.environ["PATH"].split(os.pathsep): | 237 for path in os.environ["PATH"].split(os.pathsep): |
242 if os.path.exists(path) and 'gsutil' in os.listdir(path): | 238 if os.path.exists(path) and 'gsutil' in os.listdir(path): |
243 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | 239 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) |
244 if not gsutil: | 240 if not gsutil: |
245 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 241 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
246 GSUTIL_DEFAULT_PATH) | 242 GSUTIL_DEFAULT_PATH) |
247 | 243 |
248 base_url = 'gs://%s' % options.bucket | 244 base_url = 'gs://%s' % options.bucket |
249 | 245 |
250 # Check we have a valid bucket with valid permissions. | 246 # Check we have a valid bucket with valid permissions. |
251 code = check_bucket_permissions(base_url, gsutil) | 247 code = check_bucket_permissions(base_url, gsutil) |
252 if code: | 248 if code: |
253 return code | 249 return code |
254 | 250 |
255 return upload_to_google_storage( | 251 return upload_to_google_storage( |
256 input_filenames, base_url, gsutil, options.force, options.use_md5, | 252 input_filenames, base_url, gsutil, options.force, options.use_md5, |
257 options.num_threads, options.skip_hashing) | 253 options.num_threads, options.skip_hashing) |
258 | 254 |
259 | 255 |
260 if __name__ == '__main__': | 256 if __name__ == '__main__': |
261 sys.exit(main(sys.argv)) | 257 sys.exit(main(sys.argv)) |
OLD | NEW |