OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Uploads files to Google Storage content addressed.""" | 6 """Uploads files to Google Storage content addressed.""" |
7 | 7 |
8 import hashlib | 8 import hashlib |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import re | 12 import re |
13 import stat | 13 import stat |
14 import sys | 14 import sys |
15 import tarfile | |
15 import threading | 16 import threading |
16 import time | 17 import time |
17 | 18 |
18 from download_from_google_storage import check_bucket_permissions | 19 from download_from_google_storage import check_bucket_permissions |
19 from download_from_google_storage import get_sha1 | 20 from download_from_google_storage import get_sha1 |
20 from download_from_google_storage import Gsutil | 21 from download_from_google_storage import Gsutil |
21 from download_from_google_storage import printer_worker | 22 from download_from_google_storage import printer_worker |
22 from download_from_google_storage import GSUTIL_DEFAULT_PATH | 23 from download_from_google_storage import GSUTIL_DEFAULT_PATH |
23 | 24 |
24 USAGE_STRING = """%prog [options] target [target2 ...]. | 25 USAGE_STRING = """%prog [options] target [target2 ...]. |
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
123 if use_null_terminator: | 124 if use_null_terminator: |
124 return sys.stdin.read().split('\0') | 125 return sys.stdin.read().split('\0') |
125 else: | 126 else: |
126 return sys.stdin.read().splitlines() | 127 return sys.stdin.read().splitlines() |
127 else: | 128 else: |
128 return args | 129 return args |
129 | 130 |
130 | 131 |
131 def upload_to_google_storage( | 132 def upload_to_google_storage( |
132 input_filenames, base_url, gsutil, force, | 133 input_filenames, base_url, gsutil, force, |
133 use_md5, num_threads, skip_hashing): | 134 use_md5, num_threads, skip_hashing, archive): |
134 # We only want one MD5 calculation happening at a time to avoid HD thrashing. | 135 # We only want one MD5 calculation happening at a time to avoid HD thrashing. |
135 md5_lock = threading.Lock() | 136 md5_lock = threading.Lock() |
136 | 137 |
137 # Start up all the worker threads plus the printer thread. | 138 # Start up all the worker threads plus the printer thread. |
138 all_threads = [] | 139 all_threads = [] |
139 ret_codes = Queue.Queue() | 140 ret_codes = Queue.Queue() |
140 ret_codes.put((0, None)) | 141 ret_codes.put((0, None)) |
141 upload_queue = Queue.Queue() | 142 upload_queue = Queue.Queue() |
142 upload_timer = time.time() | 143 upload_timer = time.time() |
143 stdout_queue = Queue.Queue() | 144 stdout_queue = Queue.Queue() |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
196 for ret_code, message in ret_codes.queue: | 197 for ret_code, message in ret_codes.queue: |
197 max_ret_code = max(ret_code, max_ret_code) | 198 max_ret_code = max(ret_code, max_ret_code) |
198 if message: | 199 if message: |
199 print >> sys.stderr, message | 200 print >> sys.stderr, message |
200 | 201 |
201 if not max_ret_code: | 202 if not max_ret_code: |
202 print 'Success!' | 203 print 'Success!' |
203 | 204 |
204 return max_ret_code | 205 return max_ret_code |
205 | 206 |
207 def create_archives(dirs): | |
208 archive_names = [] | |
209 for name in dirs: | |
210 tarname = '%s.tar.gz' % name | |
211 with tarfile.open(tarname, 'w:gz') as tar: | |
212 tar.add(name) | |
213 archive_names.append(tarname) | |
214 return archive_names | |
215 | |
216 def validate_archive_dirs(dirs): | |
217 # We don't allow .. in paths in our archives. | |
hinoka
2015/02/11 00:16:57
Each of these deserve their own error message.
| |
218 if any(map(lambda x: '..' in x, dirs)): | |
219 return True | |
220 # We only allow dirs. | |
221 if any(map(lambda x: not os.path.isdir(x), dirs)): | |
222 return True | |
223 # We don't allow sym links in our archives. | |
224 if any(map(os.path.islink, dirs)): | |
225 return True | |
226 # We required that the subdirectories we are archiving are all just below | |
227 # cwd. | |
228 return any(map(lambda x: x not in next(os.walk('.'))[1], dirs)) | |
206 | 229 |
207 def main(args): | 230 def main(args): |
208 parser = optparse.OptionParser(USAGE_STRING) | 231 parser = optparse.OptionParser(USAGE_STRING) |
209 parser.add_option('-b', '--bucket', | 232 parser.add_option('-b', '--bucket', |
210 help='Google Storage bucket to upload to.') | 233 help='Google Storage bucket to upload to.') |
211 parser.add_option('-e', '--boto', help='Specify a custom boto file.') | 234 parser.add_option('-e', '--boto', help='Specify a custom boto file.') |
235 parser.add_option('-z', '--archive', action='store_true', | |
236 help='Archive directory as a tar.gz file') | |
212 parser.add_option('-f', '--force', action='store_true', | 237 parser.add_option('-f', '--force', action='store_true', |
213 help='Force upload even if remote file exists.') | 238 help='Force upload even if remote file exists.') |
214 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, | |
215 help='Path to the gsutil script.') | |
216 parser.add_option('-m', '--use_md5', action='store_true', | 239 parser.add_option('-m', '--use_md5', action='store_true', |
217 help='Generate MD5 files when scanning, and don\'t check ' | 240 help='Generate MD5 files when scanning, and don\'t check ' |
218 'the MD5 checksum if a .md5 file is found.') | 241 'the MD5 checksum if a .md5 file is found.') |
219 parser.add_option('-t', '--num_threads', default=1, type='int', | 242 parser.add_option('-t', '--num_threads', default=1, type='int', |
220 help='Number of uploader threads to run.') | 243 help='Number of uploader threads to run.') |
221 parser.add_option('-s', '--skip_hashing', action='store_true', | 244 parser.add_option('-s', '--skip_hashing', action='store_true', |
222 help='Skip hashing if .sha1 file exists.') | 245 help='Skip hashing if .sha1 file exists.') |
223 parser.add_option('-0', '--use_null_terminator', action='store_true', | 246 parser.add_option('-0', '--use_null_terminator', action='store_true', |
224 help='Use \\0 instead of \\n when parsing ' | 247 help='Use \\0 instead of \\n when parsing ' |
225 'the file list from stdin. This is useful if the input ' | 248 'the file list from stdin. This is useful if the input ' |
226 'is coming from "find ... -print0".') | 249 'is coming from "find ... -print0".') |
227 (options, args) = parser.parse_args() | 250 (options, args) = parser.parse_args() |
228 | 251 |
229 # Enumerate our inputs. | 252 # Enumerate our inputs. |
230 input_filenames = get_targets(args, parser, options.use_null_terminator) | 253 input_filenames = get_targets(args, parser, options.use_null_terminator) |
231 | 254 |
255 | |
256 if options.archive: | |
257 if validate_archive_dirs(input_filenames): | |
258 parser.error('Only directories just below cwd are valid entries when ' | |
259 'using the --archive argument. Entries can not contain .. ' | |
260 ' and entries can not be symlinks. Entries was %s' % | |
261 input_filenames) | |
262 return 1 | |
263 input_filenames = create_archives(input_filenames) | |
264 | |
232 # Make sure we can find a working instance of gsutil. | 265 # Make sure we can find a working instance of gsutil. |
233 if os.path.exists(GSUTIL_DEFAULT_PATH): | 266 if os.path.exists(GSUTIL_DEFAULT_PATH): |
234 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) | 267 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
235 else: | 268 else: |
236 gsutil = None | 269 gsutil = None |
237 for path in os.environ["PATH"].split(os.pathsep): | 270 for path in os.environ["PATH"].split(os.pathsep): |
238 if os.path.exists(path) and 'gsutil' in os.listdir(path): | 271 if os.path.exists(path) and 'gsutil' in os.listdir(path): |
239 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | 272 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) |
240 if not gsutil: | 273 if not gsutil: |
241 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 274 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
242 GSUTIL_DEFAULT_PATH) | 275 GSUTIL_DEFAULT_PATH) |
243 | 276 |
244 base_url = 'gs://%s' % options.bucket | 277 base_url = 'gs://%s' % options.bucket |
245 | 278 |
246 # Check we have a valid bucket with valid permissions. | 279 # Check we have a valid bucket with valid permissions. |
247 code = check_bucket_permissions(base_url, gsutil) | 280 code = check_bucket_permissions(base_url, gsutil) |
248 if code: | 281 if code: |
249 return code | 282 return code |
250 | 283 |
251 return upload_to_google_storage( | 284 return upload_to_google_storage( |
252 input_filenames, base_url, gsutil, options.force, options.use_md5, | 285 input_filenames, base_url, gsutil, options.force, options.use_md5, |
253 options.num_threads, options.skip_hashing) | 286 options.num_threads, options.skip_hashing, options.archive) |
254 | 287 |
255 | 288 |
256 if __name__ == '__main__': | 289 if __name__ == '__main__': |
257 sys.exit(main(sys.argv)) | 290 sys.exit(main(sys.argv)) |
OLD | NEW |