OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Uploads files to Google Storage content addressed.""" | 6 """Uploads files to Google Storage content addressed.""" |
7 | 7 |
8 import hashlib | 8 import hashlib |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import re | 12 import re |
13 import stat | 13 import stat |
14 import sys | 14 import sys |
15 import tarfile | |
15 import threading | 16 import threading |
16 import time | 17 import time |
17 | 18 |
18 from download_from_google_storage import get_sha1 | 19 from download_from_google_storage import get_sha1 |
19 from download_from_google_storage import Gsutil | 20 from download_from_google_storage import Gsutil |
20 from download_from_google_storage import printer_worker | 21 from download_from_google_storage import printer_worker |
21 from download_from_google_storage import GSUTIL_DEFAULT_PATH | 22 from download_from_google_storage import GSUTIL_DEFAULT_PATH |
22 | 23 |
23 USAGE_STRING = """%prog [options] target [target2 ...]. | 24 USAGE_STRING = """%prog [options] target [target2 ...]. |
24 Target is the file intended to be uploaded to Google Storage. | 25 Target is the file intended to be uploaded to Google Storage. |
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
200 max_ret_code = max(ret_code, max_ret_code) | 201 max_ret_code = max(ret_code, max_ret_code) |
201 if message: | 202 if message: |
202 print >> sys.stderr, message | 203 print >> sys.stderr, message |
203 | 204 |
204 if not max_ret_code: | 205 if not max_ret_code: |
205 print 'Success!' | 206 print 'Success!' |
206 | 207 |
207 return max_ret_code | 208 return max_ret_code |
208 | 209 |
209 | 210 |
211 def create_archives(dirs): | |
212 archive_names = [] | |
213 for name in dirs: | |
214 tarname = '%s.tar.gz' % name | |
215 with tarfile.open(tarname, 'w:gz') as tar: | |
216 tar.add(name) | |
217 archive_names.append(tarname) | |
218 return archive_names | |
219 | |
220 | |
221 def validate_archive_dirs(dirs): | |
222 # We don't allow .. in paths in our archives. | |
223 if any(map(lambda x: '..' in x, dirs)): | |
224 return False | |
225 # We only allow dirs. | |
226 if any(map(lambda x: not os.path.isdir(x), dirs)): | |
227 return False | |
228 # We don't allow sym links in our archives. | |
229 if any(map(os.path.islink, dirs)): | |
230 return False | |
231 # We required that the subdirectories we are archiving are all just below | |
232 # cwd. | |
233 return not any(map(lambda x: x not in next(os.walk('.'))[1], dirs)) | |
234 | |
235 | |
210 def main(): | 236 def main(): |
211 parser = optparse.OptionParser(USAGE_STRING) | 237 parser = optparse.OptionParser(USAGE_STRING) |
212 parser.add_option('-b', '--bucket', | 238 parser.add_option('-b', '--bucket', |
213 help='Google Storage bucket to upload to.') | 239 help='Google Storage bucket to upload to.') |
214 parser.add_option('-e', '--boto', help='Specify a custom boto file.') | 240 parser.add_option('-e', '--boto', help='Specify a custom boto file.') |
241 parser.add_option('-z', '--archive', action='store_true', | |
242 help='Archive directory as a tar.gz file') | |
azarchs
2015/06/30 14:49:37
This conflicts with (and duplicates?) the -z optio
| |
215 parser.add_option('-f', '--force', action='store_true', | 243 parser.add_option('-f', '--force', action='store_true', |
216 help='Force upload even if remote file exists.') | 244 help='Force upload even if remote file exists.') |
217 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, | 245 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, |
218 help='Path to the gsutil script.') | 246 help='Path to the gsutil script.') |
219 parser.add_option('-m', '--use_md5', action='store_true', | 247 parser.add_option('-m', '--use_md5', action='store_true', |
220 help='Generate MD5 files when scanning, and don\'t check ' | 248 help='Generate MD5 files when scanning, and don\'t check ' |
221 'the MD5 checksum if a .md5 file is found.') | 249 'the MD5 checksum if a .md5 file is found.') |
222 parser.add_option('-t', '--num_threads', default=1, type='int', | 250 parser.add_option('-t', '--num_threads', default=1, type='int', |
223 help='Number of uploader threads to run.') | 251 help='Number of uploader threads to run.') |
224 parser.add_option('-s', '--skip_hashing', action='store_true', | 252 parser.add_option('-s', '--skip_hashing', action='store_true', |
225 help='Skip hashing if .sha1 file exists.') | 253 help='Skip hashing if .sha1 file exists.') |
226 parser.add_option('-0', '--use_null_terminator', action='store_true', | 254 parser.add_option('-0', '--use_null_terminator', action='store_true', |
227 help='Use \\0 instead of \\n when parsing ' | 255 help='Use \\0 instead of \\n when parsing ' |
228 'the file list from stdin. This is useful if the input ' | 256 'the file list from stdin. This is useful if the input ' |
229 'is coming from "find ... -print0".') | 257 'is coming from "find ... -print0".') |
230 parser.add_option('-z', '--gzip', metavar='ext', | 258 parser.add_option('-z', '--gzip', metavar='ext', |
231 help='Gzip files which end in ext. ' | 259 help='Gzip files which end in ext. ' |
232 'ext is a comma-separated list') | 260 'ext is a comma-separated list') |
233 (options, args) = parser.parse_args() | 261 (options, args) = parser.parse_args() |
234 | 262 |
235 # Enumerate our inputs. | 263 # Enumerate our inputs. |
236 input_filenames = get_targets(args, parser, options.use_null_terminator) | 264 input_filenames = get_targets(args, parser, options.use_null_terminator) |
237 | 265 |
266 if options.archive: | |
267 if not validate_archive_dirs(input_filenames): | |
268 parser.error('Only directories just below cwd are valid entries when ' | |
269 'using the --archive argument. Entries can not contain .. ' | |
270 ' and entries can not be symlinks. Entries was %s' % | |
271 input_filenames) | |
272 return 1 | |
273 input_filenames = create_archives(input_filenames) | |
274 | |
238 # Make sure we can find a working instance of gsutil. | 275 # Make sure we can find a working instance of gsutil. |
239 if os.path.exists(GSUTIL_DEFAULT_PATH): | 276 if os.path.exists(GSUTIL_DEFAULT_PATH): |
240 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) | 277 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
241 else: | 278 else: |
242 gsutil = None | 279 gsutil = None |
243 for path in os.environ["PATH"].split(os.pathsep): | 280 for path in os.environ["PATH"].split(os.pathsep): |
244 if os.path.exists(path) and 'gsutil' in os.listdir(path): | 281 if os.path.exists(path) and 'gsutil' in os.listdir(path): |
245 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | 282 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) |
246 if not gsutil: | 283 if not gsutil: |
247 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 284 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
248 GSUTIL_DEFAULT_PATH) | 285 GSUTIL_DEFAULT_PATH) |
249 | 286 |
250 base_url = 'gs://%s' % options.bucket | 287 base_url = 'gs://%s' % options.bucket |
251 | 288 |
252 return upload_to_google_storage( | 289 return upload_to_google_storage( |
253 input_filenames, base_url, gsutil, options.force, options.use_md5, | 290 input_filenames, base_url, gsutil, options.force, options.use_md5, |
254 options.num_threads, options.skip_hashing, options.gzip) | 291 options.num_threads, options.skip_hashing, options.gzip) |
255 | 292 |
256 | 293 |
257 if __name__ == '__main__': | 294 if __name__ == '__main__': |
258 try: | 295 try: |
259 sys.exit(main()) | 296 sys.exit(main()) |
260 except KeyboardInterrupt: | 297 except KeyboardInterrupt: |
261 sys.stderr.write('interrupted\n') | 298 sys.stderr.write('interrupted\n') |
262 sys.exit(1) | 299 sys.exit(1) |
OLD | NEW |