OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Uploads files to Google Storage content addressed.""" | 6 """Uploads files to Google Storage content addressed.""" |
7 | 7 |
8 import hashlib | 8 import hashlib |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import re | 12 import re |
13 import stat | 13 import stat |
14 import sys | 14 import sys |
15 import tarfile | |
16 import threading | 15 import threading |
17 import time | 16 import time |
18 | 17 |
19 from download_from_google_storage import get_sha1 | 18 from download_from_google_storage import get_sha1 |
20 from download_from_google_storage import Gsutil | 19 from download_from_google_storage import Gsutil |
21 from download_from_google_storage import printer_worker | 20 from download_from_google_storage import printer_worker |
22 from download_from_google_storage import GSUTIL_DEFAULT_PATH | 21 from download_from_google_storage import GSUTIL_DEFAULT_PATH |
23 | 22 |
24 USAGE_STRING = """%prog [options] target [target2 ...]. | 23 USAGE_STRING = """%prog [options] target [target2 ...]. |
25 Target is the file intended to be uploaded to Google Storage. | 24 Target is the file intended to be uploaded to Google Storage. |
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
201 max_ret_code = max(ret_code, max_ret_code) | 200 max_ret_code = max(ret_code, max_ret_code) |
202 if message: | 201 if message: |
203 print >> sys.stderr, message | 202 print >> sys.stderr, message |
204 | 203 |
205 if not max_ret_code: | 204 if not max_ret_code: |
206 print 'Success!' | 205 print 'Success!' |
207 | 206 |
208 return max_ret_code | 207 return max_ret_code |
209 | 208 |
210 | 209 |
211 def create_archives(dirs): | |
212 archive_names = [] | |
213 for name in dirs: | |
214 tarname = '%s.tar.gz' % name | |
215 with tarfile.open(tarname, 'w:gz') as tar: | |
216 tar.add(name) | |
217 archive_names.append(tarname) | |
218 return archive_names | |
219 | |
220 | |
221 def validate_archive_dirs(dirs): | |
222 # We don't allow .. in paths in our archives. | |
223 if any(map(lambda x: '..' in x, dirs)): | |
224 return False | |
225 # We only allow dirs. | |
226 if any(map(lambda x: not os.path.isdir(x), dirs)): | |
227 return False | |
228 # We don't allow sym links in our archives. | |
229 if any(map(os.path.islink, dirs)): | |
230 return False | |
231 # We required that the subdirectories we are archiving are all just below | |
232 # cwd. | |
233 return not any(map(lambda x: x not in next(os.walk('.'))[1], dirs)) | |
234 | |
235 | |
236 def main(): | 210 def main(): |
237 parser = optparse.OptionParser(USAGE_STRING) | 211 parser = optparse.OptionParser(USAGE_STRING) |
238 parser.add_option('-b', '--bucket', | 212 parser.add_option('-b', '--bucket', |
239 help='Google Storage bucket to upload to.') | 213 help='Google Storage bucket to upload to.') |
240 parser.add_option('-e', '--boto', help='Specify a custom boto file.') | 214 parser.add_option('-e', '--boto', help='Specify a custom boto file.') |
241 parser.add_option('-z', '--archive', action='store_true', | |
242 help='Archive directory as a tar.gz file') | |
243 parser.add_option('-f', '--force', action='store_true', | 215 parser.add_option('-f', '--force', action='store_true', |
244 help='Force upload even if remote file exists.') | 216 help='Force upload even if remote file exists.') |
245 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, | 217 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, |
246 help='Path to the gsutil script.') | 218 help='Path to the gsutil script.') |
247 parser.add_option('-m', '--use_md5', action='store_true', | 219 parser.add_option('-m', '--use_md5', action='store_true', |
248 help='Generate MD5 files when scanning, and don\'t check ' | 220 help='Generate MD5 files when scanning, and don\'t check ' |
249 'the MD5 checksum if a .md5 file is found.') | 221 'the MD5 checksum if a .md5 file is found.') |
250 parser.add_option('-t', '--num_threads', default=1, type='int', | 222 parser.add_option('-t', '--num_threads', default=1, type='int', |
251 help='Number of uploader threads to run.') | 223 help='Number of uploader threads to run.') |
252 parser.add_option('-s', '--skip_hashing', action='store_true', | 224 parser.add_option('-s', '--skip_hashing', action='store_true', |
253 help='Skip hashing if .sha1 file exists.') | 225 help='Skip hashing if .sha1 file exists.') |
254 parser.add_option('-0', '--use_null_terminator', action='store_true', | 226 parser.add_option('-0', '--use_null_terminator', action='store_true', |
255 help='Use \\0 instead of \\n when parsing ' | 227 help='Use \\0 instead of \\n when parsing ' |
256 'the file list from stdin. This is useful if the input ' | 228 'the file list from stdin. This is useful if the input ' |
257 'is coming from "find ... -print0".') | 229 'is coming from "find ... -print0".') |
258 parser.add_option('-z', '--gzip', metavar='ext', | 230 parser.add_option('-z', '--gzip', metavar='ext', |
259 help='Gzip files which end in ext. ' | 231 help='Gzip files which end in ext. ' |
260 'ext is a comma-separated list') | 232 'ext is a comma-separated list') |
261 (options, args) = parser.parse_args() | 233 (options, args) = parser.parse_args() |
262 | 234 |
263 # Enumerate our inputs. | 235 # Enumerate our inputs. |
264 input_filenames = get_targets(args, parser, options.use_null_terminator) | 236 input_filenames = get_targets(args, parser, options.use_null_terminator) |
265 | 237 |
266 if options.archive: | |
267 if not validate_archive_dirs(input_filenames): | |
268 parser.error('Only directories just below cwd are valid entries when ' | |
269 'using the --archive argument. Entries can not contain .. ' | |
270 ' and entries can not be symlinks. Entries was %s' % | |
271 input_filenames) | |
272 return 1 | |
273 input_filenames = create_archives(input_filenames) | |
274 | |
275 # Make sure we can find a working instance of gsutil. | 238 # Make sure we can find a working instance of gsutil. |
276 if os.path.exists(GSUTIL_DEFAULT_PATH): | 239 if os.path.exists(GSUTIL_DEFAULT_PATH): |
277 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) | 240 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
278 else: | 241 else: |
279 gsutil = None | 242 gsutil = None |
280 for path in os.environ["PATH"].split(os.pathsep): | 243 for path in os.environ["PATH"].split(os.pathsep): |
281 if os.path.exists(path) and 'gsutil' in os.listdir(path): | 244 if os.path.exists(path) and 'gsutil' in os.listdir(path): |
282 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | 245 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) |
283 if not gsutil: | 246 if not gsutil: |
284 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 247 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
285 GSUTIL_DEFAULT_PATH) | 248 GSUTIL_DEFAULT_PATH) |
286 | 249 |
287 base_url = 'gs://%s' % options.bucket | 250 base_url = 'gs://%s' % options.bucket |
288 | 251 |
289 return upload_to_google_storage( | 252 return upload_to_google_storage( |
290 input_filenames, base_url, gsutil, options.force, options.use_md5, | 253 input_filenames, base_url, gsutil, options.force, options.use_md5, |
291 options.num_threads, options.skip_hashing, options.gzip) | 254 options.num_threads, options.skip_hashing, options.gzip) |
292 | 255 |
293 | 256 |
294 if __name__ == '__main__': | 257 if __name__ == '__main__': |
295 try: | 258 try: |
296 sys.exit(main()) | 259 sys.exit(main()) |
297 except KeyboardInterrupt: | 260 except KeyboardInterrupt: |
298 sys.stderr.write('interrupted\n') | 261 sys.stderr.write('interrupted\n') |
299 sys.exit(1) | 262 sys.exit(1) |
OLD | NEW |