OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Uploads files to Google Storage content addressed.""" | 6 """Uploads files to Google Storage content addressed.""" |
7 | 7 |
8 import hashlib | 8 import hashlib |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import re | 12 import re |
13 import stat | 13 import stat |
14 import sys | 14 import sys |
15 import tarfile | |
15 import threading | 16 import threading |
16 import time | 17 import time |
17 | 18 |
18 from download_from_google_storage import get_sha1 | 19 from download_from_google_storage import get_sha1 |
19 from download_from_google_storage import Gsutil | 20 from download_from_google_storage import Gsutil |
20 from download_from_google_storage import printer_worker | 21 from download_from_google_storage import printer_worker |
21 from download_from_google_storage import GSUTIL_DEFAULT_PATH | 22 from download_from_google_storage import GSUTIL_DEFAULT_PATH |
22 | 23 |
23 USAGE_STRING = """%prog [options] target [target2 ...]. | 24 USAGE_STRING = """%prog [options] target [target2 ...]. |
24 Target is the file intended to be uploaded to Google Storage. | 25 Target is the file intended to be uploaded to Google Storage. |
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
198 max_ret_code = 0 | 199 max_ret_code = 0 |
199 for ret_code, message in ret_codes.queue: | 200 for ret_code, message in ret_codes.queue: |
200 max_ret_code = max(ret_code, max_ret_code) | 201 max_ret_code = max(ret_code, max_ret_code) |
201 if message: | 202 if message: |
202 print >> sys.stderr, message | 203 print >> sys.stderr, message |
203 | 204 |
204 if not max_ret_code: | 205 if not max_ret_code: |
205 print 'Success!' | 206 print 'Success!' |
206 | 207 |
207 return max_ret_code | 208 return max_ret_code |
208 | 209 |
Ryan Tseng
2015/06/25 17:19:14
top level function, 2 newlines
ricow1
2015/06/26 06:18:05
Done.
| |
210 def create_archives(dirs): | |
211 archive_names = [] | |
212 for name in dirs: | |
213 tarname = '%s.tar.gz' % name | |
214 with tarfile.open(tarname, 'w:gz') as tar: | |
215 tar.add(name) | |
216 archive_names.append(tarname) | |
217 return archive_names | |
218 | |
Ryan Tseng
2015/06/25 17:19:14
2 newlines
ricow1
2015/06/26 06:18:05
Done.
| |
219 def validate_archive_dirs(dirs): | |
220 # We don't allow .. in paths in our archives. | |
221 if any(map(lambda x: '..' in x, dirs)): | |
222 return False | |
223 # We only allow dirs. | |
224 if any(map(lambda x: not os.path.isdir(x), dirs)): | |
225 return False | |
226 # We don't allow sym links in our archives. | |
227 if any(map(os.path.islink, dirs)): | |
228 return False | |
229 # We required that the subdirectories we are archiving are all just below | |
230 # cwd. | |
231 return not any(map(lambda x: x not in next(os.walk('.'))[1], dirs)) | |
232 | |
Ryan Tseng
2015/06/25 17:19:14
remove 1 newline so there are 2 total
ricow1
2015/06/26 06:18:05
Done.
| |
233 | |
209 | 234 |
210 def main(): | 235 def main(): |
211 parser = optparse.OptionParser(USAGE_STRING) | 236 parser = optparse.OptionParser(USAGE_STRING) |
212 parser.add_option('-b', '--bucket', | 237 parser.add_option('-b', '--bucket', |
213 help='Google Storage bucket to upload to.') | 238 help='Google Storage bucket to upload to.') |
214 parser.add_option('-e', '--boto', help='Specify a custom boto file.') | 239 parser.add_option('-e', '--boto', help='Specify a custom boto file.') |
240 parser.add_option('-z', '--archive', action='store_true', | |
241 help='Archive directory as a tar.gz file') | |
215 parser.add_option('-f', '--force', action='store_true', | 242 parser.add_option('-f', '--force', action='store_true', |
216 help='Force upload even if remote file exists.') | 243 help='Force upload even if remote file exists.') |
217 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, | 244 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, |
218 help='Path to the gsutil script.') | 245 help='Path to the gsutil script.') |
219 parser.add_option('-m', '--use_md5', action='store_true', | 246 parser.add_option('-m', '--use_md5', action='store_true', |
220 help='Generate MD5 files when scanning, and don\'t check ' | 247 help='Generate MD5 files when scanning, and don\'t check ' |
221 'the MD5 checksum if a .md5 file is found.') | 248 'the MD5 checksum if a .md5 file is found.') |
222 parser.add_option('-t', '--num_threads', default=1, type='int', | 249 parser.add_option('-t', '--num_threads', default=1, type='int', |
223 help='Number of uploader threads to run.') | 250 help='Number of uploader threads to run.') |
224 parser.add_option('-s', '--skip_hashing', action='store_true', | 251 parser.add_option('-s', '--skip_hashing', action='store_true', |
225 help='Skip hashing if .sha1 file exists.') | 252 help='Skip hashing if .sha1 file exists.') |
226 parser.add_option('-0', '--use_null_terminator', action='store_true', | 253 parser.add_option('-0', '--use_null_terminator', action='store_true', |
227 help='Use \\0 instead of \\n when parsing ' | 254 help='Use \\0 instead of \\n when parsing ' |
228 'the file list from stdin. This is useful if the input ' | 255 'the file list from stdin. This is useful if the input ' |
229 'is coming from "find ... -print0".') | 256 'is coming from "find ... -print0".') |
230 parser.add_option('-z', '--gzip', metavar='ext', | 257 parser.add_option('-z', '--gzip', metavar='ext', |
231 help='Gzip files which end in ext. ' | 258 help='Gzip files which end in ext. ' |
232 'ext is a comma-separated list') | 259 'ext is a comma-separated list') |
233 (options, args) = parser.parse_args() | 260 (options, args) = parser.parse_args() |
234 | 261 |
235 # Enumerate our inputs. | 262 # Enumerate our inputs. |
236 input_filenames = get_targets(args, parser, options.use_null_terminator) | 263 input_filenames = get_targets(args, parser, options.use_null_terminator) |
237 | 264 |
265 | |
Ryan Tseng
2015/06/25 17:19:14
remove newline (inside function, just 1 is enough)
ricow1
2015/06/26 06:18:05
Done.
| |
266 if options.archive: | |
267 if validate_archive_dirs(input_filenames): | |
Ryan Tseng
2015/06/25 17:19:15
if not validate_archive_dirs()?
ricow1
2015/06/26 06:18:05
Done.
| |
268 parser.error('Only directories just below cwd are valid entries when ' | |
269 'using the --archive argument. Entries can not contain .. ' | |
270 ' and entries can not be symlinks. Entries was %s' % | |
271 input_filenames) | |
272 return 1 | |
273 input_filenames = create_archives(input_filenames) | |
274 | |
238 # Make sure we can find a working instance of gsutil. | 275 # Make sure we can find a working instance of gsutil. |
239 if os.path.exists(GSUTIL_DEFAULT_PATH): | 276 if os.path.exists(GSUTIL_DEFAULT_PATH): |
240 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) | 277 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
241 else: | 278 else: |
242 gsutil = None | 279 gsutil = None |
243 for path in os.environ["PATH"].split(os.pathsep): | 280 for path in os.environ["PATH"].split(os.pathsep): |
244 if os.path.exists(path) and 'gsutil' in os.listdir(path): | 281 if os.path.exists(path) and 'gsutil' in os.listdir(path): |
245 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) | 282 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) |
246 if not gsutil: | 283 if not gsutil: |
247 parser.error('gsutil not found in %s, bad depot_tools checkout?' % | 284 parser.error('gsutil not found in %s, bad depot_tools checkout?' % |
248 GSUTIL_DEFAULT_PATH) | 285 GSUTIL_DEFAULT_PATH) |
249 | 286 |
250 base_url = 'gs://%s' % options.bucket | 287 base_url = 'gs://%s' % options.bucket |
251 | 288 |
252 return upload_to_google_storage( | 289 return upload_to_google_storage( |
253 input_filenames, base_url, gsutil, options.force, options.use_md5, | 290 input_filenames, base_url, gsutil, options.force, options.use_md5, |
254 options.num_threads, options.skip_hashing, options.gzip) | 291 options.num_threads, options.skip_hashing, options.gzip) |
255 | 292 |
256 | 293 |
257 if __name__ == '__main__': | 294 if __name__ == '__main__': |
258 try: | 295 try: |
259 sys.exit(main()) | 296 sys.exit(main()) |
260 except KeyboardInterrupt: | 297 except KeyboardInterrupt: |
261 sys.stderr.write('interrupted\n') | 298 sys.stderr.write('interrupted\n') |
262 sys.exit(1) | 299 sys.exit(1) |
OLD | NEW |