Index: upload_to_google_storage.py |
=================================================================== |
--- upload_to_google_storage.py (revision 293975) |
+++ upload_to_google_storage.py (working copy) |
@@ -12,6 +12,7 @@ |
import re |
import stat |
import sys |
+import tarfile |
import threading |
import time |
@@ -130,7 +131,7 @@ |
def upload_to_google_storage( |
input_filenames, base_url, gsutil, force, |
- use_md5, num_threads, skip_hashing): |
+ use_md5, num_threads, skip_hashing, archive): |
# We only want one MD5 calculation happening at a time to avoid HD thrashing. |
md5_lock = threading.Lock() |
@@ -203,16 +204,29 @@ |
return max_ret_code |
+def create_archives(dirs): |
+ archive_names = [] |
+ for name in dirs: |
+ tarname = '%s.tar.gz' % name |
+ with tarfile.open(tarname, 'w:gz') as tar: |
+ tar.add(name) |
+ archive_names.append(tarname) |
+ return archive_names |
+def validate_archive_dirs(dirs): |
hinoka
2015/02/06 18:54:35
This seems a little confusing to read... Really wh
ricow1
2015/02/10 11:15:45
Added isdir, not islink, not .. in path. Requiring
|
+ def just_below_cwd(name): |
+ return name in next(os.walk('.'))[1] |
hinoka
2015/02/06 18:54:35
Why are you walking from cwd?
ricow1
2015/02/10 11:15:45
Look at the parser.error at the callsite (which I
|
+ return reduce(lambda x, y: x + 1 if not just_below_cwd(y) else x, dirs, 0) |
hinoka
2015/02/06 18:54:35
I prefer any(map(fn, dirs)) over reduce()
ricow1
2015/02/10 11:15:45
done
|
+ |
def main(args): |
parser = optparse.OptionParser(USAGE_STRING) |
parser.add_option('-b', '--bucket', |
help='Google Storage bucket to upload to.') |
parser.add_option('-e', '--boto', help='Specify a custom boto file.') |
+ parser.add_option('-z', '--archive', action='store_true', |
+ help='Archive directory as a tar.gz file') |
parser.add_option('-f', '--force', action='store_true', |
help='Force upload even if remote file exists.') |
- parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, |
- help='Path to the gsutil script.') |
parser.add_option('-m', '--use_md5', action='store_true', |
help='Generate MD5 files when scanning, and don\'t check ' |
'the MD5 checksum if a .md5 file is found.') |
@@ -229,6 +243,15 @@ |
# Enumerate our inputs. |
input_filenames = get_targets(args, parser, options.use_null_terminator) |
+ |
+ if options.archive: |
hinoka
2015/02/06 18:54:35
Also add an isdir() check.
ricow1
2015/02/10 11:15:45
Added in validate_archive_dirs
|
+ if validate_archive_dirs(input_filenames): |
+ parser.error('Only directories just below cwd are valid entries when ' |
+ 'using the --archive argument. Entries was %s' % |
+ input_filenames) |
+ return 1 |
+ input_filenames = create_archives(input_filenames) |
+ |
# Make sure we can find a working instance of gsutil. |
if os.path.exists(GSUTIL_DEFAULT_PATH): |
gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) |
@@ -250,7 +273,7 @@ |
return upload_to_google_storage( |
input_filenames, base_url, gsutil, options.force, options.use_md5, |
- options.num_threads, options.skip_hashing) |
+ options.num_threads, options.skip_hashing, options.archive) |
if __name__ == '__main__': |