Add support for tar.gz archive files to download from download_from_google_storage

upload_to_google_storage.py

Index: upload_to_google_storage.py
===================================================================
--- upload_to_google_storage.py	(revision 293975)
+++ upload_to_google_storage.py	(working copy)
@@ -12,6 +12,7 @@
 import re
 import stat
 import sys
+import tarfile
 import threading
 import time
 
@@ -130,7 +131,7 @@
 
 def upload_to_google_storage(
     input_filenames, base_url, gsutil, force,
-    use_md5, num_threads, skip_hashing):
+    use_md5, num_threads, skip_hashing, archive):
   # We only want one MD5 calculation happening at a time to avoid HD thrashing.
   md5_lock = threading.Lock()
 
@@ -203,16 +204,29 @@
 
   return max_ret_code
 
+def create_archives(dirs):
+  archive_names = []
+  for name in dirs:
+    tarname = '%s.tar.gz' % name
+    with tarfile.open(tarname, 'w:gz') as tar:
+      tar.add(name)
+    archive_names.append(tarname)
+  return archive_names
 
+def validate_archive_dirs(dirs):

[hinoka, 2015/02/06 18:54:35]

This seems a little confusing to read... Really what we want is to know if:
1. The path contains ".."
2. The path contains a symlink

In fact a better way might be to just utilize tarfile.add() filter=
capabilities.

[ricow1, 2015/02/10 11:15:45]

Added isdir, not islink, not .. in path. Requiring the dir to be just below cwd
actually makes sure it does not contain .. anyway

+  def just_below_cwd(name):
+    return name in next(os.walk('.'))[1]

[hinoka, 2015/02/06 18:54:35]

Why are you walking from cwd?

[ricow1, 2015/02/10 11:15:45]

Look at the parser.error at the callsite (which I now changed to also include
info on .. and symlinks.
I don't think it makes sense to have the tar.gz.sha1 other places than where the
directory is extracted. And this makes all the code more simple (i.e., you don't
have to do path manipulation both here and in the extraction)

+  return reduce(lambda x, y: x + 1 if not just_below_cwd(y) else x, dirs, 0)

[hinoka, 2015/02/06 18:54:35]

I prefer any(map(fn, dirs)) over reduce()

[ricow1, 2015/02/10 11:15:45]

done

+
 def main(args):
   parser = optparse.OptionParser(USAGE_STRING)
   parser.add_option('-b', '--bucket',
                     help='Google Storage bucket to upload to.')
   parser.add_option('-e', '--boto', help='Specify a custom boto file.')
+  parser.add_option('-z', '--archive', action='store_true',
+                    help='Archive directory as a tar.gz file')
   parser.add_option('-f', '--force', action='store_true',
                     help='Force upload even if remote file exists.')
-  parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH,
-                    help='Path to the gsutil script.')
   parser.add_option('-m', '--use_md5', action='store_true',
                     help='Generate MD5 files when scanning, and don\'t check '
                     'the MD5 checksum if a .md5 file is found.')
@@ -229,6 +243,15 @@
   # Enumerate our inputs.
   input_filenames = get_targets(args, parser, options.use_null_terminator)
 
+
+  if options.archive:

[hinoka, 2015/02/06 18:54:35]

Also add an isdir() check.

[ricow1, 2015/02/10 11:15:45]

Added in validate_archive_dirs

+    if validate_archive_dirs(input_filenames):
+      parser.error('Only directories just below cwd are valid entries when '
+                   'using the --archive argument. Entries was %s' %
+                    input_filenames)
+      return 1
+    input_filenames = create_archives(input_filenames)
+
   # Make sure we can find a working instance of gsutil.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
     gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
@@ -250,7 +273,7 @@
 
   return upload_to_google_storage(
       input_filenames, base_url, gsutil, options.force, options.use_md5,
-      options.num_threads, options.skip_hashing)
+      options.num_threads, options.skip_hashing, options.archive)
 
 
 if __name__ == '__main__':