In upload_to_google_storage, pass -z argument through to gsutil.

upload_to_google_storage.py

Index: upload_to_google_storage.py
diff --git a/upload_to_google_storage.py b/upload_to_google_storage.py
index d88597cef0cfab13226e158fd829f85a387db791..b07dab6da9de4e020cf8341fb183fbbb150389ce 100755
--- a/upload_to_google_storage.py
+++ b/upload_to_google_storage.py
@@ -67,7 +67,7 @@ def get_md5_cached(filename):
 
 def _upload_worker(
     thread_num, upload_queue, base_url, gsutil, md5_lock, force,
-    use_md5, stdout_queue, ret_codes):
+    use_md5, stdout_queue, ret_codes, gzip):
   while True:
     filename, sha1_sum = upload_queue.get()
     if not filename:
@@ -92,7 +92,13 @@ def _upload_worker(
           continue
     stdout_queue.put('%d> Uploading %s...' % (
         thread_num, filename))
-    code, _, err = gsutil.check_call('cp', filename, file_url)
+    gsutil_args = ['cp']
+    if gzip:
+      gsutil_args.append('-z')

[hinoka, 2015/04/03 09:59:31]

nit: use .extend()

[azarchs, 2015/04/03 10:07:21]

Done.

+      gsutil_args.append(gzip)
+    gsutil_args.append(filename)
+    gsutil_args.append(file_url)
+    code, _, err = gsutil.check_call(*gsutil_args)
     if code != 0:
       ret_codes.put(
           (code,
@@ -129,7 +135,7 @@ def get_targets(args, parser, use_null_terminator):
 
 def upload_to_google_storage(
     input_filenames, base_url, gsutil, force,
-    use_md5, num_threads, skip_hashing):
+    use_md5, num_threads, skip_hashing, gzip):
   # We only want one MD5 calculation happening at a time to avoid HD thrashing.
   md5_lock = threading.Lock()
 
@@ -147,7 +153,7 @@ def upload_to_google_storage(
     t = threading.Thread(
         target=_upload_worker,
         args=[thread_num, upload_queue, base_url, gsutil, md5_lock,
-              force, use_md5, stdout_queue, ret_codes])
+              force, use_md5, stdout_queue, ret_codes, gzip])
     t.daemon = True
     t.start()
     all_threads.append(t)
@@ -223,6 +229,9 @@ def main():
                     help='Use \\0 instead of \\n when parsing '
                     'the file list from stdin.  This is useful if the input '
                     'is coming from "find ... -print0".')
+  parser.add_option('-z', dest='gzip', metavar='ext',

[hinoka, 2015/04/03 09:59:31]

I prefer ('-z', '--gzip', ...), in which case dest is implied.

[azarchs, 2015/04/03 10:07:21]

Done.

+                    help='Gzip files which end in ext. '

[pasko, 2015/04/01 18:41:39]

nit: it took me a bit of time to get the intuition of what 'ext' is, I think
'suffix' would be more readable.

nit: s/may be/is/

[azarchs, 2015/04/02 09:47:23]

It's standard terminology in windows to call the suffix the file extension. 
More importantly, ext is what it's called in the documentation for gsutil.  For
consistency's sake I think it's better to keep it that way.
Done.

+                         'ext may be a comma-separated list')
   (options, args) = parser.parse_args()
 
   # Enumerate our inputs.
@@ -244,7 +253,7 @@ def main():
 
   return upload_to_google_storage(
       input_filenames, base_url, gsutil, options.force, options.use_md5,
-      options.num_threads, options.skip_hashing)
+      options.num_threads, options.skip_hashing, options.gzip)
 
 
 if __name__ == '__main__':