Add support for tar.gz archive files to download from download_from_google_storage

upload_to_google_storage.py

Index: upload_to_google_storage.py
===================================================================
--- upload_to_google_storage.py	(revision 293696)
+++ upload_to_google_storage.py	(working copy)
@@ -68,7 +68,7 @@
 
 def _upload_worker(
     thread_num, upload_queue, base_url, gsutil, md5_lock, force,
-    use_md5, stdout_queue, ret_codes):
+    use_md5, stdout_queue, ret_codes, public):
   while True:
     filename, sha1_sum = upload_queue.get()
     if not filename:
@@ -93,7 +93,12 @@
           continue
     stdout_queue.put('%d> Uploading %s...' % (
         thread_num, filename))
-    code, _, err = gsutil.check_call('cp', filename, file_url)
+    args = ['cp']
+    if public:
+      args.extend(['-a', 'public-read'])
+    args.extend([filename, file_url])
+    code, _, err = gsutil.check_call(*args)
+    
     if code != 0:
       ret_codes.put(
           (code,
@@ -130,7 +135,7 @@
 
 def upload_to_google_storage(
     input_filenames, base_url, gsutil, force,
-    use_md5, num_threads, skip_hashing):
+    use_md5, num_threads, skip_hashing, public):
   # We only want one MD5 calculation happening at a time to avoid HD thrashing.
   md5_lock = threading.Lock()
 
@@ -148,7 +153,7 @@
     t = threading.Thread(
         target=_upload_worker,
         args=[thread_num, upload_queue, base_url, gsutil, md5_lock,
-              force, use_md5, stdout_queue, ret_codes])
+              force, use_md5, stdout_queue, ret_codes, public])
     t.daemon = True
     t.start()
     all_threads.append(t)
@@ -211,8 +216,6 @@
   parser.add_option('-e', '--boto', help='Specify a custom boto file.')
   parser.add_option('-f', '--force', action='store_true',
                     help='Force upload even if remote file exists.')
-  parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH,

[ricow1, 2015/01/19 16:02:28]

this does not seem to be supported anyway, so removing

[hinoka, 2015/01/20 19:28:52]

Acknowledged.

-                    help='Path to the gsutil script.')
   parser.add_option('-m', '--use_md5', action='store_true',
                     help='Generate MD5 files when scanning, and don\'t check '
                     'the MD5 checksum if a .md5 file is found.')
@@ -220,6 +223,8 @@
                     help='Number of uploader threads to run.')
   parser.add_option('-s', '--skip_hashing', action='store_true',
                     help='Skip hashing if .sha1 file exists.')
+  parser.add_option('-p', '--public', action='store_true',

[hinoka, 2015/01/20 19:28:52]

Why?

In general, I prefer to set this on a bucket basis (set the default ACL) instead
of an object basis.  This is to force us to keep public buckets fully public and
private buckets fully private.  IMO setting object ACLs individually leads to
pain and suffering.

[ricow1, 2015/01/22 15:46:04]

Valid point, removed support for this

+                    help='Make the uploaded file public read.')
   parser.add_option('-0', '--use_null_terminator', action='store_true',
                     help='Use \\0 instead of \\n when parsing '
                     'the file list from stdin.  This is useful if the input '
@@ -250,7 +255,7 @@
 
   return upload_to_google_storage(
       input_filenames, base_url, gsutil, options.force, options.use_md5,
-      options.num_threads, options.skip_hashing)
+      options.num_threads, options.skip_hashing, options.public)
 
 
 if __name__ == '__main__':