Add support for tar.gz archive files to download from download_from_google_storage

upload_to_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Uploads files to Google Storage content addressed."""
 
 import hashlib
 import optparse
 import os
@@ skipping 50 matching lines @@
         return md5_match.group(1)
   else:
     md5_hash = get_md5(filename)
     with open('%s.md5' % filename, 'wb') as f:
       f.write(md5_hash)
     return md5_hash
 
 
 def _upload_worker(
     thread_num, upload_queue, base_url, gsutil, md5_lock, force,
-    use_md5, stdout_queue, ret_codes):
+    use_md5, stdout_queue, ret_codes, public):
   while True:
     filename, sha1_sum = upload_queue.get()
     if not filename:
       break
     file_url = '%s/%s' % (base_url, sha1_sum)
     if gsutil.check_call('ls', file_url)[0] == 0 and not force:
       # File exists, check MD5 hash.
       _, out, _ = gsutil.check_call('ls', '-L', file_url)
       etag_match = re.search('ETag:\s+([a-z0-9]{32})', out)
       if etag_match:
         remote_md5 = etag_match.group(1)
         # Calculate the MD5 checksum to match it to Google Storage's ETag.
         with md5_lock:
           if use_md5:
             local_md5 = get_md5_cached(filename)
           else:
             local_md5 = get_md5(filename)
         if local_md5 == remote_md5:
           stdout_queue.put(
               '%d> File %s already exists and MD5 matches, upload skipped' %
               (thread_num, filename))
           continue
     stdout_queue.put('%d> Uploading %s...' % (
         thread_num, filename))
-    code, _, err = gsutil.check_call('cp', filename, file_url)
+    args = ['cp']
+    if public:
+      args.extend(['-a', 'public-read'])
+    args.extend([filename, file_url])
+    code, _, err = gsutil.check_call(*args)
+    
     if code != 0:
       ret_codes.put(
           (code,
            'Encountered error on uploading %s to %s\n%s' %
            (filename, file_url, err)))
       continue
 
     # Mark executable files with the header "x-goog-meta-executable: 1" which
     # the download script will check for to preserve the executable bit.
     if not sys.platform.startswith('win'):
@@ skipping 16 matching lines @@
     if use_null_terminator:
       return sys.stdin.read().split('\0')
     else:
       return sys.stdin.read().splitlines()
   else:
     return args
 
 
 def upload_to_google_storage(
     input_filenames, base_url, gsutil, force,
-    use_md5, num_threads, skip_hashing):
+    use_md5, num_threads, skip_hashing, public):
   # We only want one MD5 calculation happening at a time to avoid HD thrashing.
   md5_lock = threading.Lock()
 
   # Start up all the worker threads plus the printer thread.
   all_threads = []
   ret_codes = Queue.Queue()
   ret_codes.put((0, None))
   upload_queue = Queue.Queue()
   upload_timer = time.time()
   stdout_queue = Queue.Queue()
   printer_thread = threading.Thread(target=printer_worker, args=[stdout_queue])
   printer_thread.daemon = True
   printer_thread.start()
   for thread_num in range(num_threads):
     t = threading.Thread(
         target=_upload_worker,
         args=[thread_num, upload_queue, base_url, gsutil, md5_lock,
-              force, use_md5, stdout_queue, ret_codes])
+              force, use_md5, stdout_queue, ret_codes, public])
     t.daemon = True
     t.start()
     all_threads.append(t)
 
   # We want to hash everything in a single thread since its faster.
   # The bottleneck is in disk IO, not CPU.
   hashing_start = time.time()
   for filename in input_filenames:
     if not os.path.exists(filename):
       stdout_queue.put('Main> Error: %s not found, skipping.' % filename)
@@ skipping 42 matching lines @@
   return max_ret_code
 
 
 def main(args):
   parser = optparse.OptionParser(USAGE_STRING)
   parser.add_option('-b', '--bucket',
                     help='Google Storage bucket to upload to.')
   parser.add_option('-e', '--boto', help='Specify a custom boto file.')
   parser.add_option('-f', '--force', action='store_true',
                     help='Force upload even if remote file exists.')
-  parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH,

[ricow1, 2015/01/19 16:02:28]

this does not seem to be supported anyway, so removing

[hinoka, 2015/01/20 19:28:52]

Acknowledged.

-                    help='Path to the gsutil script.')
   parser.add_option('-m', '--use_md5', action='store_true',
                     help='Generate MD5 files when scanning, and don\'t check '
                     'the MD5 checksum if a .md5 file is found.')
   parser.add_option('-t', '--num_threads', default=1, type='int',
                     help='Number of uploader threads to run.')
   parser.add_option('-s', '--skip_hashing', action='store_true',
                     help='Skip hashing if .sha1 file exists.')
+  parser.add_option('-p', '--public', action='store_true',

[hinoka, 2015/01/20 19:28:52]

Why?

In general, I prefer to set this on a bucket basis (set the default ACL) instead
of an object basis.  This is to force us to keep public buckets fully public and
private buckets fully private.  IMO setting object ACLs individually leads to
pain and suffering.

[ricow1, 2015/01/22 15:46:04]

Valid point, removed support for this

+                    help='Make the uploaded file public read.')
   parser.add_option('-0', '--use_null_terminator', action='store_true',
                     help='Use \\0 instead of \\n when parsing '
                     'the file list from stdin.  This is useful if the input '
                     'is coming from "find ... -print0".')
   (options, args) = parser.parse_args()
 
   # Enumerate our inputs.
   input_filenames = get_targets(args, parser, options.use_null_terminator)
 
   # Make sure we can find a working instance of gsutil.
@@ skipping 10 matching lines @@
 
   base_url = 'gs://%s' % options.bucket
 
   # Check we have a valid bucket with valid permissions.
   code = check_bucket_permissions(base_url, gsutil)
   if code:
     return code
 
   return upload_to_google_storage(
       input_filenames, base_url, gsutil, options.force, options.use_md5,
-      options.num_threads, options.skip_hashing)
+      options.num_threads, options.skip_hashing, options.public)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))