In upload_to_google_storage, pass -z argument through to gsutil.

upload_to_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Uploads files to Google Storage content addressed."""
 
 import hashlib
 import optparse
 import os
@@ skipping 49 matching lines @@
         return md5_match.group(1)
   else:
     md5_hash = get_md5(filename)
     with open('%s.md5' % filename, 'wb') as f:
       f.write(md5_hash)
     return md5_hash
 
 
 def _upload_worker(
     thread_num, upload_queue, base_url, gsutil, md5_lock, force,
-    use_md5, stdout_queue, ret_codes):
+    use_md5, stdout_queue, ret_codes, gzip):
   while True:
     filename, sha1_sum = upload_queue.get()
     if not filename:
       break
     file_url = '%s/%s' % (base_url, sha1_sum)
     if gsutil.check_call('ls', file_url)[0] == 0 and not force:
       # File exists, check MD5 hash.
       _, out, _ = gsutil.check_call('ls', '-L', file_url)
       etag_match = re.search('ETag:\s+([a-z0-9]{32})', out)
       if etag_match:
         remote_md5 = etag_match.group(1)
         # Calculate the MD5 checksum to match it to Google Storage's ETag.
         with md5_lock:
           if use_md5:
             local_md5 = get_md5_cached(filename)
           else:
             local_md5 = get_md5(filename)
         if local_md5 == remote_md5:
           stdout_queue.put(
               '%d> File %s already exists and MD5 matches, upload skipped' %
               (thread_num, filename))
           continue
     stdout_queue.put('%d> Uploading %s...' % (
         thread_num, filename))
-    code, _, err = gsutil.check_call('cp', filename, file_url)
+    gsutil_args = ['cp']
+    if gzip:
+      gsutil_args.extend(['-z', gzip])
+    gsutil_args.extend([filename, file_url])
+    code, _, err = gsutil.check_call(*gsutil_args)
     if code != 0:
       ret_codes.put(
           (code,
            'Encountered error on uploading %s to %s\n%s' %
            (filename, file_url, err)))
       continue
 
     # Mark executable files with the header "x-goog-meta-executable: 1" which
     # the download script will check for to preserve the executable bit.
     if not sys.platform.startswith('win'):
@@ skipping 16 matching lines @@
     if use_null_terminator:
       return sys.stdin.read().split('\0')
     else:
       return sys.stdin.read().splitlines()
   else:
     return args
 
 
 def upload_to_google_storage(
     input_filenames, base_url, gsutil, force,
-    use_md5, num_threads, skip_hashing):
+    use_md5, num_threads, skip_hashing, gzip):
   # We only want one MD5 calculation happening at a time to avoid HD thrashing.
   md5_lock = threading.Lock()
 
   # Start up all the worker threads plus the printer thread.
   all_threads = []
   ret_codes = Queue.Queue()
   ret_codes.put((0, None))
   upload_queue = Queue.Queue()
   upload_timer = time.time()
   stdout_queue = Queue.Queue()
   printer_thread = threading.Thread(target=printer_worker, args=[stdout_queue])
   printer_thread.daemon = True
   printer_thread.start()
   for thread_num in range(num_threads):
     t = threading.Thread(
         target=_upload_worker,
         args=[thread_num, upload_queue, base_url, gsutil, md5_lock,
-              force, use_md5, stdout_queue, ret_codes])
+              force, use_md5, stdout_queue, ret_codes, gzip])
     t.daemon = True
     t.start()
     all_threads.append(t)
 
   # We want to hash everything in a single thread since its faster.
   # The bottleneck is in disk IO, not CPU.
   hashing_start = time.time()
   for filename in input_filenames:
     if not os.path.exists(filename):
       stdout_queue.put('Main> Error: %s not found, skipping.' % filename)
@@ skipping 55 matching lines @@
                     help='Generate MD5 files when scanning, and don\'t check '
                     'the MD5 checksum if a .md5 file is found.')
   parser.add_option('-t', '--num_threads', default=1, type='int',
                     help='Number of uploader threads to run.')
   parser.add_option('-s', '--skip_hashing', action='store_true',
                     help='Skip hashing if .sha1 file exists.')
   parser.add_option('-0', '--use_null_terminator', action='store_true',
                     help='Use \\0 instead of \\n when parsing '
                     'the file list from stdin.  This is useful if the input '
                     'is coming from "find ... -print0".')
+  parser.add_option('-z', '--gzip', metavar='ext',
+                    help='Gzip files which end in ext. '
+                         'ext is a comma-separated list')
   (options, args) = parser.parse_args()
 
   # Enumerate our inputs.
   input_filenames = get_targets(args, parser, options.use_null_terminator)
 
   # Make sure we can find a working instance of gsutil.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
     gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
   else:
     gsutil = None
     for path in os.environ["PATH"].split(os.pathsep):
       if os.path.exists(path) and 'gsutil' in os.listdir(path):
         gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
     if not gsutil:
       parser.error('gsutil not found in %s, bad depot_tools checkout?' %
                    GSUTIL_DEFAULT_PATH)
 
   base_url = 'gs://%s' % options.bucket
 
   return upload_to_google_storage(
       input_filenames, base_url, gsutil, options.force, options.use_md5,
-      options.num_threads, options.skip_hashing)
+      options.num_threads, options.skip_hashing, options.gzip)
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main())
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)