Add support for tar.gz archive files to download from download_from_google_storage

upload_to_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Uploads files to Google Storage content addressed."""
 
 import hashlib
 import optparse
 import os
 import Queue
 import re
 import stat
 import sys
+import tarfile
 import threading
 import time
 
 from download_from_google_storage import get_sha1
 from download_from_google_storage import Gsutil
 from download_from_google_storage import printer_worker
 from download_from_google_storage import GSUTIL_DEFAULT_PATH
 
 USAGE_STRING = """%prog [options] target [target2 ...].
 Target is the file intended to be uploaded to Google Storage.
@@ skipping 173 matching lines @@
   max_ret_code = 0
   for ret_code, message in ret_codes.queue:
     max_ret_code = max(ret_code, max_ret_code)
     if message:
       print >> sys.stderr, message
 
   if not max_ret_code:
     print 'Success!'
 
   return max_ret_code
 

[Ryan Tseng, 2015/06/25 17:19:14]

top level function, 2 newlines

[ricow1, 2015/06/26 06:18:05]

Done.

+def create_archives(dirs):
+  archive_names = []
+  for name in dirs:
+    tarname = '%s.tar.gz' % name
+    with tarfile.open(tarname, 'w:gz') as tar:
+      tar.add(name)
+    archive_names.append(tarname)
+  return archive_names
+

[Ryan Tseng, 2015/06/25 17:19:14]

2 newlines

[ricow1, 2015/06/26 06:18:05]

Done.

+def validate_archive_dirs(dirs):
+  # We don't allow .. in paths in our archives.
+  if any(map(lambda x: '..' in x, dirs)):
+    return False
+  # We only allow dirs.
+  if any(map(lambda x: not os.path.isdir(x), dirs)):
+    return False
+  # We don't allow sym links in our archives.
+  if any(map(os.path.islink, dirs)):
+    return False
+  # We required that the subdirectories we are archiving are all just below
+  # cwd.
+  return not any(map(lambda x: x not in next(os.walk('.'))[1], dirs))
+

[Ryan Tseng, 2015/06/25 17:19:14]

remove 1 newline so there are 2 total

[ricow1, 2015/06/26 06:18:05]

Done.

+
 
 def main():
   parser = optparse.OptionParser(USAGE_STRING)
   parser.add_option('-b', '--bucket',
                     help='Google Storage bucket to upload to.')
   parser.add_option('-e', '--boto', help='Specify a custom boto file.')
+  parser.add_option('-z', '--archive', action='store_true',
+                    help='Archive directory as a tar.gz file')
   parser.add_option('-f', '--force', action='store_true',
                     help='Force upload even if remote file exists.')
   parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH,
                     help='Path to the gsutil script.')
   parser.add_option('-m', '--use_md5', action='store_true',
                     help='Generate MD5 files when scanning, and don\'t check '
                     'the MD5 checksum if a .md5 file is found.')
   parser.add_option('-t', '--num_threads', default=1, type='int',
                     help='Number of uploader threads to run.')
   parser.add_option('-s', '--skip_hashing', action='store_true',
                     help='Skip hashing if .sha1 file exists.')
   parser.add_option('-0', '--use_null_terminator', action='store_true',
                     help='Use \\0 instead of \\n when parsing '
                     'the file list from stdin.  This is useful if the input '
                     'is coming from "find ... -print0".')
   parser.add_option('-z', '--gzip', metavar='ext',
                     help='Gzip files which end in ext. '
                          'ext is a comma-separated list')
   (options, args) = parser.parse_args()
 
   # Enumerate our inputs.
   input_filenames = get_targets(args, parser, options.use_null_terminator)
 
+

[Ryan Tseng, 2015/06/25 17:19:14]

remove newline (inside function, just 1 is enough)

[ricow1, 2015/06/26 06:18:05]

Done.

+  if options.archive:
+    if validate_archive_dirs(input_filenames):

[Ryan Tseng, 2015/06/25 17:19:15]

if not validate_archive_dirs()?

[ricow1, 2015/06/26 06:18:05]

Done.

+      parser.error('Only directories just below cwd are valid entries when '
+                   'using the --archive argument. Entries can not contain .. '
+                   ' and entries can not be symlinks. Entries was %s' %
+                    input_filenames)
+      return 1
+    input_filenames = create_archives(input_filenames)
+
   # Make sure we can find a working instance of gsutil.
   if os.path.exists(GSUTIL_DEFAULT_PATH):
     gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
   else:
     gsutil = None
     for path in os.environ["PATH"].split(os.pathsep):
       if os.path.exists(path) and 'gsutil' in os.listdir(path):
         gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
     if not gsutil:
       parser.error('gsutil not found in %s, bad depot_tools checkout?' %
                    GSUTIL_DEFAULT_PATH)
 
   base_url = 'gs://%s' % options.bucket
 
   return upload_to_google_storage(
       input_filenames, base_url, gsutil, options.force, options.use_md5,
       options.num_threads, options.skip_hashing, options.gzip)
 
 
 if __name__ == '__main__':
   try:
     sys.exit(main())
   except KeyboardInterrupt:
     sys.stderr.write('interrupted\n')
     sys.exit(1)