Add support for tar.gz archive files to download from download_from_google_storage

download_from_google_storage.py

Index: download_from_google_storage.py
===================================================================
--- download_from_google_storage.py	(revision 293744)
+++ download_from_google_storage.py	(working copy)
@@ -11,8 +11,10 @@
 import os
 import Queue
 import re
+import shutil
 import stat
 import sys
+import tarfile
 import threading
 import time
 
@@ -204,7 +206,7 @@
 
 
 def _downloader_worker_thread(thread_num, q, force, base_url,
-                              gsutil, out_q, ret_codes, verbose):
+                              gsutil, out_q, ret_codes, verbose, extract):
   while True:
     input_sha1_sum, output_filename = q.get()
     if input_sha1_sum is None:
@@ -237,6 +239,29 @@
       out_q.put('%d> %s' % (thread_num, err))
       ret_codes.put((code, err))
 
+    if extract:
+      if (not tarfile.is_tarfile(output_filename)
+          or not output_filename.endswith('tar.gz')):

[hinoka, 2015/02/05 18:44:16]

'.tar.gz' to match with expectations on 251

[ricow1, 2015/02/06 11:52:29]

Done.

+        out_q.put('%d> Error: %s is not a tar.gz archive.' % (
+                  thread_num, output_filename))   
+        ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename)))
+        continue
+      tar = tarfile.open(output_filename, 'r:gz')

[hinoka, 2015/02/05 18:44:16]

Also validate all paths in the tarfile here (Only having checks in the upload
scripts isn't going to prevent malicious out-of-band uploads).  May have to
iterate through each file.

[ricow1, 2015/02/06 11:52:29]

Added check

+      dirname = os.path.dirname(os.path.abspath(output_filename))
+      extract_dir = output_filename[0:len(output_filename)-7]
+      out_q.put('%d> extracting %s...' % (thread_num, extract_dir))
+      if os.path.exists(extract_dir):
+        try:
+          shutil.rmtree(extract_dir)
+          out_q.put('%d> Removed %s...' % (thread_num, extract_dir))
+        except OSError:
+          out_q.put('%d> Warning: Can\'t delete: %s' % (
+                    thread_num, extract_dir))   
+          ret_codes.put((1, 'Can\'t delete %s.' % (extract_dir)))
+          continue
+        out_q.put('%d> Extracting %s to %s' % (thread_num, output_filename,
+                                               extract_dir))
+      tar.extractall(path=dirname)
     # Set executable bit.
     if sys.platform == 'cygwin':
       # Under cygwin, mark all files as executable. The executable flag in
@@ -267,7 +292,7 @@
 
 def download_from_google_storage(
     input_filename, base_url, gsutil, num_threads, directory, recursive,
-    force, output, ignore_errors, sha1_file, verbose, auto_platform):
+    force, output, ignore_errors, sha1_file, verbose, auto_platform, extract):
   # Start up all the worker threads.
   all_threads = []
   download_start = time.time()
@@ -279,7 +304,7 @@
     t = threading.Thread(
         target=_downloader_worker_thread,
         args=[thread_num, work_queue, force, base_url,
-              gsutil, stdout_queue, ret_codes, verbose])
+              gsutil, stdout_queue, ret_codes, verbose, extract])
     t.daemon = True
     t.start()
     all_threads.append(t)
@@ -367,6 +392,13 @@
                          '(linux|mac|win).  If so, the script will only '
                          'process files that are in the paths that '
                          'that matches the current platform.')
+  parser.add_option('-u', '--extract',
+                    action='store_true',
+                    help='Extract a downloaded tar.gz file after download. '
+                         'Leaves the tar.gz file around for sha verification1'
+                         'If a directory with the same name as the tar.gz '
+                         'file already exists, this is deleted (to get a '
+                         'clean state in case of update.)')
   parser.add_option('-v', '--verbose', action='store_true',
                     help='Output extra diagnostic and progress information.')
 
@@ -463,7 +495,8 @@
   return download_from_google_storage(
       input_filename, base_url, gsutil, options.num_threads, options.directory,
       options.recursive, options.force, options.output, options.ignore_errors,
-      options.sha1_file, options.verbose, options.auto_platform)
+      options.sha1_file, options.verbose, options.auto_platform,
+      options.extract)
 
 
 if __name__ == '__main__':