Force download and extract if extracted directory is missing

download_from_google_storage.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Download files from Google Storage based on SHA1 sums."""
 
 
 import hashlib
 import optparse
@@ skipping 202 matching lines @@
       return False
     return True
   return all(map(_validate, tar.getmembers()))
 
 def _downloader_worker_thread(thread_num, q, force, base_url,
                               gsutil, out_q, ret_codes, verbose, extract,
                               delete=True):
   while True:
     input_sha1_sum, output_filename = q.get()
     if input_sha1_sum is None:
       return

[Ryan Tseng, 2016/04/19 21:07:52]

extract_dir = None

[pauljensen, 2016/04/20 11:37:00]

Done.

+    if extract:
+      if not output_filename.endswith('.tar.gz'):
+        out_q.put('%d> Error: %s is not a tar.gz archive.' % (
+                  thread_num, output_filename))
+        ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename)))
+        continue
+      extract_dir = output_filename[0:len(output_filename)-7]
     if os.path.exists(output_filename) and not force:
-      if get_sha1(output_filename) == input_sha1_sum:
-        if verbose:
-          out_q.put(
-              '%d> File %s exists and SHA1 matches. Skipping.' % (
-                  thread_num, output_filename))
-        continue
+      if not extract or os.path.exists(extract_dir):
+        if get_sha1(output_filename) == input_sha1_sum:
+          if verbose:
+            out_q.put(
+                '%d> File %s exists and SHA1 matches. Skipping.' % (
+                    thread_num, output_filename))
+          continue
     # Check if file exists.
     file_url = '%s/%s' % (base_url, input_sha1_sum)
     (code, _, err) = gsutil.check_call('ls', file_url)
     if code != 0:
       if code == 404:
         out_q.put('%d> File %s for %s does not exist, skipping.' % (
             thread_num, file_url, output_filename))
         ret_codes.put((1, 'File %s for %s does not exist.' % (
             file_url, output_filename)))
       else:
@@ skipping 20 matching lines @@
 
     remote_sha1 = get_sha1(output_filename)
     if remote_sha1 != input_sha1_sum:
       msg = ('%d> ERROR remote sha1 (%s) does not match expected sha1 (%s).' %
              (thread_num, remote_sha1, input_sha1_sum))
       out_q.put(msg)
       ret_codes.put((20, msg))
       continue
 
     if extract:
-      if (not tarfile.is_tarfile(output_filename)
-          or not output_filename.endswith('.tar.gz')):
+      if not tarfile.is_tarfile(output_filename):
         out_q.put('%d> Error: %s is not a tar.gz archive.' % (
                   thread_num, output_filename))
         ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename)))
         continue
       with tarfile.open(output_filename, 'r:gz') as tar:
         dirname = os.path.dirname(os.path.abspath(output_filename))
-        extract_dir = output_filename[0:len(output_filename)-7]
         if not _validate_tar_file(tar, os.path.basename(extract_dir)):
           out_q.put('%d> Error: %s contains files outside %s.' % (
                     thread_num, output_filename, extract_dir))
           ret_codes.put((1, '%s contains invalid entries.' % (output_filename)))
           continue
         if os.path.exists(extract_dir):
           try:
             shutil.rmtree(extract_dir)
             out_q.put('%d> Removed %s...' % (thread_num, extract_dir))
           except OSError:
@@ skipping 232 matching lines @@
 
   return download_from_google_storage(
       input_filename, base_url, gsutil, options.num_threads, options.directory,
       options.recursive, options.force, options.output, options.ignore_errors,
       options.sha1_file, options.verbose, options.auto_platform,
       options.extract)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))