Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(3087)

Unified Diff: download_from_google_storage.py

Issue 807463005: Add support for tar.gz archive files to download from download_from_google_storage (Closed) Base URL: http://src.chromium.org/svn/trunk/tools/depot_tools/
Patch Set: Created 5 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | upload_to_google_storage.py » ('j') | upload_to_google_storage.py » ('J')
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: download_from_google_storage.py
===================================================================
--- download_from_google_storage.py (revision 293993)
+++ download_from_google_storage.py (working copy)
@@ -11,8 +11,10 @@
import os
import Queue
import re
+import shutil
import stat
import sys
+import tarfile
import threading
import time
@@ -49,7 +51,6 @@
return 'win32'
return sys.platform
-
# Common utilities
class Gsutil(object):
"""Call gsutil with some predefined settings. This is a convenience object,
@@ -203,8 +204,14 @@
return work_queue_size
+def _validate_tar_file(tar, prefix):
hinoka 2015/02/11 00:16:57 Also check for symbolic/hard links when decompress
ricow1 2015/06/19 09:31:27 Changed to your suggestion below
+ files = tar.getnames()
+ if any(map(lambda x: '..' in x, files)):
hinoka 2015/02/11 00:16:57 how about def _validate(tarinfo): """Returns fa
ricow1 2015/06/19 09:31:27 Done.
+ return True
+ return any(map(lambda x: not x.startswith(prefix), files))
+
def _downloader_worker_thread(thread_num, q, force, base_url,
- gsutil, out_q, ret_codes, verbose):
+ gsutil, out_q, ret_codes, verbose, extract):
while True:
input_sha1_sum, output_filename = q.get()
if input_sha1_sum is None:
@@ -237,6 +244,34 @@
out_q.put('%d> %s' % (thread_num, err))
ret_codes.put((code, err))
+ if extract:
+ if (not tarfile.is_tarfile(output_filename)
+ or not output_filename.endswith('.tar.gz')):
+ out_q.put('%d> Error: %s is not a tar.gz archive.' % (
+ thread_num, output_filename))
+ ret_codes.put((1, '%s is not a tar.gz archive.' % (output_filename)))
+ continue
+ tar = tarfile.open(output_filename, 'r:gz')
+ dirname = os.path.dirname(os.path.abspath(output_filename))
+ extract_dir = output_filename[0:len(output_filename)-7]
+ if _validate_tar_file(tar, os.path.basename(extract_dir)):
hinoka 2015/02/11 00:16:57 "_validate_tar_file" implies it would return True
ricow1 2015/06/19 09:31:27 Done.
+ out_q.put('%d> Error: %s contains files outside %s.' % (
+ thread_num, output_filename, extract_dir))
+ ret_codes.put((1, '%s contains invalid entries.' % (output_filename)))
+ continue
+ out_q.put('%d> Extracting %s...' % (thread_num, extract_dir))
+ if os.path.exists(extract_dir):
+ try:
+ shutil.rmtree(extract_dir)
+ out_q.put('%d> Removed %s...' % (thread_num, extract_dir))
+ except OSError:
+ out_q.put('%d> Warning: Can\'t delete: %s' % (
+ thread_num, extract_dir))
+ ret_codes.put((1, 'Can\'t delete %s.' % (extract_dir)))
+ continue
+ out_q.put('%d> Extracting %s to %s' % (thread_num, output_filename,
+ extract_dir))
+ tar.extractall(path=dirname)
# Set executable bit.
if sys.platform == 'cygwin':
# Under cygwin, mark all files as executable. The executable flag in
@@ -267,7 +302,7 @@
def download_from_google_storage(
input_filename, base_url, gsutil, num_threads, directory, recursive,
- force, output, ignore_errors, sha1_file, verbose, auto_platform):
+ force, output, ignore_errors, sha1_file, verbose, auto_platform, extract):
# Start up all the worker threads.
all_threads = []
download_start = time.time()
@@ -279,7 +314,7 @@
t = threading.Thread(
target=_downloader_worker_thread,
args=[thread_num, work_queue, force, base_url,
- gsutil, stdout_queue, ret_codes, verbose])
+ gsutil, stdout_queue, ret_codes, verbose, extract])
t.daemon = True
t.start()
all_threads.append(t)
@@ -367,6 +402,13 @@
'(linux|mac|win). If so, the script will only '
'process files that are in the paths that '
'that matches the current platform.')
+ parser.add_option('-u', '--extract',
+ action='store_true',
+ help='Extract a downloaded tar.gz file. '
+ 'Leaves the tar.gz file around for sha1 verification'
+ 'If a directory with the same name as the tar.gz '
+ 'file already exists, is deleted (to get a '
+ 'clean state in case of update.)')
parser.add_option('-v', '--verbose', action='store_true',
help='Output extra diagnostic and progress information.')
@@ -463,7 +505,8 @@
return download_from_google_storage(
input_filename, base_url, gsutil, options.num_threads, options.directory,
options.recursive, options.force, options.output, options.ignore_errors,
- options.sha1_file, options.verbose, options.auto_platform)
+ options.sha1_file, options.verbose, options.auto_platform,
+ options.extract)
if __name__ == '__main__':
« no previous file with comments | « no previous file | upload_to_google_storage.py » ('j') | upload_to_google_storage.py » ('J')

Powered by Google App Engine
This is Rietveld 408576698