Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(360)

Side by Side Diff: gsutil.py

Issue 870093003: Hook sys.stdio directly to the gsutil subprocess for the gsutil call (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools
Patch Set: Fix tests Created 5 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | tests/gsutil_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Run a pinned gsutil.""" 6 """Run a pinned gsutil."""
7 7
8 8
9 import argparse 9 import argparse
10 import base64 10 import base64
11 import hashlib 11 import hashlib
12 import json 12 import json
13 import os 13 import os
14 import shutil 14 import shutil
15 import subprocess 15 import subprocess
16 import sys 16 import sys
17 import urllib2 17 import urllib2
18 import zipfile 18 import zipfile
19 19
20 20
21 GSUTIL_URL = 'https://storage.googleapis.com/pub/' 21 GSUTIL_URL = 'https://storage.googleapis.com/pub/'
22 API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/' 22 API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/'
23 23
24 THIS_DIR = os.path.dirname(os.path.abspath(__file__)) 24 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
25 DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil') 25 DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil')
26 DEFAULT_FALLBACK_GSUTIL = os.path.join( 26 DEFAULT_FALLBACK_GSUTIL = os.path.join(
27 THIS_DIR, 'third_party', 'gsutil', 'gsutil') 27 THIS_DIR, 'third_party', 'gsutil', 'gsutil')
28 28
29 29
30 class SubprocessError(Exception):
31 def __init__(self, message=None, code=0):
32 super(SubprocessError, self).__init__(message)
33 self.code = code
34
35
36 class InvalidGsutilError(Exception): 30 class InvalidGsutilError(Exception):
37 pass 31 pass
38 32
39 33
40 def call(args, verbose=True, **kwargs):
41 kwargs['stdout'] = subprocess.PIPE
42 kwargs['stderr'] = subprocess.STDOUT
43 proc = subprocess.Popen(args, **kwargs)
44 out = []
45 for line in proc.stdout:
46 out.append(line)
47 if verbose:
48 sys.stdout.write(line)
49 code = proc.wait()
50 if code:
51 raise SubprocessError('%s failed with %s' % (args, code), code)
52 return ''.join(out)
53
54
55 def download_gsutil(version, target_dir): 34 def download_gsutil(version, target_dir):
56 """Downloads gsutil into the target_dir.""" 35 """Downloads gsutil into the target_dir."""
57 filename = 'gsutil_%s.zip' % version 36 filename = 'gsutil_%s.zip' % version
58 target_filename = os.path.join(target_dir, filename) 37 target_filename = os.path.join(target_dir, filename)
59 38
60 # Check if the target exists already. 39 # Check if the target exists already.
61 if os.path.exists(target_filename): 40 if os.path.exists(target_filename):
62 md5_calc = hashlib.md5() 41 md5_calc = hashlib.md5()
63 with open(target_filename, 'rb') as f: 42 with open(target_filename, 'rb') as f:
64 while True: 43 while True:
(...skipping 18 matching lines...) Expand all
83 while True: 62 while True:
84 buf = u.read(4096) 63 buf = u.read(4096)
85 if not buf: 64 if not buf:
86 break 65 break
87 f.write(buf) 66 f.write(buf)
88 return target_filename 67 return target_filename
89 68
90 69
91 def check_gsutil(gsutil_bin): 70 def check_gsutil(gsutil_bin):
92 """Run gsutil version and make sure it runs.""" 71 """Run gsutil version and make sure it runs."""
93 try: 72 return subprocess.call(
94 call([sys.executable, gsutil_bin, 'version'], verbose=False) 73 [sys.executable, gsutil_bin, 'version'],
95 return True 74 stdout=subprocess.PIPE, stderr=subprocess.STDOUT) == 0
96 except SubprocessError:
97 return False
98
99 75
100 def ensure_gsutil(version, target): 76 def ensure_gsutil(version, target):
101 bin_dir = os.path.join(target, 'gsutil_%s' % version) 77 bin_dir = os.path.join(target, 'gsutil_%s' % version)
102 gsutil_bin = os.path.join(bin_dir, 'gsutil', 'gsutil') 78 gsutil_bin = os.path.join(bin_dir, 'gsutil', 'gsutil')
103 if os.path.isfile(gsutil_bin) and check_gsutil(gsutil_bin): 79 if os.path.isfile(gsutil_bin) and check_gsutil(gsutil_bin):
104 # Everything is awesome! we're all done here. 80 # Everything is awesome! we're all done here.
105 return gsutil_bin 81 return gsutil_bin
106 82
107 if os.path.isdir(bin_dir): 83 if os.path.isdir(bin_dir):
108 # Clean up if we're redownloading a corrupted gsutil. 84 # Clean up if we're redownloading a corrupted gsutil.
(...skipping 11 matching lines...) Expand all
120 96
121 return gsutil_bin 97 return gsutil_bin
122 98
123 99
124 def run_gsutil(force_version, fallback, target, args): 100 def run_gsutil(force_version, fallback, target, args):
125 if force_version: 101 if force_version:
126 gsutil_bin = ensure_gsutil(force_version, target) 102 gsutil_bin = ensure_gsutil(force_version, target)
127 else: 103 else:
128 gsutil_bin = fallback 104 gsutil_bin = fallback
129 cmd = [sys.executable, gsutil_bin] + args 105 cmd = [sys.executable, gsutil_bin] + args
130 try: 106 return subprocess.call(cmd)
131 call(cmd)
132 except SubprocessError as e:
133 return e.code
134 return 0
135 107
136 108
137 def parse_args(): 109 def parse_args():
138 parser = argparse.ArgumentParser() 110 parser = argparse.ArgumentParser()
139 parser.add_argument('--force-version') 111 parser.add_argument('--force-version')
140 parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL) 112 parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL)
141 parser.add_argument('--target', default=DEFAULT_BIN_DIR) 113 parser.add_argument('--target', default=DEFAULT_BIN_DIR)
142 parser.add_argument('args', nargs=argparse.REMAINDER) 114 parser.add_argument('args', nargs=argparse.REMAINDER)
143 115
144 args, extras = parser.parse_known_args() 116 args, extras = parser.parse_known_args()
145 if args.args and args.args[0] == '--': 117 if args.args and args.args[0] == '--':
146 args.args.pop(0) 118 args.args.pop(0)
147 if extras: 119 if extras:
148 args.args = extras + args.args 120 args.args = extras + args.args
149 return args.force_version, args.fallback, args.target, args.args 121 return args.force_version, args.fallback, args.target, args.args
150 122
151 123
152 def main(): 124 def main():
153 force_version, fallback, target, args = parse_args() 125 force_version, fallback, target, args = parse_args()
154 return run_gsutil(force_version, fallback, target, args) 126 return run_gsutil(force_version, fallback, target, args)
155 127
156 if __name__ == '__main__': 128 if __name__ == '__main__':
157 sys.exit(main()) 129 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | tests/gsutil_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698