Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(75)

Side by Side Diff: upload_to_google_storage.py

Issue 1209033006: Revert of Add support for tar.gz archive files to download from download_from_google_storage (Closed) Base URL: http://src.chromium.org/svn/trunk/tools/depot_tools/
Patch Set: Created 5 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « tests/upload_to_google_storage_unittests.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Uploads files to Google Storage content addressed.""" 6 """Uploads files to Google Storage content addressed."""
7 7
8 import hashlib 8 import hashlib
9 import optparse 9 import optparse
10 import os 10 import os
11 import Queue 11 import Queue
12 import re 12 import re
13 import stat 13 import stat
14 import sys 14 import sys
15 import tarfile
16 import threading 15 import threading
17 import time 16 import time
18 17
19 from download_from_google_storage import get_sha1 18 from download_from_google_storage import get_sha1
20 from download_from_google_storage import Gsutil 19 from download_from_google_storage import Gsutil
21 from download_from_google_storage import printer_worker 20 from download_from_google_storage import printer_worker
22 from download_from_google_storage import GSUTIL_DEFAULT_PATH 21 from download_from_google_storage import GSUTIL_DEFAULT_PATH
23 22
24 USAGE_STRING = """%prog [options] target [target2 ...]. 23 USAGE_STRING = """%prog [options] target [target2 ...].
25 Target is the file intended to be uploaded to Google Storage. 24 Target is the file intended to be uploaded to Google Storage.
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after
201 max_ret_code = max(ret_code, max_ret_code) 200 max_ret_code = max(ret_code, max_ret_code)
202 if message: 201 if message:
203 print >> sys.stderr, message 202 print >> sys.stderr, message
204 203
205 if not max_ret_code: 204 if not max_ret_code:
206 print 'Success!' 205 print 'Success!'
207 206
208 return max_ret_code 207 return max_ret_code
209 208
210 209
211 def create_archives(dirs):
212 archive_names = []
213 for name in dirs:
214 tarname = '%s.tar.gz' % name
215 with tarfile.open(tarname, 'w:gz') as tar:
216 tar.add(name)
217 archive_names.append(tarname)
218 return archive_names
219
220
221 def validate_archive_dirs(dirs):
222 # We don't allow .. in paths in our archives.
223 if any(map(lambda x: '..' in x, dirs)):
224 return False
225 # We only allow dirs.
226 if any(map(lambda x: not os.path.isdir(x), dirs)):
227 return False
228 # We don't allow sym links in our archives.
229 if any(map(os.path.islink, dirs)):
230 return False
231 # We required that the subdirectories we are archiving are all just below
232 # cwd.
233 return not any(map(lambda x: x not in next(os.walk('.'))[1], dirs))
234
235
236 def main(): 210 def main():
237 parser = optparse.OptionParser(USAGE_STRING) 211 parser = optparse.OptionParser(USAGE_STRING)
238 parser.add_option('-b', '--bucket', 212 parser.add_option('-b', '--bucket',
239 help='Google Storage bucket to upload to.') 213 help='Google Storage bucket to upload to.')
240 parser.add_option('-e', '--boto', help='Specify a custom boto file.') 214 parser.add_option('-e', '--boto', help='Specify a custom boto file.')
241 parser.add_option('-z', '--archive', action='store_true',
242 help='Archive directory as a tar.gz file')
243 parser.add_option('-f', '--force', action='store_true', 215 parser.add_option('-f', '--force', action='store_true',
244 help='Force upload even if remote file exists.') 216 help='Force upload even if remote file exists.')
245 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH, 217 parser.add_option('-g', '--gsutil_path', default=GSUTIL_DEFAULT_PATH,
246 help='Path to the gsutil script.') 218 help='Path to the gsutil script.')
247 parser.add_option('-m', '--use_md5', action='store_true', 219 parser.add_option('-m', '--use_md5', action='store_true',
248 help='Generate MD5 files when scanning, and don\'t check ' 220 help='Generate MD5 files when scanning, and don\'t check '
249 'the MD5 checksum if a .md5 file is found.') 221 'the MD5 checksum if a .md5 file is found.')
250 parser.add_option('-t', '--num_threads', default=1, type='int', 222 parser.add_option('-t', '--num_threads', default=1, type='int',
251 help='Number of uploader threads to run.') 223 help='Number of uploader threads to run.')
252 parser.add_option('-s', '--skip_hashing', action='store_true', 224 parser.add_option('-s', '--skip_hashing', action='store_true',
253 help='Skip hashing if .sha1 file exists.') 225 help='Skip hashing if .sha1 file exists.')
254 parser.add_option('-0', '--use_null_terminator', action='store_true', 226 parser.add_option('-0', '--use_null_terminator', action='store_true',
255 help='Use \\0 instead of \\n when parsing ' 227 help='Use \\0 instead of \\n when parsing '
256 'the file list from stdin. This is useful if the input ' 228 'the file list from stdin. This is useful if the input '
257 'is coming from "find ... -print0".') 229 'is coming from "find ... -print0".')
258 parser.add_option('-z', '--gzip', metavar='ext', 230 parser.add_option('-z', '--gzip', metavar='ext',
259 help='Gzip files which end in ext. ' 231 help='Gzip files which end in ext. '
260 'ext is a comma-separated list') 232 'ext is a comma-separated list')
261 (options, args) = parser.parse_args() 233 (options, args) = parser.parse_args()
262 234
263 # Enumerate our inputs. 235 # Enumerate our inputs.
264 input_filenames = get_targets(args, parser, options.use_null_terminator) 236 input_filenames = get_targets(args, parser, options.use_null_terminator)
265 237
266 if options.archive:
267 if not validate_archive_dirs(input_filenames):
268 parser.error('Only directories just below cwd are valid entries when '
269 'using the --archive argument. Entries can not contain .. '
270 ' and entries can not be symlinks. Entries was %s' %
271 input_filenames)
272 return 1
273 input_filenames = create_archives(input_filenames)
274
275 # Make sure we can find a working instance of gsutil. 238 # Make sure we can find a working instance of gsutil.
276 if os.path.exists(GSUTIL_DEFAULT_PATH): 239 if os.path.exists(GSUTIL_DEFAULT_PATH):
277 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto) 240 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
278 else: 241 else:
279 gsutil = None 242 gsutil = None
280 for path in os.environ["PATH"].split(os.pathsep): 243 for path in os.environ["PATH"].split(os.pathsep):
281 if os.path.exists(path) and 'gsutil' in os.listdir(path): 244 if os.path.exists(path) and 'gsutil' in os.listdir(path):
282 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) 245 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
283 if not gsutil: 246 if not gsutil:
284 parser.error('gsutil not found in %s, bad depot_tools checkout?' % 247 parser.error('gsutil not found in %s, bad depot_tools checkout?' %
285 GSUTIL_DEFAULT_PATH) 248 GSUTIL_DEFAULT_PATH)
286 249
287 base_url = 'gs://%s' % options.bucket 250 base_url = 'gs://%s' % options.bucket
288 251
289 return upload_to_google_storage( 252 return upload_to_google_storage(
290 input_filenames, base_url, gsutil, options.force, options.use_md5, 253 input_filenames, base_url, gsutil, options.force, options.use_md5,
291 options.num_threads, options.skip_hashing, options.gzip) 254 options.num_threads, options.skip_hashing, options.gzip)
292 255
293 256
294 if __name__ == '__main__': 257 if __name__ == '__main__':
295 try: 258 try:
296 sys.exit(main()) 259 sys.exit(main())
297 except KeyboardInterrupt: 260 except KeyboardInterrupt:
298 sys.stderr.write('interrupted\n') 261 sys.stderr.write('interrupted\n')
299 sys.exit(1) 262 sys.exit(1)
OLDNEW
« no previous file with comments | « tests/upload_to_google_storage_unittests.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698