Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(685)

Side by Side Diff: upload_to_google_storage.py

Issue 797663003: Use gsutil.py for download_from_google_storage instead of the builtin one (Closed) Base URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git@master
Patch Set: Also modifies references in git_cache, upload_to_google_storage Created 6 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « git_cache.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Uploads files to Google Storage content addressed.""" 6 """Uploads files to Google Storage content addressed."""
7 7
8 import hashlib 8 import hashlib
9 import optparse 9 import optparse
10 import os 10 import os
11 import Queue 11 import Queue
12 import re 12 import re
13 import stat 13 import stat
14 import sys 14 import sys
15 import threading 15 import threading
16 import time 16 import time
17 17
18 from download_from_google_storage import check_bucket_permissions 18 from download_from_google_storage import check_bucket_permissions
19 from download_from_google_storage import get_sha1 19 from download_from_google_storage import get_sha1
20 from download_from_google_storage import Gsutil 20 from download_from_google_storage import Gsutil
21 from download_from_google_storage import printer_worker 21 from download_from_google_storage import printer_worker
22 22 from download_from_google_storage import GSUTIL_DEFAULT_PATH
23 GSUTIL_DEFAULT_PATH = os.path.join(
24 os.path.dirname(os.path.abspath(__file__)),
25 'third_party', 'gsutil', 'gsutil')
26 23
27 USAGE_STRING = """%prog [options] target [target2 ...]. 24 USAGE_STRING = """%prog [options] target [target2 ...].
28 Target is the file intended to be uploaded to Google Storage. 25 Target is the file intended to be uploaded to Google Storage.
29 If target is "-", then a list of files will be taken from standard input 26 If target is "-", then a list of files will be taken from standard input
30 27
31 This script will generate a file (original filename).sha1 containing the 28 This script will generate a file (original filename).sha1 containing the
32 sha1 sum of the uploaded file. 29 sha1 sum of the uploaded file.
33 It is recommended that the .sha1 file is checked into the repository, 30 It is recommended that the .sha1 file is checked into the repository,
34 the original file removed from the repository, and a hook added to the 31 the original file removed from the repository, and a hook added to the
35 DEPS file to call download_from_google_storage.py. 32 DEPS file to call download_from_google_storage.py.
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
89 local_md5 = get_md5_cached(filename) 86 local_md5 = get_md5_cached(filename)
90 else: 87 else:
91 local_md5 = get_md5(filename) 88 local_md5 = get_md5(filename)
92 if local_md5 == remote_md5: 89 if local_md5 == remote_md5:
93 stdout_queue.put( 90 stdout_queue.put(
94 '%d> File %s already exists and MD5 matches, upload skipped' % 91 '%d> File %s already exists and MD5 matches, upload skipped' %
95 (thread_num, filename)) 92 (thread_num, filename))
96 continue 93 continue
97 stdout_queue.put('%d> Uploading %s...' % ( 94 stdout_queue.put('%d> Uploading %s...' % (
98 thread_num, filename)) 95 thread_num, filename))
99 code, _, err = gsutil.check_call('cp', '-q', filename, file_url) 96 code, _, err = gsutil.check_call('cp', filename, file_url)
100 if code != 0: 97 if code != 0:
101 ret_codes.put( 98 ret_codes.put(
102 (code, 99 (code,
103 'Encountered error on uploading %s to %s\n%s' % 100 'Encountered error on uploading %s to %s\n%s' %
104 (filename, file_url, err))) 101 (filename, file_url, err)))
105 continue 102 continue
106 103
107 # Mark executable files with the header "x-goog-meta-executable: 1" which 104 # Mark executable files with the header "x-goog-meta-executable: 1" which
108 # the download script will check for to preserve the executable bit. 105 # the download script will check for to preserve the executable bit.
109 if not sys.platform.startswith('win'): 106 if not sys.platform.startswith('win'):
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after
227 help='Use \\0 instead of \\n when parsing ' 224 help='Use \\0 instead of \\n when parsing '
228 'the file list from stdin. This is useful if the input ' 225 'the file list from stdin. This is useful if the input '
229 'is coming from "find ... -print0".') 226 'is coming from "find ... -print0".')
230 (options, args) = parser.parse_args() 227 (options, args) = parser.parse_args()
231 228
232 # Enumerate our inputs. 229 # Enumerate our inputs.
233 input_filenames = get_targets(args, parser, options.use_null_terminator) 230 input_filenames = get_targets(args, parser, options.use_null_terminator)
234 231
235 # Make sure we can find a working instance of gsutil. 232 # Make sure we can find a working instance of gsutil.
236 if os.path.exists(GSUTIL_DEFAULT_PATH): 233 if os.path.exists(GSUTIL_DEFAULT_PATH):
237 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto, 234 gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
238 bypass_prodaccess=True)
239 else: 235 else:
240 gsutil = None 236 gsutil = None
241 for path in os.environ["PATH"].split(os.pathsep): 237 for path in os.environ["PATH"].split(os.pathsep):
242 if os.path.exists(path) and 'gsutil' in os.listdir(path): 238 if os.path.exists(path) and 'gsutil' in os.listdir(path):
243 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto) 239 gsutil = Gsutil(os.path.join(path, 'gsutil'), boto_path=options.boto)
244 if not gsutil: 240 if not gsutil:
245 parser.error('gsutil not found in %s, bad depot_tools checkout?' % 241 parser.error('gsutil not found in %s, bad depot_tools checkout?' %
246 GSUTIL_DEFAULT_PATH) 242 GSUTIL_DEFAULT_PATH)
247 243
248 base_url = 'gs://%s' % options.bucket 244 base_url = 'gs://%s' % options.bucket
249 245
250 # Check we have a valid bucket with valid permissions. 246 # Check we have a valid bucket with valid permissions.
251 code = check_bucket_permissions(base_url, gsutil) 247 code = check_bucket_permissions(base_url, gsutil)
252 if code: 248 if code:
253 return code 249 return code
254 250
255 return upload_to_google_storage( 251 return upload_to_google_storage(
256 input_filenames, base_url, gsutil, options.force, options.use_md5, 252 input_filenames, base_url, gsutil, options.force, options.use_md5,
257 options.num_threads, options.skip_hashing) 253 options.num_threads, options.skip_hashing)
258 254
259 255
260 if __name__ == '__main__': 256 if __name__ == '__main__':
261 sys.exit(main(sys.argv)) 257 sys.exit(main(sys.argv))
OLDNEW
« no previous file with comments | « git_cache.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698