OLD | NEW |
---|---|
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" | 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" |
6 | 6 |
7 import collections | 7 import collections |
8 import contextlib | 8 import contextlib |
9 import cStringIO | 9 import cStringIO |
10 import hashlib | 10 import hashlib |
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
196 | 196 |
197 It should: cause no changes locally or to the starting file, and will | 197 It should: cause no changes locally or to the starting file, and will |
198 overwrite any existing files in the destination location. | 198 overwrite any existing files in the destination location. |
199 """ | 199 """ |
200 url1 = 'gs://%s/%s' % (bucket_from, remote_path_from) | 200 url1 = 'gs://%s/%s' % (bucket_from, remote_path_from) |
201 url2 = 'gs://%s/%s' % (bucket_to, remote_path_to) | 201 url2 = 'gs://%s/%s' % (bucket_to, remote_path_to) |
202 logging.info('Copying %s to %s' % (url1, url2)) | 202 logging.info('Copying %s to %s' % (url1, url2)) |
203 _RunCommand(['cp', url1, url2]) | 203 _RunCommand(['cp', url1, url2]) |
204 | 204 |
205 | 205 |
206 def Delete(bucket, remote_path): | 206 def Delete(bucket, remote_path, recursive=False): |
erikchen
2015/07/14 23:36:15
If you do a zip upload, you won't need to add this
sydli
2015/07/15 22:56:24
Done.
| |
207 url = 'gs://%s/%s' % (bucket, remote_path) | 207 url = 'gs://%s/%s' % (bucket, remote_path) |
208 command_and_args = ['rm'] | |
209 if recursive: | |
210 command_and_args += ['-r'] | |
211 command_and_args += [url] | |
208 logging.info('Deleting %s' % url) | 212 logging.info('Deleting %s' % url) |
209 _RunCommand(['rm', url]) | 213 _RunCommand(command_and_args) |
210 | 214 |
211 | 215 |
212 def Get(bucket, remote_path, local_path): | 216 def Get(bucket, remote_path, local_path): |
213 url = 'gs://%s/%s' % (bucket, remote_path) | 217 url = 'gs://%s/%s' % (bucket, remote_path) |
214 logging.info('Downloading %s to %s' % (url, local_path)) | 218 logging.info('Downloading %s to %s' % (url, local_path)) |
215 try: | 219 try: |
216 _RunCommand(['cp', url, local_path]) | 220 _RunCommand(['cp', url, local_path]) |
217 except ServerError: | 221 except ServerError: |
218 logging.info('Cloud Storage server error, retrying download') | 222 logging.info('Cloud Storage server error, retrying download') |
219 _RunCommand(['cp', url, local_path]) | 223 _RunCommand(['cp', url, local_path]) |
220 | 224 |
221 | 225 |
222 def Insert(bucket, remote_path, local_path, publicly_readable=False): | 226 def Insert(bucket, remote_path, local_path, recursive=False, |
227 publicly_readable=False): | |
223 """ Upload file in |local_path| to cloud storage. | 228 """ Upload file in |local_path| to cloud storage. |
224 Args: | 229 Args: |
225 bucket: the google cloud storage bucket name. | 230 bucket: the google cloud storage bucket name. |
226 remote_path: the remote file path in |bucket|. | 231 remote_path: the remote file path in |bucket|. |
227 local_path: path of the local file to be uploaded. | 232 local_path: path of the local file to be uploaded. |
228 publicly_readable: whether the uploaded file has publicly readable | 233 publicly_readable: whether the uploaded file has publicly readable |
229 permission. | 234 permission. |
230 | 235 |
231 Returns: | 236 Returns: |
232 The url where the file is uploaded to. | 237 The url where the file is uploaded to. |
233 """ | 238 """ |
234 url = 'gs://%s/%s' % (bucket, remote_path) | 239 url = 'gs://%s/%s' % (bucket, remote_path) |
235 command_and_args = ['cp'] | 240 command_and_args = ['cp'] |
236 extra_info = '' | 241 extra_info = '' |
242 if recursive: | |
243 command_and_args += ['-r'] | |
237 if publicly_readable: | 244 if publicly_readable: |
238 command_and_args += ['-a', 'public-read'] | 245 command_and_args += ['-a', 'public-read'] |
239 extra_info = ' (publicly readable)' | 246 extra_info = ' (publicly readable)' |
240 command_and_args += [local_path, url] | 247 command_and_args += [local_path, url] |
241 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) | 248 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) |
242 _RunCommand(command_and_args) | 249 _RunCommand(command_and_args) |
243 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( | 250 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( |
244 bucket, remote_path) | 251 bucket, remote_path) |
245 | 252 |
246 | 253 |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
296 chunk = f.read(1024*1024) | 303 chunk = f.read(1024*1024) |
297 if not chunk: | 304 if not chunk: |
298 break | 305 break |
299 sha1.update(chunk) | 306 sha1.update(chunk) |
300 return sha1.hexdigest() | 307 return sha1.hexdigest() |
301 | 308 |
302 | 309 |
303 def ReadHash(hash_path): | 310 def ReadHash(hash_path): |
304 with open(hash_path, 'rb') as f: | 311 with open(hash_path, 'rb') as f: |
305 return f.read(1024).rstrip() | 312 return f.read(1024).rstrip() |
OLD | NEW |