Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" | 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" |
| 6 | 6 |
| 7 import collections | 7 import collections |
| 8 import contextlib | 8 import contextlib |
| 9 import cStringIO | 9 import cStringIO |
| 10 import hashlib | 10 import hashlib |
| (...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 196 | 196 |
| 197 It should: cause no changes locally or to the starting file, and will | 197 It should: cause no changes locally or to the starting file, and will |
| 198 overwrite any existing files in the destination location. | 198 overwrite any existing files in the destination location. |
| 199 """ | 199 """ |
| 200 url1 = 'gs://%s/%s' % (bucket_from, remote_path_from) | 200 url1 = 'gs://%s/%s' % (bucket_from, remote_path_from) |
| 201 url2 = 'gs://%s/%s' % (bucket_to, remote_path_to) | 201 url2 = 'gs://%s/%s' % (bucket_to, remote_path_to) |
| 202 logging.info('Copying %s to %s' % (url1, url2)) | 202 logging.info('Copying %s to %s' % (url1, url2)) |
| 203 _RunCommand(['cp', url1, url2]) | 203 _RunCommand(['cp', url1, url2]) |
| 204 | 204 |
| 205 | 205 |
| 206 def Delete(bucket, remote_path): | 206 def Delete(bucket, remote_path, recursive=False): |
|
erikchen
2015/07/14 23:36:15
If you do a zip upload, you won't need to add this
sydli
2015/07/15 22:56:24
Done.
| |
| 207 url = 'gs://%s/%s' % (bucket, remote_path) | 207 url = 'gs://%s/%s' % (bucket, remote_path) |
| 208 command_and_args = ['rm'] | |
| 209 if recursive: | |
| 210 command_and_args += ['-r'] | |
| 211 command_and_args += [url] | |
| 208 logging.info('Deleting %s' % url) | 212 logging.info('Deleting %s' % url) |
| 209 _RunCommand(['rm', url]) | 213 _RunCommand(command_and_args) |
| 210 | 214 |
| 211 | 215 |
| 212 def Get(bucket, remote_path, local_path): | 216 def Get(bucket, remote_path, local_path): |
| 213 url = 'gs://%s/%s' % (bucket, remote_path) | 217 url = 'gs://%s/%s' % (bucket, remote_path) |
| 214 logging.info('Downloading %s to %s' % (url, local_path)) | 218 logging.info('Downloading %s to %s' % (url, local_path)) |
| 215 try: | 219 try: |
| 216 _RunCommand(['cp', url, local_path]) | 220 _RunCommand(['cp', url, local_path]) |
| 217 except ServerError: | 221 except ServerError: |
| 218 logging.info('Cloud Storage server error, retrying download') | 222 logging.info('Cloud Storage server error, retrying download') |
| 219 _RunCommand(['cp', url, local_path]) | 223 _RunCommand(['cp', url, local_path]) |
| 220 | 224 |
| 221 | 225 |
| 222 def Insert(bucket, remote_path, local_path, publicly_readable=False): | 226 def Insert(bucket, remote_path, local_path, recursive=False, |
| 227 publicly_readable=False): | |
| 223 """ Upload file in |local_path| to cloud storage. | 228 """ Upload file in |local_path| to cloud storage. |
| 224 Args: | 229 Args: |
| 225 bucket: the google cloud storage bucket name. | 230 bucket: the google cloud storage bucket name. |
| 226 remote_path: the remote file path in |bucket|. | 231 remote_path: the remote file path in |bucket|. |
| 227 local_path: path of the local file to be uploaded. | 232 local_path: path of the local file to be uploaded. |
| 228 publicly_readable: whether the uploaded file has publicly readable | 233 publicly_readable: whether the uploaded file has publicly readable |
| 229 permission. | 234 permission. |
| 230 | 235 |
| 231 Returns: | 236 Returns: |
| 232 The url where the file is uploaded to. | 237 The url where the file is uploaded to. |
| 233 """ | 238 """ |
| 234 url = 'gs://%s/%s' % (bucket, remote_path) | 239 url = 'gs://%s/%s' % (bucket, remote_path) |
| 235 command_and_args = ['cp'] | 240 command_and_args = ['cp'] |
| 236 extra_info = '' | 241 extra_info = '' |
| 242 if recursive: | |
| 243 command_and_args += ['-r'] | |
| 237 if publicly_readable: | 244 if publicly_readable: |
| 238 command_and_args += ['-a', 'public-read'] | 245 command_and_args += ['-a', 'public-read'] |
| 239 extra_info = ' (publicly readable)' | 246 extra_info = ' (publicly readable)' |
| 240 command_and_args += [local_path, url] | 247 command_and_args += [local_path, url] |
| 241 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) | 248 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) |
| 242 _RunCommand(command_and_args) | 249 _RunCommand(command_and_args) |
| 243 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( | 250 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( |
| 244 bucket, remote_path) | 251 bucket, remote_path) |
| 245 | 252 |
| 246 | 253 |
| (...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 296 chunk = f.read(1024*1024) | 303 chunk = f.read(1024*1024) |
| 297 if not chunk: | 304 if not chunk: |
| 298 break | 305 break |
| 299 sha1.update(chunk) | 306 sha1.update(chunk) |
| 300 return sha1.hexdigest() | 307 return sha1.hexdigest() |
| 301 | 308 |
| 302 | 309 |
| 303 def ReadHash(hash_path): | 310 def ReadHash(hash_path): |
| 304 with open(hash_path, 'rb') as f: | 311 with open(hash_path, 'rb') as f: |
| 305 return f.read(1024).rstrip() | 312 return f.read(1024).rstrip() |
| OLD | NEW |