| OLD | NEW |
| 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" | 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" |
| 6 | 6 |
| 7 import cStringIO | 7 import cStringIO |
| 8 import hashlib | 8 import hashlib |
| 9 import logging | 9 import logging |
| 10 import os | 10 import os |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 102 raise NotFoundError(stderr) | 102 raise NotFoundError(stderr) |
| 103 raise CloudStorageError(stderr) | 103 raise CloudStorageError(stderr) |
| 104 | 104 |
| 105 return stdout | 105 return stdout |
| 106 | 106 |
| 107 | 107 |
| 108 def List(bucket): | 108 def List(bucket): |
| 109 stdout = _RunCommand(['ls', 'gs://%s' % bucket]) | 109 stdout = _RunCommand(['ls', 'gs://%s' % bucket]) |
| 110 return [url.split('/')[-1] for url in stdout.splitlines()] | 110 return [url.split('/')[-1] for url in stdout.splitlines()] |
| 111 | 111 |
| 112 def Exists(bucket, remote_path): |
| 113 try: |
| 114 _RunCommand(['ls', 'gs://%s/%s' % (bucket, remote_path)]) |
| 115 return True |
| 116 except NotFoundError: |
| 117 return False |
| 112 | 118 |
| 113 def Delete(bucket, remote_path): | 119 def Delete(bucket, remote_path): |
| 114 url = 'gs://%s/%s' % (bucket, remote_path) | 120 url = 'gs://%s/%s' % (bucket, remote_path) |
| 115 logging.info('Deleting %s' % url) | 121 logging.info('Deleting %s' % url) |
| 116 _RunCommand(['rm', url]) | 122 _RunCommand(['rm', url]) |
| 117 | 123 |
| 118 | 124 |
| 119 def Get(bucket, remote_path, local_path): | 125 def Get(bucket, remote_path, local_path): |
| 120 url = 'gs://%s/%s' % (bucket, remote_path) | 126 url = 'gs://%s/%s' % (bucket, remote_path) |
| 121 logging.info('Downloading %s to %s' % (url, local_path)) | 127 logging.info('Downloading %s to %s' % (url, local_path)) |
| 122 _RunCommand(['cp', url, local_path]) | 128 _RunCommand(['cp', url, local_path]) |
| 123 | 129 |
| 124 | 130 |
| 125 def Insert(bucket, remote_path, local_path): | 131 def Insert(bucket, remote_path, local_path, publicly_readable=False): |
| 126 url = 'gs://%s/%s' % (bucket, remote_path) | 132 url = 'gs://%s/%s' % (bucket, remote_path) |
| 127 logging.info('Uploading %s to %s' % (local_path, url)) | 133 command_and_args = ['cp'] |
| 128 _RunCommand(['cp', local_path, url]) | 134 extra_info = '' |
| 135 if publicly_readable: |
| 136 command_and_args += ['-a', 'public-read'] |
| 137 extra_info = ' (publicly readable)' |
| 138 command_and_args += [local_path, url] |
| 139 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) |
| 140 _RunCommand(command_and_args) |
| 129 | 141 |
| 130 | 142 |
| 131 def GetIfChanged(bucket, file_path): | 143 def GetIfChanged(bucket, file_path): |
| 132 """Gets the file at file_path if it has a hash file that doesn't match. | 144 """Gets the file at file_path if it has a hash file that doesn't match. |
| 133 | 145 |
| 134 If the file is not in Cloud Storage, log a warning instead of raising an | 146 If the file is not in Cloud Storage, log a warning instead of raising an |
| 135 exception. We assume that the user just hasn't uploaded the file yet. | 147 exception. We assume that the user just hasn't uploaded the file yet. |
| 136 | 148 |
| 137 Returns: | 149 Returns: |
| 138 True if the binary was changed. | 150 True if the binary was changed. |
| (...skipping 20 matching lines...) Expand all Loading... |
| 159 """Calculates and returns the hash of the file at file_path.""" | 171 """Calculates and returns the hash of the file at file_path.""" |
| 160 sha1 = hashlib.sha1() | 172 sha1 = hashlib.sha1() |
| 161 with open(file_path, 'rb') as f: | 173 with open(file_path, 'rb') as f: |
| 162 while True: | 174 while True: |
| 163 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. | 175 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. |
| 164 chunk = f.read(1024*1024) | 176 chunk = f.read(1024*1024) |
| 165 if not chunk: | 177 if not chunk: |
| 166 break | 178 break |
| 167 sha1.update(chunk) | 179 sha1.update(chunk) |
| 168 return sha1.hexdigest() | 180 return sha1.hexdigest() |
| OLD | NEW |