OLD | NEW |
1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" | 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" |
6 | 6 |
7 import cStringIO | 7 import cStringIO |
8 import hashlib | 8 import hashlib |
9 import logging | 9 import logging |
10 import os | 10 import os |
(...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
169 """Gets the file at file_path if it has a hash file that doesn't match. | 169 """Gets the file at file_path if it has a hash file that doesn't match. |
170 | 170 |
171 If the file is not in Cloud Storage, log a warning instead of raising an | 171 If the file is not in Cloud Storage, log a warning instead of raising an |
172 exception. We assume that the user just hasn't uploaded the file yet. | 172 exception. We assume that the user just hasn't uploaded the file yet. |
173 | 173 |
174 Returns: | 174 Returns: |
175 True if the binary was changed. | 175 True if the binary was changed. |
176 """ | 176 """ |
177 hash_path = file_path + '.sha1' | 177 hash_path = file_path + '.sha1' |
178 if not os.path.exists(hash_path): | 178 if not os.path.exists(hash_path): |
| 179 logging.warning('Hash file not found: %s' % hash_path) |
179 return False | 180 return False |
180 | 181 |
181 expected_hash = ReadHash(hash_path) | 182 expected_hash = ReadHash(hash_path) |
182 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: | 183 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: |
| 184 logging.info('File up to date: %s' % file_path) |
183 return False | 185 return False |
184 | 186 |
185 if bucket: | 187 if bucket: |
186 buckets = [bucket] | 188 buckets = [bucket] |
187 else: | 189 else: |
188 buckets = [PUBLIC_BUCKET, PARTNER_BUCKET, INTERNAL_BUCKET] | 190 buckets = [PUBLIC_BUCKET, PARTNER_BUCKET, INTERNAL_BUCKET] |
189 | 191 |
190 found = False | 192 found = False |
191 for bucket in buckets: | 193 for bucket in buckets: |
192 try: | 194 try: |
193 url = 'gs://%s/%s' % (bucket, expected_hash) | 195 url = 'gs://%s/%s' % (bucket, expected_hash) |
| 196 logging.info('Running gsutil command: cp %s %s' % (url, file_path)) |
194 _RunCommand(['cp', url, file_path]) | 197 _RunCommand(['cp', url, file_path]) |
195 logging.info('Downloaded %s to %s' % (url, file_path)) | 198 logging.info('Downloaded %s to %s' % (url, file_path)) |
196 found = True | 199 found = True |
197 except NotFoundError: | 200 except NotFoundError: |
198 continue | 201 continue |
199 | 202 |
200 if not found: | 203 if not found: |
201 logging.warning('Unable to find file in Cloud Storage: %s', file_path) | 204 logging.warning('Unable to find file in Cloud Storage: %s', file_path) |
202 return found | 205 return found |
203 | 206 |
204 | 207 |
205 def CalculateHash(file_path): | 208 def CalculateHash(file_path): |
206 """Calculates and returns the hash of the file at file_path.""" | 209 """Calculates and returns the hash of the file at file_path.""" |
207 sha1 = hashlib.sha1() | 210 sha1 = hashlib.sha1() |
208 with open(file_path, 'rb') as f: | 211 with open(file_path, 'rb') as f: |
209 while True: | 212 while True: |
210 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. | 213 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. |
211 chunk = f.read(1024*1024) | 214 chunk = f.read(1024*1024) |
212 if not chunk: | 215 if not chunk: |
213 break | 216 break |
214 sha1.update(chunk) | 217 sha1.update(chunk) |
215 return sha1.hexdigest() | 218 return sha1.hexdigest() |
216 | 219 |
217 | 220 |
218 def ReadHash(hash_path): | 221 def ReadHash(hash_path): |
219 with open(hash_path, 'rb') as f: | 222 with open(hash_path, 'rb') as f: |
220 return f.read(1024).rstrip() | 223 return f.read(1024).rstrip() |
OLD | NEW |