| OLD | NEW |
| 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" | 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" |
| 6 | 6 |
| 7 import cStringIO | 7 import cStringIO |
| 8 import hashlib | 8 import hashlib |
| 9 import logging | 9 import logging |
| 10 import os | 10 import os |
| (...skipping 158 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 169 """Gets the file at file_path if it has a hash file that doesn't match. | 169 """Gets the file at file_path if it has a hash file that doesn't match. |
| 170 | 170 |
| 171 If the file is not in Cloud Storage, log a warning instead of raising an | 171 If the file is not in Cloud Storage, log a warning instead of raising an |
| 172 exception. We assume that the user just hasn't uploaded the file yet. | 172 exception. We assume that the user just hasn't uploaded the file yet. |
| 173 | 173 |
| 174 Returns: | 174 Returns: |
| 175 True if the binary was changed. | 175 True if the binary was changed. |
| 176 """ | 176 """ |
| 177 hash_path = file_path + '.sha1' | 177 hash_path = file_path + '.sha1' |
| 178 if not os.path.exists(hash_path): | 178 if not os.path.exists(hash_path): |
| 179 logging.warning('Hash file not found: %s' % hash_path) |
| 179 return False | 180 return False |
| 180 | 181 |
| 181 expected_hash = ReadHash(hash_path) | 182 expected_hash = ReadHash(hash_path) |
| 182 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: | 183 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: |
| 184 logging.info('File up to date: %s' % file_path) |
| 183 return False | 185 return False |
| 184 | 186 |
| 185 if bucket: | 187 if bucket: |
| 186 buckets = [bucket] | 188 buckets = [bucket] |
| 187 else: | 189 else: |
| 188 buckets = [PUBLIC_BUCKET, PARTNER_BUCKET, INTERNAL_BUCKET] | 190 buckets = [PUBLIC_BUCKET, PARTNER_BUCKET, INTERNAL_BUCKET] |
| 189 | 191 |
| 190 found = False | 192 found = False |
| 191 for bucket in buckets: | 193 for bucket in buckets: |
| 192 try: | 194 try: |
| 193 url = 'gs://%s/%s' % (bucket, expected_hash) | 195 url = 'gs://%s/%s' % (bucket, expected_hash) |
| 196 logging.info('Running gsutil command: cp %s %s' % (url, file_path)) |
| 194 _RunCommand(['cp', url, file_path]) | 197 _RunCommand(['cp', url, file_path]) |
| 195 logging.info('Downloaded %s to %s' % (url, file_path)) | 198 logging.info('Downloaded %s to %s' % (url, file_path)) |
| 196 found = True | 199 found = True |
| 197 except NotFoundError: | 200 except NotFoundError: |
| 198 continue | 201 continue |
| 199 | 202 |
| 200 if not found: | 203 if not found: |
| 201 logging.warning('Unable to find file in Cloud Storage: %s', file_path) | 204 logging.warning('Unable to find file in Cloud Storage: %s', file_path) |
| 202 return found | 205 return found |
| 203 | 206 |
| 204 | 207 |
| 205 def CalculateHash(file_path): | 208 def CalculateHash(file_path): |
| 206 """Calculates and returns the hash of the file at file_path.""" | 209 """Calculates and returns the hash of the file at file_path.""" |
| 207 sha1 = hashlib.sha1() | 210 sha1 = hashlib.sha1() |
| 208 with open(file_path, 'rb') as f: | 211 with open(file_path, 'rb') as f: |
| 209 while True: | 212 while True: |
| 210 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. | 213 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. |
| 211 chunk = f.read(1024*1024) | 214 chunk = f.read(1024*1024) |
| 212 if not chunk: | 215 if not chunk: |
| 213 break | 216 break |
| 214 sha1.update(chunk) | 217 sha1.update(chunk) |
| 215 return sha1.hexdigest() | 218 return sha1.hexdigest() |
| 216 | 219 |
| 217 | 220 |
| 218 def ReadHash(hash_path): | 221 def ReadHash(hash_path): |
| 219 with open(hash_path, 'rb') as f: | 222 with open(hash_path, 'rb') as f: |
| 220 return f.read(1024).rstrip() | 223 return f.read(1024).rstrip() |
| OLD | NEW |