OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" | 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" |
6 | 6 |
7 import contextlib | 7 import contextlib |
8 import collections | 8 import collections |
9 import cStringIO | 9 import cStringIO |
10 import hashlib | 10 import hashlib |
11 import logging | 11 import logging |
12 import os | 12 import os |
13 import subprocess | 13 import subprocess |
14 import sys | 14 import sys |
15 import tarfile | 15 import tarfile |
16 import urllib2 | 16 import urllib2 |
17 | 17 |
| 18 from telemetry import decorators |
18 from telemetry.core import util | 19 from telemetry.core import util |
19 from telemetry.util import path | 20 from telemetry.util import path |
20 | 21 |
21 | 22 |
22 PUBLIC_BUCKET = 'chromium-telemetry' | 23 PUBLIC_BUCKET = 'chromium-telemetry' |
23 PARTNER_BUCKET = 'chrome-partner-telemetry' | 24 PARTNER_BUCKET = 'chrome-partner-telemetry' |
24 INTERNAL_BUCKET = 'chrome-telemetry' | 25 INTERNAL_BUCKET = 'chrome-telemetry' |
25 | 26 |
26 | 27 |
27 # Uses ordered dict to make sure that bucket's key-value items are ordered from | 28 # Uses ordered dict to make sure that bucket's key-value items are ordered from |
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
213 command_and_args += ['-a', 'public-read'] | 214 command_and_args += ['-a', 'public-read'] |
214 extra_info = ' (publicly readable)' | 215 extra_info = ' (publicly readable)' |
215 command_and_args += [local_path, url] | 216 command_and_args += [local_path, url] |
216 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) | 217 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) |
217 _RunCommand(command_and_args) | 218 _RunCommand(command_and_args) |
218 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( | 219 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( |
219 bucket, remote_path) | 220 bucket, remote_path) |
220 | 221 |
221 | 222 |
222 def GetIfChanged(file_path, bucket): | 223 def GetIfChanged(file_path, bucket): |
223 """Gets the file at file_path if it has a hash file that doesn't match. | 224 """Gets the file at file_path if it has a hash file that doesn't match or |
224 | 225 if there is no local copy of file_path, but there is a hash file for it. |
225 If the file is not in Cloud Storage, log a warning instead of raising an | |
226 exception. We assume that the user just hasn't uploaded the file yet. | |
227 | 226 |
228 Returns: | 227 Returns: |
229 True if the binary was changed. | 228 True if the binary was changed. |
230 Raises: | 229 Raises: |
231 CredentialsError if the user has no configured credentials. | 230 CredentialsError if the user has no configured credentials. |
232 PermissionError if the user does not have permission to access the bucket. | 231 PermissionError if the user does not have permission to access the bucket. |
233 NotFoundError if the file is not in the given bucket in cloud_storage. | 232 NotFoundError if the file is not in the given bucket in cloud_storage. |
234 """ | 233 """ |
235 hash_path = file_path + '.sha1' | 234 hash_path = file_path + '.sha1' |
236 if not os.path.exists(hash_path): | 235 if not os.path.exists(hash_path): |
237 logging.warning('Hash file not found: %s' % hash_path) | 236 logging.warning('Hash file not found: %s' % hash_path) |
238 return False | 237 return False |
239 | 238 |
240 expected_hash = ReadHash(hash_path) | 239 expected_hash = ReadHash(hash_path) |
241 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: | 240 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: |
242 return False | 241 return False |
243 | 242 |
244 Get(bucket, expected_hash, file_path) | 243 Get(bucket, expected_hash, file_path) |
245 return True | 244 return True |
246 | 245 |
| 246 # TODO(aiolos): remove @decorators.Cache for http://crbug.com/459787 |
| 247 @decorators.Cache |
| 248 def GetFilesInDirectoryIfChanged(directory, bucket): |
| 249 """ Scan the directory for .sha1 files, and download them from the given |
| 250 bucket in cloud storage if the local and remote hash don't match or |
| 251 there is no local copy. |
| 252 """ |
| 253 if not os.path.isdir(directory): |
| 254 raise ValueError('Must provide a valid directory.') |
| 255 # Don't allow the root directory to be a serving_dir. |
| 256 if directory == os.path.abspath(os.sep): |
| 257 raise ValueError('Trying to serve root directory from HTTP server.') |
| 258 for dirpath, _, filenames in os.walk(directory): |
| 259 for filename in filenames: |
| 260 path_name, extension = os.path.splitext( |
| 261 os.path.join(dirpath, filename)) |
| 262 if extension != '.sha1': |
| 263 continue |
| 264 GetIfChanged(path_name, bucket) |
247 | 265 |
248 def CalculateHash(file_path): | 266 def CalculateHash(file_path): |
249 """Calculates and returns the hash of the file at file_path.""" | 267 """Calculates and returns the hash of the file at file_path.""" |
250 sha1 = hashlib.sha1() | 268 sha1 = hashlib.sha1() |
251 with open(file_path, 'rb') as f: | 269 with open(file_path, 'rb') as f: |
252 while True: | 270 while True: |
253 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. | 271 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. |
254 chunk = f.read(1024*1024) | 272 chunk = f.read(1024*1024) |
255 if not chunk: | 273 if not chunk: |
256 break | 274 break |
257 sha1.update(chunk) | 275 sha1.update(chunk) |
258 return sha1.hexdigest() | 276 return sha1.hexdigest() |
259 | 277 |
260 | 278 |
261 def ReadHash(hash_path): | 279 def ReadHash(hash_path): |
262 with open(hash_path, 'rb') as f: | 280 with open(hash_path, 'rb') as f: |
263 return f.read(1024).rstrip() | 281 return f.read(1024).rstrip() |
OLD | NEW |