OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" | 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" |
6 | 6 |
7 import contextlib | 7 import contextlib |
8 import collections | 8 import collections |
9 import cStringIO | 9 import cStringIO |
10 import hashlib | 10 import hashlib |
11 import logging | 11 import logging |
12 import os | 12 import os |
13 import subprocess | 13 import subprocess |
14 import sys | 14 import sys |
15 import tarfile | 15 import tarfile |
16 import urllib2 | 16 import urllib2 |
17 | 17 |
18 from telemetry import decorators | |
19 from telemetry.core import util | 18 from telemetry.core import util |
20 from telemetry.util import path | 19 from telemetry.util import path |
21 | 20 |
22 | 21 |
23 PUBLIC_BUCKET = 'chromium-telemetry' | 22 PUBLIC_BUCKET = 'chromium-telemetry' |
24 PARTNER_BUCKET = 'chrome-partner-telemetry' | 23 PARTNER_BUCKET = 'chrome-partner-telemetry' |
25 INTERNAL_BUCKET = 'chrome-telemetry' | 24 INTERNAL_BUCKET = 'chrome-telemetry' |
26 | 25 |
27 | 26 |
28 # Uses ordered dict to make sure that bucket's key-value items are ordered from | 27 # Uses ordered dict to make sure that bucket's key-value items are ordered from |
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
214 command_and_args += ['-a', 'public-read'] | 213 command_and_args += ['-a', 'public-read'] |
215 extra_info = ' (publicly readable)' | 214 extra_info = ' (publicly readable)' |
216 command_and_args += [local_path, url] | 215 command_and_args += [local_path, url] |
217 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) | 216 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) |
218 _RunCommand(command_and_args) | 217 _RunCommand(command_and_args) |
219 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( | 218 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( |
220 bucket, remote_path) | 219 bucket, remote_path) |
221 | 220 |
222 | 221 |
223 def GetIfChanged(file_path, bucket): | 222 def GetIfChanged(file_path, bucket): |
224 """Gets the file at file_path if it has a hash file that doesn't match or | 223 """Gets the file at file_path if it has a hash file that doesn't match. |
225 if there is no local copy of file_path, but there is a hash file for it. | 224 |
| 225 If the file is not in Cloud Storage, log a warning instead of raising an |
| 226 exception. We assume that the user just hasn't uploaded the file yet. |
226 | 227 |
227 Returns: | 228 Returns: |
228 True if the binary was changed. | 229 True if the binary was changed. |
229 Raises: | 230 Raises: |
230 CredentialsError if the user has no configured credentials. | 231 CredentialsError if the user has no configured credentials. |
231 PermissionError if the user does not have permission to access the bucket. | 232 PermissionError if the user does not have permission to access the bucket. |
232 NotFoundError if the file is not in the given bucket in cloud_storage. | 233 NotFoundError if the file is not in the given bucket in cloud_storage. |
233 """ | 234 """ |
234 hash_path = file_path + '.sha1' | 235 hash_path = file_path + '.sha1' |
235 if not os.path.exists(hash_path): | 236 if not os.path.exists(hash_path): |
236 logging.warning('Hash file not found: %s' % hash_path) | 237 logging.warning('Hash file not found: %s' % hash_path) |
237 return False | 238 return False |
238 | 239 |
239 expected_hash = ReadHash(hash_path) | 240 expected_hash = ReadHash(hash_path) |
240 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: | 241 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: |
241 return False | 242 return False |
242 | 243 |
243 Get(bucket, expected_hash, file_path) | 244 Get(bucket, expected_hash, file_path) |
244 return True | 245 return True |
245 | 246 |
246 # TODO(aiolos): remove @decorators.Cache for http://crbug.com/459787 | |
247 @decorators.Cache | |
248 def GetFilesInDirectoryIfChanged(directory, bucket): | |
249 """ Scan the directory for .sha1 files, and download them from the given | |
250 bucket in cloud storage if the local and remote hash don't match or | |
251 there is no local copy. | |
252 """ | |
253 if not os.path.isdir(directory): | |
254 raise ValueError('Must provide a valid directory.') | |
255 # Don't allow the root directory to be a serving_dir. | |
256 if directory == os.path.abspath(os.sep): | |
257 raise ValueError('Trying to serve root directory from HTTP server.') | |
258 for dirpath, _, filenames in os.walk(directory): | |
259 for filename in filenames: | |
260 path_name, extension = os.path.splitext( | |
261 os.path.join(dirpath, filename)) | |
262 if extension != '.sha1': | |
263 continue | |
264 GetIfChanged(path_name, bucket) | |
265 | 247 |
266 def CalculateHash(file_path): | 248 def CalculateHash(file_path): |
267 """Calculates and returns the hash of the file at file_path.""" | 249 """Calculates and returns the hash of the file at file_path.""" |
268 sha1 = hashlib.sha1() | 250 sha1 = hashlib.sha1() |
269 with open(file_path, 'rb') as f: | 251 with open(file_path, 'rb') as f: |
270 while True: | 252 while True: |
271 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. | 253 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. |
272 chunk = f.read(1024*1024) | 254 chunk = f.read(1024*1024) |
273 if not chunk: | 255 if not chunk: |
274 break | 256 break |
275 sha1.update(chunk) | 257 sha1.update(chunk) |
276 return sha1.hexdigest() | 258 return sha1.hexdigest() |
277 | 259 |
278 | 260 |
279 def ReadHash(hash_path): | 261 def ReadHash(hash_path): |
280 with open(hash_path, 'rb') as f: | 262 with open(hash_path, 'rb') as f: |
281 return f.read(1024).rstrip() | 263 return f.read(1024).rstrip() |
OLD | NEW |