Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(33)

Side by Side Diff: tools/telemetry/telemetry/util/cloud_storage.py

Issue 1005903004: Revert of Refactor serving_dirs. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage.""" 5 """Wrappers for gsutil, for basic interaction with Google Cloud Storage."""
6 6
7 import contextlib 7 import contextlib
8 import collections 8 import collections
9 import cStringIO 9 import cStringIO
10 import hashlib 10 import hashlib
11 import logging 11 import logging
12 import os 12 import os
13 import subprocess 13 import subprocess
14 import sys 14 import sys
15 import tarfile 15 import tarfile
16 import urllib2 16 import urllib2
17 17
18 from telemetry import decorators
19 from telemetry.core import util 18 from telemetry.core import util
20 from telemetry.util import path 19 from telemetry.util import path
21 20
22 21
23 PUBLIC_BUCKET = 'chromium-telemetry' 22 PUBLIC_BUCKET = 'chromium-telemetry'
24 PARTNER_BUCKET = 'chrome-partner-telemetry' 23 PARTNER_BUCKET = 'chrome-partner-telemetry'
25 INTERNAL_BUCKET = 'chrome-telemetry' 24 INTERNAL_BUCKET = 'chrome-telemetry'
26 25
27 26
28 # Uses ordered dict to make sure that bucket's key-value items are ordered from 27 # Uses ordered dict to make sure that bucket's key-value items are ordered from
(...skipping 185 matching lines...) Expand 10 before | Expand all | Expand 10 after
214 command_and_args += ['-a', 'public-read'] 213 command_and_args += ['-a', 'public-read']
215 extra_info = ' (publicly readable)' 214 extra_info = ' (publicly readable)'
216 command_and_args += [local_path, url] 215 command_and_args += [local_path, url]
217 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info)) 216 logging.info('Uploading %s to %s%s' % (local_path, url, extra_info))
218 _RunCommand(command_and_args) 217 _RunCommand(command_and_args)
219 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % ( 218 return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % (
220 bucket, remote_path) 219 bucket, remote_path)
221 220
222 221
223 def GetIfChanged(file_path, bucket): 222 def GetIfChanged(file_path, bucket):
224 """Gets the file at file_path if it has a hash file that doesn't match or 223 """Gets the file at file_path if it has a hash file that doesn't match.
225 if there is no local copy of file_path, but there is a hash file for it. 224
225 If the file is not in Cloud Storage, log a warning instead of raising an
226 exception. We assume that the user just hasn't uploaded the file yet.
226 227
227 Returns: 228 Returns:
228 True if the binary was changed. 229 True if the binary was changed.
229 Raises: 230 Raises:
230 CredentialsError if the user has no configured credentials. 231 CredentialsError if the user has no configured credentials.
231 PermissionError if the user does not have permission to access the bucket. 232 PermissionError if the user does not have permission to access the bucket.
232 NotFoundError if the file is not in the given bucket in cloud_storage. 233 NotFoundError if the file is not in the given bucket in cloud_storage.
233 """ 234 """
234 hash_path = file_path + '.sha1' 235 hash_path = file_path + '.sha1'
235 if not os.path.exists(hash_path): 236 if not os.path.exists(hash_path):
236 logging.warning('Hash file not found: %s' % hash_path) 237 logging.warning('Hash file not found: %s' % hash_path)
237 return False 238 return False
238 239
239 expected_hash = ReadHash(hash_path) 240 expected_hash = ReadHash(hash_path)
240 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash: 241 if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash:
241 return False 242 return False
242 243
243 Get(bucket, expected_hash, file_path) 244 Get(bucket, expected_hash, file_path)
244 return True 245 return True
245 246
246 # TODO(aiolos): remove @decorators.Cache for http://crbug.com/459787
247 @decorators.Cache
248 def GetFilesInDirectoryIfChanged(directory, bucket):
249 """ Scan the directory for .sha1 files, and download them from the given
250 bucket in cloud storage if the local and remote hash don't match or
251 there is no local copy.
252 """
253 if not os.path.isdir(directory):
254 raise ValueError('Must provide a valid directory.')
255 # Don't allow the root directory to be a serving_dir.
256 if directory == os.path.abspath(os.sep):
257 raise ValueError('Trying to serve root directory from HTTP server.')
258 for dirpath, _, filenames in os.walk(directory):
259 for filename in filenames:
260 path_name, extension = os.path.splitext(
261 os.path.join(dirpath, filename))
262 if extension != '.sha1':
263 continue
264 GetIfChanged(path_name, bucket)
265 247
266 def CalculateHash(file_path): 248 def CalculateHash(file_path):
267 """Calculates and returns the hash of the file at file_path.""" 249 """Calculates and returns the hash of the file at file_path."""
268 sha1 = hashlib.sha1() 250 sha1 = hashlib.sha1()
269 with open(file_path, 'rb') as f: 251 with open(file_path, 'rb') as f:
270 while True: 252 while True:
271 # Read in 1mb chunks, so it doesn't all have to be loaded into memory. 253 # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
272 chunk = f.read(1024*1024) 254 chunk = f.read(1024*1024)
273 if not chunk: 255 if not chunk:
274 break 256 break
275 sha1.update(chunk) 257 sha1.update(chunk)
276 return sha1.hexdigest() 258 return sha1.hexdigest()
277 259
278 260
279 def ReadHash(hash_path): 261 def ReadHash(hash_path):
280 with open(hash_path, 'rb') as f: 262 with open(hash_path, 'rb') as f:
281 return f.read(1024).rstrip() 263 return f.read(1024).rstrip()
OLDNEW
« no previous file with comments | « tools/telemetry/telemetry/user_story/user_story_set.py ('k') | tools/telemetry/telemetry/util/cloud_storage_unittest.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698