Refactor serving_dirs.

tools/telemetry/telemetry/util/cloud_storage.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Wrappers for gsutil, for basic interaction with Google Cloud Storage."""
 
 import contextlib
 import cStringIO
 import hashlib
 import logging
 import os
 import subprocess
 import sys
 import tarfile
 import urllib2
 
+from telemetry import decorators
 from telemetry.core import util
 from telemetry.util import path
 
 
 PUBLIC_BUCKET = 'chromium-telemetry'
 PARTNER_BUCKET = 'chrome-partner-telemetry'
 INTERNAL_BUCKET = 'chrome-telemetry'
 
 
 BUCKET_ALIASES = {
@@ skipping 183 matching lines @@
     command_and_args += ['-a', 'public-read']
     extra_info = ' (publicly readable)'
   command_and_args += [local_path, url]
   logging.info('Uploading %s to %s%s' % (local_path, url, extra_info))
   _RunCommand(command_and_args)
   return 'https://console.developers.google.com/m/cloudstorage/b/%s/o/%s' % (
       bucket, remote_path)
 
 
 def GetIfChanged(file_path, bucket):
-  """Gets the file at file_path if it has a hash file that doesn't match.
-
-  If the file is not in Cloud Storage, log a warning instead of raising an
-  exception. We assume that the user just hasn't uploaded the file yet.
+  """Gets the file at file_path if it has a hash file that doesn't match or
+  if there is no local copy of file_path, but there is a hash file for it.
 
   Returns:
     True if the binary was changed.
   Raises:
     CredentialsError if the user has no configured credentials.
     PermissionError if the user does not have permission to access the bucket.
     NotFoundError if the file is not in the given bucket in cloud_storage.
   """
   hash_path = file_path + '.sha1'
   if not os.path.exists(hash_path):
     logging.warning('Hash file not found: %s' % hash_path)
     return False
 
   expected_hash = ReadHash(hash_path)
   if os.path.exists(file_path) and CalculateHash(file_path) == expected_hash:
     return False
 
   Get(bucket, expected_hash, file_path)
   return True
 
+@decorators.Cache

[sullivan, 2015/02/20 17:47:37]

Since this function is moved, it's good to keep the functionality the same as
before.

But we should do a follow-up to remove usage of @decorators.Cache (see
http://crbug.com/459787)

[aiolos (Not reviewing), 2015/02/20 19:06:51]

Acknowledged. I added a TODO for it.

+def GetFilesInDirectoryIfChanged(directory, bucket):
+  """ Scan the directory for .sha1 files, and download them from the given
+  bucket in cloud storage if the local and remote hash don't match or
+  there is no local copy.
+  """
+  if os.path.splitdrive(directory)[1] == '/':
+    raise ValueError('Trying to serve root directory from HTTP server.')
+  for dirpath, _, filenames in os.walk(directory):
+    for filename in filenames:
+      path_name, extension = os.path.splitext(
+          os.path.join(dirpath, filename))
+      if extension != '.sha1':
+        continue
+      GetIfChanged(path_name, bucket)
 
 def CalculateHash(file_path):
   """Calculates and returns the hash of the file at file_path."""
   sha1 = hashlib.sha1()
   with open(file_path, 'rb') as f:
     while True:
       # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
       chunk = f.read(1024*1024)
       if not chunk:
         break
       sha1.update(chunk)
   return sha1.hexdigest()
 
 
 def ReadHash(hash_path):
   with open(hash_path, 'rb') as f:
     return f.read(1024).rstrip()