[Telemetry] Update cloud_storage to use telemetry/third_party/gsutilz/gsutil

tools/telemetry/catapult_base/cloud_storage.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Wrappers for gsutil, for basic interaction with Google Cloud Storage."""
 
 import collections
-import contextlib
-import cStringIO
 import hashlib
 import logging
 import os
 import subprocess
 import sys
-import tarfile
-import urllib2
 
 from telemetry.core import util
 from telemetry import decorators
 from telemetry.internal.util import path
 
 
 PUBLIC_BUCKET = 'chromium-telemetry'
 PARTNER_BUCKET = 'chrome-partner-telemetry'
 INTERNAL_BUCKET = 'chrome-telemetry'
 
 
 # Uses ordered dict to make sure that bucket's key-value items are ordered from
 # the most open to the most restrictive.
 BUCKET_ALIASES = collections.OrderedDict((
     ('public', PUBLIC_BUCKET),
     ('partner', PARTNER_BUCKET),
     ('internal', INTERNAL_BUCKET),
 ))
 
 
-_GSUTIL_URL = 'http://storage.googleapis.com/pub/gsutil.tar.gz'
-_DOWNLOAD_PATH = os.path.join(path.GetTelemetryDir(), 'third_party', 'gsutil')
+_GSUTIL_PATH = os.path.join(path.GetTelemetryDir(), 'third_party', 'gsutil',
+                            'gsutil')
+
 # TODO(tbarzic): A workaround for http://crbug.com/386416 and
 #     http://crbug.com/359293. See |_RunCommand|.
 _CROS_GSUTIL_HOME_WAR = '/home/chromeos-test/'
 
 
 class CloudStorageError(Exception):
   @staticmethod
-  def _GetConfigInstructions(gsutil_path):
-    if SupportsProdaccess(gsutil_path) and _FindExecutableInPath('prodaccess'):
-      return 'Run prodaccess to authenticate.'
-    else:
-      if util.IsRunningOnCrosDevice():
-        gsutil_path = ('HOME=%s %s' % (_CROS_GSUTIL_HOME_WAR, gsutil_path))
-      return ('To configure your credentials:\n'
-              '  1. Run "%s config" and follow its instructions.\n'
-              '  2. If you have a @google.com account, use that account.\n'
-              '  3. For the project-id, just enter 0.' % gsutil_path)
+  def _GetConfigInstructions():
+    command = _GSUTIL_PATH
+    if util.IsRunningOnCrosDevice():
+      command = 'HOME=%s %s' % (_CROS_GSUTIL_HOME_WAR, _GSUTIL_PATH)
+    return ('To configure your credentials:\n'
+            '  1. Run "%s config" and follow its instructions.\n'
+            '  2. If you have a @google.com account, use that account.\n'
+            '  3. For the project-id, just enter 0.' % command)
 
 
 class PermissionError(CloudStorageError):
-  def __init__(self, gsutil_path):
+  def __init__(self):
     super(PermissionError, self).__init__(
         'Attempted to access a file from Cloud Storage but you don\'t '
-        'have permission. ' + self._GetConfigInstructions(gsutil_path))
+        'have permission. ' + self._GetConfigInstructions())
 
 
 class CredentialsError(CloudStorageError):
-  def __init__(self, gsutil_path):
+  def __init__(self):
     super(CredentialsError, self).__init__(
         'Attempted to access a file from Cloud Storage but you have no '
-        'configured credentials. ' + self._GetConfigInstructions(gsutil_path))
+        'configured credentials. ' + self._GetConfigInstructions())
 
 
 class NotFoundError(CloudStorageError):
   pass
 
 
 class ServerError(CloudStorageError):
   pass
 
 
 # TODO(tonyg/dtu): Can this be replaced with distutils.spawn.find_executable()?
 def _FindExecutableInPath(relative_executable_path, *extra_search_paths):
   search_paths = list(extra_search_paths) + os.environ['PATH'].split(os.pathsep)
   for search_path in search_paths:
     executable_path = os.path.join(search_path, relative_executable_path)
     if path.IsExecutable(executable_path):
       return executable_path
   return None
 
 
-def _DownloadGsutil():
-  logging.info('Downloading gsutil')
-  with contextlib.closing(urllib2.urlopen(_GSUTIL_URL, timeout=60)) as response:
-    with tarfile.open(fileobj=cStringIO.StringIO(response.read())) as tar_file:
-      tar_file.extractall(os.path.dirname(_DOWNLOAD_PATH))
-  logging.info('Downloaded gsutil to %s' % _DOWNLOAD_PATH)
-
-  return os.path.join(_DOWNLOAD_PATH, 'gsutil')
-
-
-def FindGsutil():
-  """Return the gsutil executable path. If we can't find it, download it."""
-  # Look for a depot_tools installation.
-  # FIXME: gsutil in depot_tools is not working correctly. crbug.com/413414
-  #gsutil_path = _FindExecutableInPath(
-  #    os.path.join('third_party', 'gsutil', 'gsutil'), _DOWNLOAD_PATH)
-  #if gsutil_path:
-  #  return gsutil_path
-
-  # Look for a gsutil installation.
-  gsutil_path = _FindExecutableInPath('gsutil', _DOWNLOAD_PATH)
-  if gsutil_path:
-    return gsutil_path
-
-  # Failed to find it. Download it!
-  return _DownloadGsutil()
-
-
-def SupportsProdaccess(gsutil_path):

[nednguyen, 2015/07/29 18:01:19]

I delete this because this code doesn't make any sense to me. Seems it check the
source code of gsutil script to find the word 'prodaccess'? :-/

[aiolos (Not reviewing), 2015/07/29 23:22:27]

I *think* this is fine to remove at this point, but you'll need to update the
unittests at least. (ie, I don't think gsutil authenticates through prodaccess
anymore.) (I think find_dependencies_unittest.py is the only one that covers
this code path.)

[aiolos (Not reviewing), 2015/07/29 23:26:29]

It might make more sense to do this in a separate cl.

[nednguyen, 2015/08/01 04:16:17]

Done.

-  with open(gsutil_path, 'r') as gsutil:
-    return 'prodaccess' in gsutil.read()
-
-
 def _RunCommand(args):
-  gsutil_path = FindGsutil()
-
   # On cros device, as telemetry is running as root, home will be set to /root/,
   # which is not writable. gsutil will attempt to create a download tracker dir
   # in home dir and fail. To avoid this, override HOME dir to something writable
   # when running on cros device.
   #
   # TODO(tbarzic): Figure out a better way to handle gsutil on cros.
   #     http://crbug.com/386416, http://crbug.com/359293.
   gsutil_env = None
   if util.IsRunningOnCrosDevice():
     gsutil_env = os.environ.copy()
     gsutil_env['HOME'] = _CROS_GSUTIL_HOME_WAR
 
   if os.name == 'nt':
     # If Windows, prepend python. Python scripts aren't directly executable.
-    args = [sys.executable, gsutil_path] + args
+    args = [sys.executable, _GSUTIL_PATH] + args
   else:
     # Don't do it on POSIX, in case someone is using a shell script to redirect.
-    args = [gsutil_path] + args
+    args = [_GSUTIL_PATH] + args
 
   gsutil = subprocess.Popen(args, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, env=gsutil_env)
   stdout, stderr = gsutil.communicate()
 
   if gsutil.returncode:
     if stderr.startswith((
         'You are attempting to access protected data with no configured',
         'Failure: No handler was ready to authenticate.')):
-      raise CredentialsError(gsutil_path)
+      raise CredentialsError()
     if ('status=403' in stderr or 'status 403' in stderr or
         '403 Forbidden' in stderr):
-      raise PermissionError(gsutil_path)
+      raise PermissionError()
     if (stderr.startswith('InvalidUriError') or 'No such object' in stderr or
         'No URLs matched' in stderr or 'One or more URLs matched no' in stderr):
       raise NotFoundError(stderr)
     if '500 Internal Server Error' in stderr:
       raise ServerError(stderr)
     raise CloudStorageError(stderr)
 
   return stdout
 
 
@@ skipping 129 matching lines @@
       chunk = f.read(1024*1024)
       if not chunk:
         break
       sha1.update(chunk)
   return sha1.hexdigest()
 
 
 def ReadHash(hash_path):
   with open(hash_path, 'rb') as f:
     return f.read(1024).rstrip()