Update checked in version of gsutil to version 4.6

gslib/tests/util.py

Index: gslib/tests/util.py
===================================================================
--- gslib/tests/util.py	(revision 33376)
+++ gslib/tests/util.py	(working copy)
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2013 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,31 +13,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import boto
+from __future__ import absolute_import
+
+from contextlib import contextmanager
 import functools
 import os
+import pkgutil
 import posixpath
-import pkgutil
 import re
 import tempfile
+import unittest
 import urlparse
 
-import unittest
-if not hasattr(unittest.TestCase, 'assertIsNone'):
-  # external dependency unittest2 required for Python <= 2.6
-  import unittest2 as unittest
-
+import boto
 from boto.provider import Provider
-from contextlib import contextmanager
 import gslib.tests as gslib_tests
 
+if not hasattr(unittest.TestCase, 'assertIsNone'):
+  # external dependency unittest2 required for Python <= 2.6
+  import unittest2 as unittest  # pylint: disable=g-import-not-at-top
 
 # Flags for running different types of tests.
 RUN_INTEGRATION_TESTS = True
 RUN_UNIT_TESTS = True
+RUN_S3_TESTS = False
 
-# Whether the tests are running verbose or not.
-VERBOSE_OUTPUT = False
+PARALLEL_COMPOSITE_UPLOAD_TEST_CONFIG = '/tmp/.boto.parallel_upload_test_config'
 
 
 def _HasS3Credentials():
@@ -48,9 +50,27 @@
 HAS_S3_CREDS = _HasS3Credentials()
 
 
+def _HasGSHost():
+  return boto.config.get('Credentials', 'gs_host', None) is not None
+
+HAS_GS_HOST = _HasGSHost()
+
+
+def _UsingJSONApi():
+  return boto.config.get('GSUtil', 'prefer_api', 'json').upper() != 'XML'
+
+USING_JSON_API = _UsingJSONApi()
+
+
 def _NormalizeURI(uri):
   """Normalizes the path component of a URI.
 
+  Args:
+    uri: URI to normalize.
+
+  Returns:
+    Normalized URI.
+
   Examples:
     gs://foo//bar -> gs://foo/bar
     gs://foo/./bar -> gs://foo/bar
@@ -79,8 +99,12 @@
     obj: The object to get the URI from. Can be a file object, a subclass of
          boto.storage_uri.StorageURI, or a string. If a string, it is assumed to
          be a local on-disk path.
-    suffixes: Suffixes to append. For example, ObjectToUri(bucketuri, 'foo')
-              would return the URI for a key name 'foo' inside the given bucket.
+    *suffixes: Suffixes to append. For example, ObjectToUri(bucketuri, 'foo')
+               would return the URI for a key name 'foo' inside the given
+               bucket.
+
+  Returns:
+    Storage URI string.
   """
   if isinstance(obj, file):
     return 'file://%s' % os.path.abspath(os.path.join(obj.name, *suffixes))
@@ -95,40 +119,152 @@
     uri = uri[:-1]
   return uri
 
+# The mock storage service comes from the Boto library, but it is not
+# distributed with Boto when installed as a package. To get around this, we
+# copy the file to gslib/tests/mock_storage_service.py when building the gsutil
+# package. Try and import from both places here.
+# pylint: disable=g-import-not-at-top
+try:
+  from gslib.tests import mock_storage_service
+except ImportError:
+  try:
+    from boto.tests.integration.s3 import mock_storage_service
+  except ImportError:
+    try:
+      from tests.integration.s3 import mock_storage_service
+    except ImportError:
+      import mock_storage_service
+
+
+class GSMockConnection(mock_storage_service.MockConnection):
+
+  def __init__(self, *args, **kwargs):
+    kwargs['provider'] = 'gs'
+    super(GSMockConnection, self).__init__(*args, **kwargs)
+
+mock_connection = GSMockConnection()
+
+
+class GSMockBucketStorageUri(mock_storage_service.MockBucketStorageUri):
+
+  def connect(self, access_key_id=None, secret_access_key=None):
+    return mock_connection
+
+  def compose(self, components, headers=None):
+    """Dummy implementation to allow parallel uploads with tests."""
+    return self.new_key()
+
+
+TEST_BOTO_REMOVE_SECTION = 'TestRemoveSection'
+
+
+def _SetBotoConfig(section, name, value, revert_list):
+  """Sets boto configuration temporarily for testing.
+
+  SetBotoConfigForTest and SetBotoConfigFileForTest should be called by tests
+  instead of this function. Those functions will ensure that the configuration
+  is reverted to its original setting using _RevertBotoConfig.
+
+  Args:
+    section: Boto config section to set
+    name: Boto config name to set
+    value: Value to set
+    revert_list: List for tracking configs to revert.
+  """
+  prev_value = boto.config.get(section, name, None)
+  if not boto.config.has_section(section):
+    revert_list.append((section, TEST_BOTO_REMOVE_SECTION, None))
+    boto.config.add_section(section)
+  revert_list.append((section, name, prev_value))
+  if value is None:
+    boto.config.remove_option(section, name)
+  else:
+    boto.config.set(section, name, value)
+
+
+def _RevertBotoConfig(revert_list):
+  """Reverts boto config modifications made by _SetBotoConfig.
+
+  Args:
+    revert_list: List of boto config modifications created by calls to
+                 _SetBotoConfig.
+  """
+  sections_to_remove = []
+  for section, name, value in revert_list:
+    if value is None:
+      if name == TEST_BOTO_REMOVE_SECTION:
+        sections_to_remove.append(section)
+      else:
+        boto.config.remove_option(section, name)
+    else:
+      boto.config.set(section, name, value)
+  for section in sections_to_remove:
+    boto.config.remove_section(section)
+
+
 def PerformsFileToObjectUpload(func):
-  """Decorator used to indicate that a test performs an upload from a local
-     file to an object. This forces the test to run once normally, and again
-     with a special .boto config file that will ensure that the test follows
-     the parallel composite upload code path.
+  """Decorator indicating that a test uploads from a local file to an object.
+
+  This forces the test to run once normally, and again with special boto
+  config settings that will ensure that the test follows the parallel composite
+  upload code path.
+
+  Args:
+    func: Function to wrap.
+
+  Returns:
+    Wrapped function.
   """
   @functools.wraps(func)
-  def wrapper(*args, **kwargs):
-    tmp_fd, tmp_filename = tempfile.mkstemp(prefix='gsutil-temp-cfg')
-    os.close(tmp_fd)
+  def Wrapper(*args, **kwargs):
+    # Run the test normally once.
+    func(*args, **kwargs)
 
-    try:
-      # Run the test normally once.
+    # Try again, forcing parallel composite uploads.
+    with SetBotoConfigForTest([
+        ('GSUtil', 'parallel_composite_upload_threshold', '1'),
+        ('GSUtil', 'check_hashes', 'always')]):
       func(*args, **kwargs)
 
-      # Try again, forcing parallel composite uploads.
-      boto.config.set('GSUtil', 'parallel_composite_upload_threshold', '1')
-      with open(tmp_filename, 'w') as tmp_file:
-        boto.config.write(tmp_file)
+  return Wrapper
 
-      with SetBotoConfigForTest(tmp_filename):
-        func(*args, **kwargs)
-    finally:
+
+@contextmanager
+def SetBotoConfigForTest(boto_config_list):
+  """Sets the input list of boto configs for the duration of a 'with' clause.
+
+  Args:
+    boto_config_list: list of tuples of:
+      (boto config section to set, boto config name to set, value to set)
+
+  Yields:
+    Once after config is set.
+  """
+  revert_configs = []
+  tmp_filename = None
+  try:
+    tmp_fd, tmp_filename = tempfile.mkstemp(prefix='gsutil-temp-cfg')
+    os.close(tmp_fd)
+    for boto_config in boto_config_list:
+      _SetBotoConfig(boto_config[0], boto_config[1], boto_config[2],
+                     revert_configs)
+    with open(tmp_filename, 'w') as tmp_file:
+      boto.config.write(tmp_file)
+
+    with SetBotoConfigFileForTest(tmp_filename):
+      yield
+  finally:
+    _RevertBotoConfig(revert_configs)
+    if tmp_filename:
       try:
         os.remove(tmp_filename)
       except OSError:
         pass
 
-  return wrapper
 
 @contextmanager
-def SetBotoConfigForTest(boto_config_path):
+def SetBotoConfigFileForTest(boto_config_path):
   """Sets a given file as the boto config file for a single test."""
-
   # Setup for entering "with" block.
   try:
     old_boto_config_env_variable = os.environ['BOTO_CONFIG']
@@ -146,11 +282,12 @@
     else:
       os.environ.pop('BOTO_CONFIG', None)
 
+
 def GetTestNames():
   """Returns a list of the names of the test modules in gslib.tests."""
   matcher = re.compile(r'^test_(?P<name>.*)$')
   names = []
-  for importer, modname, ispkg in pkgutil.iter_modules(gslib_tests.__path__):
+  for _, modname, _ in pkgutil.iter_modules(gslib_tests.__path__):
     m = matcher.match(modname)
     if m:
       names.append(m.group('name'))