Roll version of gsutil to 4.15.

third_party/gsutil/gslib/tests/util.py

 # -*- coding: utf-8 -*-
 # Copyright 2013 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
 from __future__ import absolute_import
 
 from contextlib import contextmanager
 import functools
 import os
 import pkgutil
 import posixpath
 import re
 import tempfile
 import unittest
 import urlparse
 
 import boto
+import crcmod
 import gslib.tests as gslib_tests
+from gslib.util import UsingCrcmodExtension
 
 if not hasattr(unittest.TestCase, 'assertIsNone'):
   # external dependency unittest2 required for Python <= 2.6
   import unittest2 as unittest  # pylint: disable=g-import-not-at-top
 
 # Flags for running different types of tests.
 RUN_INTEGRATION_TESTS = True
 RUN_UNIT_TESTS = True
 RUN_S3_TESTS = False
 
@@ skipping 179 matching lines @@
       if name == TEST_BOTO_REMOVE_SECTION:
         sections_to_remove.append(section)
       else:
         boto.config.remove_option(section, name)
     else:
       boto.config.set(section, name, value)
   for section in sections_to_remove:
     boto.config.remove_section(section)
 
 
-def PerformsFileToObjectUpload(func):
-  """Decorator indicating that a test uploads from a local file to an object.
+def SequentialAndParallelTransfer(func):
+  """Decorator for tests that perform file to object transfers, or vice versa.
 
   This forces the test to run once normally, and again with special boto
   config settings that will ensure that the test follows the parallel composite
-  upload code path.
+  upload and/or sliced object download code paths.
 
   Args:
     func: Function to wrap.
 
   Returns:
     Wrapped function.
   """
   @functools.wraps(func)
   def Wrapper(*args, **kwargs):
     # Run the test normally once.
     func(*args, **kwargs)
 
-    # Try again, forcing parallel composite uploads.
-    with SetBotoConfigForTest([
-        ('GSUtil', 'parallel_composite_upload_threshold', '1'),
-        ('GSUtil', 'check_hashes', 'always')]):
-      func(*args, **kwargs)
+    if not RUN_S3_TESTS and UsingCrcmodExtension(crcmod):
+      # Try again, forcing parallel upload and sliced download.
+      with SetBotoConfigForTest([
+          ('GSUtil', 'parallel_composite_upload_threshold', '1'),
+          ('GSUtil', 'sliced_object_download_threshold', '1'),
+          ('GSUtil', 'sliced_object_download_max_components', '3'),
+          ('GSUtil', 'check_hashes', 'always')]):
+        func(*args, **kwargs)
 
   return Wrapper
 
 
 @contextmanager
 def SetBotoConfigForTest(boto_config_list):
   """Sets the input list of boto configs for the duration of a 'with' clause.
 
   Args:
     boto_config_list: list of tuples of:
@@ skipping 98 matching lines @@
     pass
 
   if new_working_directory:
     os.chdir(new_working_directory)
 
   try:
     yield
   finally:
     if new_working_directory and prev_working_directory:
       os.chdir(prev_working_directory)