Roll version of gsutil to 4.15.

third_party/gsutil/gslib/tests/test_parallel_cp.py

 # -*- coding: utf-8 -*-
 # Copyright 2010 Google Inc. All Rights Reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the
 # "Software"), to deal in the Software without restriction, including
 # without limitation the rights to use, copy, modify, merge, publish, dis-
 # tribute, sublicense, and/or sell copies of the Software, and to permit
 # persons to whom the Software is furnished to do so, subject to the fol-
 # lowing conditions:
@@ skipping 18 matching lines @@
 mock storage service rewrite.
 
 Tests for relative paths are not included as integration_testcase does not
 support modifying the current working directory.
 """
 
 import os
 
 import gslib.tests.testcase as testcase
 from gslib.tests.util import ObjectToURI as suri
-from gslib.tests.util import PerformsFileToObjectUpload
+from gslib.tests.util import SequentialAndParallelTransfer
 from gslib.util import Retry
 
 
 class TestParallelCp(testcase.GsUtilIntegrationTestCase):
   """Unit tests for gsutil naming logic."""
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingTopLevelFileToBucket(self):
     """Tests copying one top-level file to a bucket."""
     src_file = self.CreateTempFile(file_name='f0')
     dst_bucket_uri = self.CreateBucket()
     self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri)])
 
     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1)
     self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0])
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingMultipleFilesToBucket(self):
     """Tests copying multiple files to a bucket."""
     src_file0 = self.CreateTempFile(file_name='f0')
     src_file1 = self.CreateTempFile(file_name='f1')
     dst_bucket_uri = self.CreateBucket()
     self.RunGsUtil(['cp', src_file0, src_file1, suri(dst_bucket_uri)])
 
     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2)
     self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0])
     self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[1])
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingNestedFileToBucketSubdir(self):
     """Tests copying a nested file to a bucket subdir.
 
     Tests that we correctly translate local FS-specific delimiters ('\' on
     Windows) to bucket delimiter (/).
     """
     tmpdir = self.CreateTempDir()
     subdir = os.path.join(tmpdir, 'subdir')
     os.mkdir(subdir)
     src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents='')
     dst_bucket_uri = self.CreateBucket()
     # Make an object under subdir so next copy will treat subdir as a subdir.
     self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir/a')])
     self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir')])
 
     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2)
     self.assertEqual(suri(dst_bucket_uri, 'subdir/a'), lines[0])
     self.assertEqual(suri(dst_bucket_uri, 'subdir/obj'), lines[1])
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingAbsolutePathDirToBucket(self):
     """Tests recursively copying absolute path directory to a bucket."""
     dst_bucket_uri = self.CreateBucket()
     src_dir_root = self.CreateTempDir(test_files=[
         'f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')])
     self.RunGsUtil(['cp', '-R', src_dir_root, suri(dst_bucket_uri)])
     src_tmpdir = os.path.split(src_dir_root)[1]
 
     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4)
     self.assertEqual(suri(dst_bucket_uri, src_tmpdir,
                           'dir0', 'dir1', 'nested'), lines[0])
     self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f0'), lines[1])
     self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f1'), lines[2])
     self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f2.txt'), lines[3])
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingDirContainingOneFileToBucket(self):
     """Tests copying a directory containing 1 file to a bucket.
 
     We test this case to ensure that correct bucket handling isn't dependent
     on the copy being treated as a multi-source copy.
     """
     dst_bucket_uri = self.CreateBucket()
     src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')])
     self.RunGsUtil(['cp', '-R', os.path.join(src_dir, 'dir0', 'dir1'),
                     suri(dst_bucket_uri)])
 
     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1)
     self.assertEqual(suri(dst_bucket_uri, 'dir1', 'foo'), lines[0])
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingFileToObjectWithConsecutiveSlashes(self):
     """Tests copying a file to an object containing consecutive slashes."""
     src_file = self.CreateTempFile(file_name='f0')
     dst_bucket_uri = self.CreateBucket()
     self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri) + '//obj'])
 
     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1)
     self.assertEqual(suri(dst_bucket_uri) + '//obj', lines[0])
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingObjsAndFilesToBucket(self):
     """Tests copying objects and files to a bucket."""
     src_bucket_uri = self.CreateBucket()
     self.CreateObject(src_bucket_uri, object_name='f1', contents='foo')
     src_dir = self.CreateTempDir(test_files=['f2'])
     dst_bucket_uri = self.CreateBucket()
     self.RunGsUtil(['cp', '-R', suri(src_bucket_uri, '**'),
                     '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri)])
 
     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2)
     self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[0])
     self.assertEqual(suri(dst_bucket_uri, 'f2'), lines[1])
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingSubdirRecursiveToNonexistentSubdir(self):
     """Tests copying a directory with a single file recursively to a bucket.
 
     The file should end up in a new bucket subdirectory with the file's
     directory structure starting below the recursive copy point, as in Unix cp.
 
     Example:
       filepath: dir1/dir2/foo
       cp -r dir1 dir3
       Results in dir3/dir2/foo being created.
     """
     src_dir = self.CreateTempDir()
     self.CreateTempFile(tmpdir=src_dir + '/dir1/dir2', file_name='foo')
     dst_bucket_uri = self.CreateBucket()
     self.RunGsUtil(['cp', '-R', src_dir + '/dir1',
                     suri(dst_bucket_uri, 'dir3')])
 
     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1)
     self.assertEqual(suri(dst_bucket_uri, 'dir3/dir2/foo'), lines[0])
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingWildcardedFilesToBucketSubDir(self):
     """Tests copying wildcarded files to a bucket subdir."""
     # Test with and without final slash on dest subdir.
     for final_dst_char in ('', '/'):
       dst_bucket_uri = self.CreateBucket()
       self.CreateObject(dst_bucket_uri, object_name='subdir0/existing',
                         contents='foo')
       self.CreateObject(dst_bucket_uri, object_name='subdir1/existing',
                         contents='foo')
       src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2'])
@@ skipping 11 matching lines @@
                                   return_stdout=True)
           lines = stdout.split('\n')
           self.assertEqual(5, len(lines))
           self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'existing'),
                            lines[0])
           self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f0'), lines[1])
           self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f1'), lines[2])
           self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f2'), lines[3])
         _Check1()
 
-  @PerformsFileToObjectUpload
+  @SequentialAndParallelTransfer
   def testCopyingOneNestedFileToBucketSubDir(self):
     """Tests copying one nested file to a bucket subdir."""
     # Test with and without final slash on dest subdir.
     for final_dst_char in ('', '/'):
 
       dst_bucket_uri = self.CreateBucket()
       self.CreateObject(dst_bucket_uri, object_name='d0/placeholder',
                         contents='foo')
       self.CreateObject(dst_bucket_uri, object_name='d1/placeholder',
                         contents='foo')
 
       for i in range(2):
         src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')])
         self.RunGsUtil(['cp', '-r', suri(src_dir, 'd3'),
                         suri(dst_bucket_uri, 'd%d' % i) + final_dst_char])
 
       lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4)
       self.assertEqual(suri(dst_bucket_uri, 'd0', 'd3', 'd4', 'nested', 'f1'),
                        lines[0])
       self.assertEqual(suri(dst_bucket_uri, 'd0', 'placeholder'), lines[1])
       self.assertEqual(suri(dst_bucket_uri, 'd1', 'd3', 'd4', 'nested', 'f1'),
                        lines[2])
       self.assertEqual(suri(dst_bucket_uri, 'd1', 'placeholder'), lines[3])