| OLD | NEW |
|---|
| (Empty) |
| 1 # -*- coding: utf-8 -*- | |
| 2 # Copyright 2010 Google Inc. All Rights Reserved. | |
| 3 # | |
| 4 # Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 # copy of this software and associated documentation files (the | |
| 6 # "Software"), to deal in the Software without restriction, including | |
| 7 # without limitation the rights to use, copy, modify, merge, publish, dis- | |
| 8 # tribute, sublicense, and/or sell copies of the Software, and to permit | |
| 9 # persons to whom the Software is furnished to do so, subject to the fol- | |
| 10 # lowing conditions: | |
| 11 # | |
| 12 # The above copyright notice and this permission notice shall be included | |
| 13 # in all copies or substantial portions of the Software. | |
| 14 # | |
| 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS | |
| 16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- | |
| 17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT | |
| 18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, | |
| 19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| 20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
| 21 # IN THE SOFTWARE. | |
| 22 """Tests for parallel uploads ported from gsutil naming tests. | |
| 23 | |
| 24 Currently, the mock storage service is not thread-safe and therefore not | |
| 25 suitable for multiprocess/multithreaded testing. Since parallel composite | |
| 26 uploads necessarily create at least one worker thread outside of main, | |
| 27 these tests are present in this file as temporary (slower) integration tests | |
| 28 to provide validation for parallel composite uploads until a thread-safe | |
| 29 mock storage service rewrite. | |
| 30 | |
| 31 Tests for relative paths are not included as integration_testcase does not | |
| 32 support modifying the current working directory. | |
| 33 """ | |
| 34 | |
| 35 import os | |
| 36 | |
| 37 import gslib.tests.testcase as testcase | |
| 38 from gslib.tests.util import ObjectToURI as suri | |
| 39 from gslib.tests.util import PerformsFileToObjectUpload | |
| 40 from gslib.util import Retry | |
| 41 | |
| 42 | |
| 43 class TestParallelCp(testcase.GsUtilIntegrationTestCase): | |
| 44 """Unit tests for gsutil naming logic.""" | |
| 45 | |
| 46 @PerformsFileToObjectUpload | |
| 47 def testCopyingTopLevelFileToBucket(self): | |
| 48 """Tests copying one top-level file to a bucket.""" | |
| 49 src_file = self.CreateTempFile(file_name='f0') | |
| 50 dst_bucket_uri = self.CreateBucket() | |
| 51 self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri)]) | |
| 52 | |
| 53 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) | |
| 54 self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0]) | |
| 55 | |
| 56 @PerformsFileToObjectUpload | |
| 57 def testCopyingMultipleFilesToBucket(self): | |
| 58 """Tests copying multiple files to a bucket.""" | |
| 59 src_file0 = self.CreateTempFile(file_name='f0') | |
| 60 src_file1 = self.CreateTempFile(file_name='f1') | |
| 61 dst_bucket_uri = self.CreateBucket() | |
| 62 self.RunGsUtil(['cp', src_file0, src_file1, suri(dst_bucket_uri)]) | |
| 63 | |
| 64 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) | |
| 65 self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0]) | |
| 66 self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[1]) | |
| 67 | |
| 68 @PerformsFileToObjectUpload | |
| 69 def testCopyingNestedFileToBucketSubdir(self): | |
| 70 """Tests copying a nested file to a bucket subdir. | |
| 71 | |
| 72 Tests that we correctly translate local FS-specific delimiters ('\' on | |
| 73 Windows) to bucket delimiter (/). | |
| 74 """ | |
| 75 tmpdir = self.CreateTempDir() | |
| 76 subdir = os.path.join(tmpdir, 'subdir') | |
| 77 os.mkdir(subdir) | |
| 78 src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents='') | |
| 79 dst_bucket_uri = self.CreateBucket() | |
| 80 # Make an object under subdir so next copy will treat subdir as a subdir. | |
| 81 self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir/a')]) | |
| 82 self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir')]) | |
| 83 | |
| 84 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) | |
| 85 self.assertEqual(suri(dst_bucket_uri, 'subdir/a'), lines[0]) | |
| 86 self.assertEqual(suri(dst_bucket_uri, 'subdir/obj'), lines[1]) | |
| 87 | |
| 88 @PerformsFileToObjectUpload | |
| 89 def testCopyingAbsolutePathDirToBucket(self): | |
| 90 """Tests recursively copying absolute path directory to a bucket.""" | |
| 91 dst_bucket_uri = self.CreateBucket() | |
| 92 src_dir_root = self.CreateTempDir(test_files=[ | |
| 93 'f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')]) | |
| 94 self.RunGsUtil(['cp', '-R', src_dir_root, suri(dst_bucket_uri)]) | |
| 95 src_tmpdir = os.path.split(src_dir_root)[1] | |
| 96 | |
| 97 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4) | |
| 98 self.assertEqual(suri(dst_bucket_uri, src_tmpdir, | |
| 99 'dir0', 'dir1', 'nested'), lines[0]) | |
| 100 self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f0'), lines[1]) | |
| 101 self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f1'), lines[2]) | |
| 102 self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f2.txt'), lines[3]) | |
| 103 | |
| 104 @PerformsFileToObjectUpload | |
| 105 def testCopyingDirContainingOneFileToBucket(self): | |
| 106 """Tests copying a directory containing 1 file to a bucket. | |
| 107 | |
| 108 We test this case to ensure that correct bucket handling isn't dependent | |
| 109 on the copy being treated as a multi-source copy. | |
| 110 """ | |
| 111 dst_bucket_uri = self.CreateBucket() | |
| 112 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')]) | |
| 113 self.RunGsUtil(['cp', '-R', os.path.join(src_dir, 'dir0', 'dir1'), | |
| 114 suri(dst_bucket_uri)]) | |
| 115 | |
| 116 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) | |
| 117 self.assertEqual(suri(dst_bucket_uri, 'dir1', 'foo'), lines[0]) | |
| 118 | |
| 119 @PerformsFileToObjectUpload | |
| 120 def testCopyingFileToObjectWithConsecutiveSlashes(self): | |
| 121 """Tests copying a file to an object containing consecutive slashes.""" | |
| 122 src_file = self.CreateTempFile(file_name='f0') | |
| 123 dst_bucket_uri = self.CreateBucket() | |
| 124 self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri) + '//obj']) | |
| 125 | |
| 126 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) | |
| 127 self.assertEqual(suri(dst_bucket_uri) + '//obj', lines[0]) | |
| 128 | |
| 129 @PerformsFileToObjectUpload | |
| 130 def testCopyingObjsAndFilesToBucket(self): | |
| 131 """Tests copying objects and files to a bucket.""" | |
| 132 src_bucket_uri = self.CreateBucket() | |
| 133 self.CreateObject(src_bucket_uri, object_name='f1', contents='foo') | |
| 134 src_dir = self.CreateTempDir(test_files=['f2']) | |
| 135 dst_bucket_uri = self.CreateBucket() | |
| 136 self.RunGsUtil(['cp', '-R', suri(src_bucket_uri, '**'), | |
| 137 '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri)]) | |
| 138 | |
| 139 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) | |
| 140 self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[0]) | |
| 141 self.assertEqual(suri(dst_bucket_uri, 'f2'), lines[1]) | |
| 142 | |
| 143 @PerformsFileToObjectUpload | |
| 144 def testCopyingSubdirRecursiveToNonexistentSubdir(self): | |
| 145 """Tests copying a directory with a single file recursively to a bucket. | |
| 146 | |
| 147 The file should end up in a new bucket subdirectory with the file's | |
| 148 directory structure starting below the recursive copy point, as in Unix cp. | |
| 149 | |
| 150 Example: | |
| 151 filepath: dir1/dir2/foo | |
| 152 cp -r dir1 dir3 | |
| 153 Results in dir3/dir2/foo being created. | |
| 154 """ | |
| 155 src_dir = self.CreateTempDir() | |
| 156 self.CreateTempFile(tmpdir=src_dir + '/dir1/dir2', file_name='foo') | |
| 157 dst_bucket_uri = self.CreateBucket() | |
| 158 self.RunGsUtil(['cp', '-R', src_dir + '/dir1', | |
| 159 suri(dst_bucket_uri, 'dir3')]) | |
| 160 | |
| 161 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) | |
| 162 self.assertEqual(suri(dst_bucket_uri, 'dir3/dir2/foo'), lines[0]) | |
| 163 | |
| 164 @PerformsFileToObjectUpload | |
| 165 def testCopyingWildcardedFilesToBucketSubDir(self): | |
| 166 """Tests copying wildcarded files to a bucket subdir.""" | |
| 167 # Test with and without final slash on dest subdir. | |
| 168 for final_dst_char in ('', '/'): | |
| 169 dst_bucket_uri = self.CreateBucket() | |
| 170 self.CreateObject(dst_bucket_uri, object_name='subdir0/existing', | |
| 171 contents='foo') | |
| 172 self.CreateObject(dst_bucket_uri, object_name='subdir1/existing', | |
| 173 contents='foo') | |
| 174 src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2']) | |
| 175 | |
| 176 for i in range(2): | |
| 177 self.RunGsUtil( | |
| 178 ['cp', os.path.join(src_dir, 'f?'), | |
| 179 suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char]) | |
| 180 | |
| 181 @Retry(AssertionError, tries=3, timeout_secs=1) | |
| 182 def _Check1(): | |
| 183 """Validate files were copied to the correct destinations.""" | |
| 184 stdout = self.RunGsUtil(['ls', suri(dst_bucket_uri, 'subdir%d' % i, | |
| 185 '**')], | |
| 186 return_stdout=True) | |
| 187 lines = stdout.split('\n') | |
| 188 self.assertEqual(5, len(lines)) | |
| 189 self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'existing'), | |
| 190 lines[0]) | |
| 191 self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f0'), lines[1]) | |
| 192 self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f1'), lines[2]) | |
| 193 self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f2'), lines[3]) | |
| 194 _Check1() | |
| 195 | |
| 196 @PerformsFileToObjectUpload | |
| 197 def testCopyingOneNestedFileToBucketSubDir(self): | |
| 198 """Tests copying one nested file to a bucket subdir.""" | |
| 199 # Test with and without final slash on dest subdir. | |
| 200 for final_dst_char in ('', '/'): | |
| 201 | |
| 202 dst_bucket_uri = self.CreateBucket() | |
| 203 self.CreateObject(dst_bucket_uri, object_name='d0/placeholder', | |
| 204 contents='foo') | |
| 205 self.CreateObject(dst_bucket_uri, object_name='d1/placeholder', | |
| 206 contents='foo') | |
| 207 | |
| 208 for i in range(2): | |
| 209 src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')]) | |
| 210 self.RunGsUtil(['cp', '-r', suri(src_dir, 'd3'), | |
| 211 suri(dst_bucket_uri, 'd%d' % i) + final_dst_char]) | |
| 212 | |
| 213 lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4) | |
| 214 self.assertEqual(suri(dst_bucket_uri, 'd0', 'd3', 'd4', 'nested', 'f1'), | |
| 215 lines[0]) | |
| 216 self.assertEqual(suri(dst_bucket_uri, 'd0', 'placeholder'), lines[1]) | |
| 217 self.assertEqual(suri(dst_bucket_uri, 'd1', 'd3', 'd4', 'nested', 'f1'), | |
| 218 lines[2]) | |
| 219 self.assertEqual(suri(dst_bucket_uri, 'd1', 'placeholder'), lines[3]) | |
| 220 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1493973002:
Remove telemetry/third_party/gsutilz (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@gsutil_changes