Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1337)

Side by Side Diff: third_party/gsutil/gslib/tests/test_naming.py

Issue 1380943003: Roll version of gsutil to 4.15. (Closed) Base URL: https://github.com/catapult-project/catapult.git@master
Patch Set: rebase Created 5 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # -*- coding: utf-8 -*- 1 # -*- coding: utf-8 -*-
2 # Copyright 2010 Google Inc. All Rights Reserved. 2 # Copyright 2010 Google Inc. All Rights Reserved.
3 # 3 #
4 # Permission is hereby granted, free of charge, to any person obtaining a 4 # Permission is hereby granted, free of charge, to any person obtaining a
5 # copy of this software and associated documentation files (the 5 # copy of this software and associated documentation files (the
6 # "Software"), to deal in the Software without restriction, including 6 # "Software"), to deal in the Software without restriction, including
7 # without limitation the rights to use, copy, modify, merge, publish, dis- 7 # without limitation the rights to use, copy, modify, merge, publish, dis-
8 # tribute, sublicense, and/or sell copies of the Software, and to permit 8 # tribute, sublicense, and/or sell copies of the Software, and to permit
9 # persons to whom the Software is furnished to do so, subject to the fol- 9 # persons to whom the Software is furnished to do so, subject to the fol-
10 # lowing conditions: 10 # lowing conditions:
(...skipping 21 matching lines...) Expand all
32 from __future__ import absolute_import 32 from __future__ import absolute_import
33 33
34 import gzip 34 import gzip
35 import os 35 import os
36 import StringIO 36 import StringIO
37 37
38 from gslib import copy_helper 38 from gslib import copy_helper
39 from gslib.cloud_api import NotFoundException 39 from gslib.cloud_api import NotFoundException
40 from gslib.cloud_api import ServiceException 40 from gslib.cloud_api import ServiceException
41 from gslib.exception import CommandException 41 from gslib.exception import CommandException
42 from gslib.exception import HashMismatchException
43 from gslib.storage_url import StorageUrlFromString 42 from gslib.storage_url import StorageUrlFromString
44 import gslib.tests.testcase as testcase 43 import gslib.tests.testcase as testcase
45 from gslib.tests.util import ObjectToURI as suri 44 from gslib.tests.util import ObjectToURI as suri
46 from gslib.tests.util import SetBotoConfigForTest 45 from gslib.tests.util import SetBotoConfigForTest
47 from gslib.util import UTF8 46 from gslib.util import UTF8
48 47
49 48
50 def _Overwrite(fp): 49 def _Overwrite(fp):
51 """Overwrite first byte in an open file and flush contents.""" 50 """Overwrite first byte in an open file and flush contents."""
52 fp.seek(0) 51 fp.seek(0)
53 fp.write('x') 52 fp.write('x')
54 fp.flush() 53 fp.flush()
55 54
56 55
57 def _Append(fp): 56 def _Append(fp):
58 """Append a byte at end of an open file and flush contents.""" 57 """Append a byte at end of an open file and flush contents."""
59 fp.seek(0, 2) 58 fp.seek(0, 2)
60 fp.write('x') 59 fp.write('x')
61 fp.flush() 60 fp.flush()
62 61
63 62
64 # TODO: Re-enable PerformsFileToObjectUpload decorator on tests in this file 63 # TODO: Re-enable SequentialAndParallelTransfer decorator on tests in this file
65 # once we refactor to a thread-safe mock storage service implementation. 64 # once we refactor to a thread-safe mock storage service implementation.
66 class GsutilNamingTests(testcase.GsUtilUnitTestCase): 65 class GsutilNamingTests(testcase.GsUtilUnitTestCase):
67 """Unit tests for gsutil naming logic.""" 66 """Unit tests for gsutil naming logic."""
68 67
69 def testGetPathBeforeFinalDir(self): 68 def testGetPathBeforeFinalDir(self):
70 """Tests GetPathBeforeFinalDir() (unit test).""" 69 """Tests GetPathBeforeFinalDir() (unit test)."""
71 self.assertEqual( 70 self.assertEqual(
72 'gs://', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString( 71 'gs://', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString(
73 'gs://bucket/'))) 72 'gs://bucket/')))
74 self.assertEqual( 73 self.assertEqual(
75 'gs://bucket', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString( 74 'gs://bucket', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString(
76 'gs://bucket/dir/'))) 75 'gs://bucket/dir/')))
77 self.assertEqual( 76 self.assertEqual(
78 'gs://bucket', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString( 77 'gs://bucket', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString(
79 'gs://bucket/dir'))) 78 'gs://bucket/dir')))
80 self.assertEqual( 79 self.assertEqual(
81 'gs://bucket/dir', copy_helper.GetPathBeforeFinalDir( 80 'gs://bucket/dir', copy_helper.GetPathBeforeFinalDir(
82 StorageUrlFromString('gs://bucket/dir/obj'))) 81 StorageUrlFromString('gs://bucket/dir/obj')))
83 src_dir = self.CreateTempDir() 82 src_dir = self.CreateTempDir()
84 subdir = os.path.join(src_dir, 'subdir') 83 subdir = os.path.join(src_dir, 'subdir')
85 os.mkdir(subdir) 84 os.mkdir(subdir)
86 self.assertEqual(suri(src_dir), 85 self.assertEqual(suri(src_dir),
87 copy_helper.GetPathBeforeFinalDir( 86 copy_helper.GetPathBeforeFinalDir(
88 StorageUrlFromString(suri(subdir)))) 87 StorageUrlFromString(suri(subdir))))
89 88
90 # @PerformsFileToObjectUpload 89 # @SequentialAndParallelTransfer
91 def testCopyingTopLevelFileToBucket(self): 90 def testCopyingTopLevelFileToBucket(self):
92 """Tests copying one top-level file to a bucket.""" 91 """Tests copying one top-level file to a bucket."""
93 src_file = self.CreateTempFile(file_name='f0') 92 src_file = self.CreateTempFile(file_name='f0')
94 dst_bucket_uri = self.CreateBucket() 93 dst_bucket_uri = self.CreateBucket()
95 self.RunCommand('cp', [src_file, suri(dst_bucket_uri)]) 94 self.RunCommand('cp', [src_file, suri(dst_bucket_uri)])
96 actual = list(self._test_wildcard_iterator( 95 actual = list(self._test_wildcard_iterator(
97 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 96 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
98 self.assertEqual(1, len(actual)) 97 self.assertEqual(1, len(actual))
99 self.assertEqual('f0', actual[0].root_object.name) 98 self.assertEqual('f0', actual[0].root_object.name)
100 99
101 # @PerformsFileToObjectUpload 100 # @SequentialAndParallelTransfer
102 def testCopyingMultipleFilesToBucket(self): 101 def testCopyingMultipleFilesToBucket(self):
103 """Tests copying multiple files to a bucket.""" 102 """Tests copying multiple files to a bucket."""
104 src_file0 = self.CreateTempFile(file_name='f0') 103 src_file0 = self.CreateTempFile(file_name='f0')
105 src_file1 = self.CreateTempFile(file_name='f1') 104 src_file1 = self.CreateTempFile(file_name='f1')
106 dst_bucket_uri = self.CreateBucket() 105 dst_bucket_uri = self.CreateBucket()
107 self.RunCommand('cp', [src_file0, src_file1, suri(dst_bucket_uri)]) 106 self.RunCommand('cp', [src_file0, src_file1, suri(dst_bucket_uri)])
108 actual = set(str(u) for u in self._test_wildcard_iterator( 107 actual = set(str(u) for u in self._test_wildcard_iterator(
109 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 108 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
110 expected = set([ 109 expected = set([
111 suri(dst_bucket_uri, 'f0'), 110 suri(dst_bucket_uri, 'f0'),
112 suri(dst_bucket_uri, 'f1'), 111 suri(dst_bucket_uri, 'f1'),
113 ]) 112 ])
114 self.assertEqual(expected, actual) 113 self.assertEqual(expected, actual)
115 114
116 # @PerformsFileToObjectUpload 115 # @SequentialAndParallelTransfer
117 def testCopyingNestedFileToBucketSubdir(self): 116 def testCopyingNestedFileToBucketSubdir(self):
118 """Tests copying a nested file to a bucket subdir. 117 """Tests copying a nested file to a bucket subdir.
119 118
120 Tests that we correctly translate local FS-specific delimiters ('\' on 119 Tests that we correctly translate local FS-specific delimiters ('\' on
121 Windows) to bucket delimiter (/). 120 Windows) to bucket delimiter (/).
122 """ 121 """
123 tmpdir = self.CreateTempDir() 122 tmpdir = self.CreateTempDir()
124 subdir = os.path.join(tmpdir, 'subdir') 123 subdir = os.path.join(tmpdir, 'subdir')
125 os.mkdir(subdir) 124 os.mkdir(subdir)
126 src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents='') 125 src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents='')
127 dst_bucket_uri = self.CreateBucket() 126 dst_bucket_uri = self.CreateBucket()
128 # Make an object under subdir so next copy will treat subdir as a subdir. 127 # Make an object under subdir so next copy will treat subdir as a subdir.
129 self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir/a')]) 128 self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir/a')])
130 self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir')]) 129 self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir')])
131 actual = set(str(u) for u in self._test_wildcard_iterator( 130 actual = set(str(u) for u in self._test_wildcard_iterator(
132 suri(dst_bucket_uri, '**')).IterObjects()) 131 suri(dst_bucket_uri, '**')).IterObjects())
133 expected = set([ 132 expected = set([
134 suri(dst_bucket_uri, 'subdir', 'a'), 133 suri(dst_bucket_uri, 'subdir', 'a'),
135 suri(dst_bucket_uri, 'subdir', 'obj'), 134 suri(dst_bucket_uri, 'subdir', 'obj'),
136 ]) 135 ])
137 self.assertEqual(expected, actual) 136 self.assertEqual(expected, actual)
138 137
139 # @PerformsFileToObjectUpload 138 # @SequentialAndParallelTransfer
140 def testCopyingAbsolutePathDirToBucket(self): 139 def testCopyingAbsolutePathDirToBucket(self):
141 """Tests recursively copying absolute path directory to a bucket.""" 140 """Tests recursively copying absolute path directory to a bucket."""
142 dst_bucket_uri = self.CreateBucket() 141 dst_bucket_uri = self.CreateBucket()
143 src_dir_root = self.CreateTempDir(test_files=[ 142 src_dir_root = self.CreateTempDir(test_files=[
144 'f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')]) 143 'f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')])
145 self.RunCommand('cp', ['-R', src_dir_root, suri(dst_bucket_uri)]) 144 self.RunCommand('cp', ['-R', src_dir_root, suri(dst_bucket_uri)])
146 actual = set(str(u) for u in self._test_wildcard_iterator( 145 actual = set(str(u) for u in self._test_wildcard_iterator(
147 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 146 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
148 src_tmpdir = os.path.split(src_dir_root)[1] 147 src_tmpdir = os.path.split(src_dir_root)[1]
149 expected = set([ 148 expected = set([
150 suri(dst_bucket_uri, src_tmpdir, 'f0'), 149 suri(dst_bucket_uri, src_tmpdir, 'f0'),
151 suri(dst_bucket_uri, src_tmpdir, 'f1'), 150 suri(dst_bucket_uri, src_tmpdir, 'f1'),
152 suri(dst_bucket_uri, src_tmpdir, 'f2.txt'), 151 suri(dst_bucket_uri, src_tmpdir, 'f2.txt'),
153 suri(dst_bucket_uri, src_tmpdir, 'dir0', 'dir1', 'nested')]) 152 suri(dst_bucket_uri, src_tmpdir, 'dir0', 'dir1', 'nested')])
154 self.assertEqual(expected, actual) 153 self.assertEqual(expected, actual)
155 154
156 # @PerformsFileToObjectUpload 155 # @SequentialAndParallelTransfer
157 def testCopyingRelativePathDirToBucket(self): 156 def testCopyingRelativePathDirToBucket(self):
158 """Tests recursively copying relative directory to a bucket.""" 157 """Tests recursively copying relative directory to a bucket."""
159 dst_bucket_uri = self.CreateBucket() 158 dst_bucket_uri = self.CreateBucket()
160 src_dir = self.CreateTempDir(test_files=[('dir0', 'f1')]) 159 src_dir = self.CreateTempDir(test_files=[('dir0', 'f1')])
161 self.RunCommand('cp', ['-R', 'dir0', suri(dst_bucket_uri)], cwd=src_dir) 160 self.RunCommand('cp', ['-R', 'dir0', suri(dst_bucket_uri)], cwd=src_dir)
162 actual = set(str(u) for u in self._test_wildcard_iterator( 161 actual = set(str(u) for u in self._test_wildcard_iterator(
163 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 162 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
164 expected = set([suri(dst_bucket_uri, 'dir0', 'f1')]) 163 expected = set([suri(dst_bucket_uri, 'dir0', 'f1')])
165 self.assertEqual(expected, actual) 164 self.assertEqual(expected, actual)
166 165
167 # @PerformsFileToObjectUpload 166 # @SequentialAndParallelTransfer
168 def testCopyingRelPathSubDirToBucketSubdirWithDollarFolderObj(self): 167 def testCopyingRelPathSubDirToBucketSubdirWithDollarFolderObj(self):
169 """Tests recursively copying relative sub-directory to bucket subdir. 168 """Tests recursively copying relative sub-directory to bucket subdir.
170 169
171 Subdir is signified by a $folder$ object. 170 Subdir is signified by a $folder$ object.
172 """ 171 """
173 # Create a $folder$ object to simulate a folder created by GCS manager (or 172 # Create a $folder$ object to simulate a folder created by the legacy GCS
174 # various other tools), which gsutil understands to mean there is a folder 173 # console (or various other tools), which gsutil understands to mean there
175 # into which the object is being copied. 174 # is a folder into which the object is being copied.
176 dst_bucket_uri = self.CreateBucket() 175 dst_bucket_uri = self.CreateBucket()
177 self.CreateObject(bucket_uri=dst_bucket_uri, object_name='abc_$folder$', 176 self.CreateObject(bucket_uri=dst_bucket_uri, object_name='abc_$folder$',
178 contents='') 177 contents='')
179 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) 178 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')])
180 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'), 179 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'),
181 suri(dst_bucket_uri, 'abc')], cwd=src_dir) 180 suri(dst_bucket_uri, 'abc')], cwd=src_dir)
182 actual = set(str(u) for u in self._test_wildcard_iterator( 181 actual = set(str(u) for u in self._test_wildcard_iterator(
183 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 182 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
184 expected = set([suri(dst_bucket_uri, 'abc_$folder$'), 183 expected = set([suri(dst_bucket_uri, 'abc_$folder$'),
185 suri(dst_bucket_uri, 'abc', 'dir1', 'f1')]) 184 suri(dst_bucket_uri, 'abc', 'dir1', 'f1')])
186 self.assertEqual(expected, actual) 185 self.assertEqual(expected, actual)
187 186
188 # @PerformsFileToObjectUpload 187 # @SequentialAndParallelTransfer
189 def testCopyingRelativePathSubDirToBucketSubdirSignifiedBySlash(self): 188 def testCopyingRelativePathSubDirToBucketSubdirSignifiedBySlash(self):
190 """Tests recursively copying relative sub-directory to bucket subdir. 189 """Tests recursively copying relative sub-directory to bucket subdir.
191 190
192 Subdir is signified by a / object. 191 Subdir is signified by a / object.
193 """ 192 """
194 dst_bucket_uri = self.CreateBucket() 193 dst_bucket_uri = self.CreateBucket()
195 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) 194 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')])
196 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'), 195 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'),
197 suri(dst_bucket_uri, 'abc') + '/'], cwd=src_dir) 196 suri(dst_bucket_uri, 'abc') + '/'], cwd=src_dir)
198 actual = set(str(u) for u in self._test_wildcard_iterator( 197 actual = set(str(u) for u in self._test_wildcard_iterator(
199 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 198 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
200 expected = set([suri(dst_bucket_uri, 'abc', 'dir1', 'f1')]) 199 expected = set([suri(dst_bucket_uri, 'abc', 'dir1', 'f1')])
201 self.assertEqual(expected, actual) 200 self.assertEqual(expected, actual)
202 201
203 # @PerformsFileToObjectUpload 202 # @SequentialAndParallelTransfer
204 def testCopyingRelativePathSubDirToBucket(self): 203 def testCopyingRelativePathSubDirToBucket(self):
205 """Tests recursively copying relative sub-directory to a bucket.""" 204 """Tests recursively copying relative sub-directory to a bucket."""
206 dst_bucket_uri = self.CreateBucket() 205 dst_bucket_uri = self.CreateBucket()
207 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) 206 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')])
208 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'), 207 self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'),
209 suri(dst_bucket_uri)], cwd=src_dir) 208 suri(dst_bucket_uri)], cwd=src_dir)
210 actual = set(str(u) for u in self._test_wildcard_iterator( 209 actual = set(str(u) for u in self._test_wildcard_iterator(
211 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 210 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
212 expected = set([suri(dst_bucket_uri, 'dir1', 'f1')]) 211 expected = set([suri(dst_bucket_uri, 'dir1', 'f1')])
213 self.assertEqual(expected, actual) 212 self.assertEqual(expected, actual)
214 213
215 # @PerformsFileToObjectUpload 214 # @SequentialAndParallelTransfer
216 def testCopyingDotSlashToBucket(self): 215 def testCopyingDotSlashToBucket(self):
217 """Tests copying ./ to a bucket produces expected naming.""" 216 """Tests copying ./ to a bucket produces expected naming."""
218 # When running a command like gsutil cp -r . gs://dest we expect the dest 217 # When running a command like gsutil cp -r . gs://dest we expect the dest
219 # obj names to be of the form gs://dest/abc, not gs://dest/./abc. 218 # obj names to be of the form gs://dest/abc, not gs://dest/./abc.
220 dst_bucket_uri = self.CreateBucket() 219 dst_bucket_uri = self.CreateBucket()
221 src_dir = self.CreateTempDir(test_files=['foo']) 220 src_dir = self.CreateTempDir(test_files=['foo'])
222 for rel_src_dir in ['.', '.%s' % os.sep]: 221 for rel_src_dir in ['.', '.%s' % os.sep]:
223 self.RunCommand('cp', ['-R', rel_src_dir, suri(dst_bucket_uri)], 222 self.RunCommand('cp', ['-R', rel_src_dir, suri(dst_bucket_uri)],
224 cwd=src_dir) 223 cwd=src_dir)
225 actual = set(str(u) for u in self._test_wildcard_iterator( 224 actual = set(str(u) for u in self._test_wildcard_iterator(
226 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 225 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
227 expected = set([suri(dst_bucket_uri, 'foo')]) 226 expected = set([suri(dst_bucket_uri, 'foo')])
228 self.assertEqual(expected, actual) 227 self.assertEqual(expected, actual)
229 228
230 # @PerformsFileToObjectUpload 229 # @SequentialAndParallelTransfer
231 def testCopyingDirContainingOneFileToBucket(self): 230 def testCopyingDirContainingOneFileToBucket(self):
232 """Tests copying a directory containing 1 file to a bucket. 231 """Tests copying a directory containing 1 file to a bucket.
233 232
234 We test this case to ensure that correct bucket handling isn't dependent 233 We test this case to ensure that correct bucket handling isn't dependent
235 on the copy being treated as a multi-source copy. 234 on the copy being treated as a multi-source copy.
236 """ 235 """
237 dst_bucket_uri = self.CreateBucket() 236 dst_bucket_uri = self.CreateBucket()
238 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')]) 237 src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')])
239 self.RunCommand('cp', ['-R', os.path.join(src_dir, 'dir0', 'dir1'), 238 self.RunCommand('cp', ['-R', os.path.join(src_dir, 'dir0', 'dir1'),
240 suri(dst_bucket_uri)]) 239 suri(dst_bucket_uri)])
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
294 def testCopyingFileToDir(self): 293 def testCopyingFileToDir(self):
295 """Tests copying one file to a directory.""" 294 """Tests copying one file to a directory."""
296 src_file = self.CreateTempFile(file_name='foo') 295 src_file = self.CreateTempFile(file_name='foo')
297 dst_dir = self.CreateTempDir() 296 dst_dir = self.CreateTempDir()
298 self.RunCommand('cp', [src_file, dst_dir]) 297 self.RunCommand('cp', [src_file, dst_dir])
299 actual = list(self._test_wildcard_iterator( 298 actual = list(self._test_wildcard_iterator(
300 '%s%s*' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) 299 '%s%s*' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True))
301 self.assertEqual(1, len(actual)) 300 self.assertEqual(1, len(actual))
302 self.assertEqual(suri(dst_dir, 'foo'), str(actual[0])) 301 self.assertEqual(suri(dst_dir, 'foo'), str(actual[0]))
303 302
304 # @PerformsFileToObjectUpload 303 # @SequentialAndParallelTransfer
305 def testCopyingFileToObjectWithConsecutiveSlashes(self): 304 def testCopyingFileToObjectWithConsecutiveSlashes(self):
306 """Tests copying a file to an object containing consecutive slashes.""" 305 """Tests copying a file to an object containing consecutive slashes."""
307 src_file = self.CreateTempFile(file_name='f0') 306 src_file = self.CreateTempFile(file_name='f0')
308 dst_bucket_uri = self.CreateBucket() 307 dst_bucket_uri = self.CreateBucket()
309 self.RunCommand('cp', [src_file, suri(dst_bucket_uri) + '//obj']) 308 self.RunCommand('cp', [src_file, suri(dst_bucket_uri) + '//obj'])
310 actual = list(self._test_wildcard_iterator( 309 actual = list(self._test_wildcard_iterator(
311 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 310 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
312 self.assertEqual(1, len(actual)) 311 self.assertEqual(1, len(actual))
313 self.assertEqual('/obj', actual[0].root_object.name) 312 self.assertEqual('/obj', actual[0].root_object.name)
314 313
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after
379 cwd=dst_dir) 378 cwd=dst_dir)
380 actual = set() 379 actual = set()
381 for dirname, dirnames, filenames in os.walk(dst_dir): 380 for dirname, dirnames, filenames in os.walk(dst_dir):
382 for subdirname in dirnames: 381 for subdirname in dirnames:
383 actual.add(os.path.join(dirname, subdirname)) 382 actual.add(os.path.join(dirname, subdirname))
384 for filename in filenames: 383 for filename in filenames:
385 actual.add(os.path.join(dirname, filename)) 384 actual.add(os.path.join(dirname, filename))
386 expected = set([os.path.join(dst_dir, 'f1')]) 385 expected = set([os.path.join(dst_dir, 'f1')])
387 self.assertEqual(expected, actual) 386 self.assertEqual(expected, actual)
388 387
389 # @PerformsFileToObjectUpload 388 # @SequentialAndParallelTransfer
390 def testCopyingObjsAndFilesToBucket(self): 389 def testCopyingObjsAndFilesToBucket(self):
391 """Tests copying objects and files to a bucket.""" 390 """Tests copying objects and files to a bucket."""
392 src_bucket_uri = self.CreateBucket(test_objects=['f1']) 391 src_bucket_uri = self.CreateBucket(test_objects=['f1'])
393 src_dir = self.CreateTempDir(test_files=['f2']) 392 src_dir = self.CreateTempDir(test_files=['f2'])
394 dst_bucket_uri = self.CreateBucket() 393 dst_bucket_uri = self.CreateBucket()
395 self.RunCommand('cp', ['-R', suri(src_bucket_uri, '**'), 394 self.RunCommand('cp', ['-R', suri(src_bucket_uri, '**'),
396 '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri)]) 395 '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri)])
397 actual = set(str(u) for u in self._test_wildcard_iterator( 396 actual = set(str(u) for u in self._test_wildcard_iterator(
398 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 397 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
399 expected = set([suri(dst_bucket_uri, 'f1'), suri(dst_bucket_uri, 'f2')]) 398 expected = set([suri(dst_bucket_uri, 'f1'), suri(dst_bucket_uri, 'f2')])
400 self.assertEqual(expected, actual) 399 self.assertEqual(expected, actual)
401 400
402 # @PerformsFileToObjectUpload 401 # @SequentialAndParallelTransfer
403 def testCopyingSubdirRecursiveToNonexistentSubdir(self): 402 def testCopyingSubdirRecursiveToNonexistentSubdir(self):
404 """Tests copying a directory with a single file recursively to a bucket. 403 """Tests copying a directory with a single file recursively to a bucket.
405 404
406 The file should end up in a new bucket subdirectory with the file's 405 The file should end up in a new bucket subdirectory with the file's
407 directory structure starting below the recursive copy point, as in Unix cp. 406 directory structure starting below the recursive copy point, as in Unix cp.
408 407
409 Example: 408 Example:
410 filepath: dir1/dir2/foo 409 filepath: dir1/dir2/foo
411 cp -r dir1 dir3 410 cp -r dir1 dir3
412 Results in dir3/dir2/foo being created. 411 Results in dir3/dir2/foo being created.
(...skipping 219 matching lines...) Expand 10 before | Expand all | Expand 10 after
632 631
633 def testSetDefAclOnObjectFails(self): 632 def testSetDefAclOnObjectFails(self):
634 """Test that the 'defacl set' command fails when run against an object.""" 633 """Test that the 'defacl set' command fails when run against an object."""
635 src_bucket_uri = self.CreateBucket() 634 src_bucket_uri = self.CreateBucket()
636 try: 635 try:
637 self.RunCommand('defacl', ['set', 'private', suri(src_bucket_uri, '*')]) 636 self.RunCommand('defacl', ['set', 'private', suri(src_bucket_uri, '*')])
638 self.fail('Did not get expected CommandException') 637 self.fail('Did not get expected CommandException')
639 except CommandException, e: 638 except CommandException, e:
640 self.assertIn('URL must name a bucket', e.reason) 639 self.assertIn('URL must name a bucket', e.reason)
641 640
642 # @PerformsFileToObjectUpload 641 # @SequentialAndParallelTransfer
643 def testMinusDOptionWorks(self): 642 def testMinusDOptionWorks(self):
644 """Tests using gsutil -D option.""" 643 """Tests using gsutil -D option."""
645 src_file = self.CreateTempFile(file_name='f0') 644 src_file = self.CreateTempFile(file_name='f0')
646 dst_bucket_uri = self.CreateBucket() 645 dst_bucket_uri = self.CreateBucket()
647 self.RunCommand('cp', [src_file, suri(dst_bucket_uri)], debug=3) 646 self.RunCommand('cp', [src_file, suri(dst_bucket_uri)], debug=3)
648 actual = list(self._test_wildcard_iterator( 647 actual = list(self._test_wildcard_iterator(
649 suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) 648 suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True))
650 self.assertEqual(1, len(actual)) 649 self.assertEqual(1, len(actual))
651 self.assertEqual('f0', actual[0].root_object.name) 650 self.assertEqual('f0', actual[0].root_object.name)
652 651
653 def DownloadTestHelper(self, func): 652 # @SequentialAndParallelTransfer
654 """Test resumable download with custom test function.
655
656 The custom function distorts downloaded data. We expect an exception to be
657 raised and the dest file to be removed.
658
659 Args:
660 func: Custom test function used to distort the downloaded data.
661 """
662 object_uri = self.CreateObject(contents='foo')
663 # Need to explicitly tell the key to populate its etag so that hash
664 # validation will be performed.
665 object_uri.get_key().set_etag()
666 dst_dir = self.CreateTempDir()
667 got_expected_exception = False
668 try:
669 self.RunCommand('cp', [suri(object_uri), dst_dir], test_method=func)
670 self.fail('Did not get expected CommandException')
671 except HashMismatchException:
672 self.assertFalse(os.listdir(dst_dir))
673 got_expected_exception = True
674 except Exception, e:
675 self.fail('Unexpected exception raised: %s' % e)
676 if not got_expected_exception:
677 self.fail('Did not get expected CommandException')
678
679 def testDownloadWithObjectSizeChange(self):
680 """Test resumable download on an object that changes size.
681
682 Size change occurs before the downloaded file's checksum is validated.
683 """
684 self.DownloadTestHelper(_Append)
685
686 def testDownloadWithFileContentChange(self):
687 """Tests resumable download on an object that changes content.
688
689 Content change occurs before the downloaded file's checksum is validated.
690 """
691 self.DownloadTestHelper(_Overwrite)
692
693 # @PerformsFileToObjectUpload
694 def testFlatCopyingObjsAndFilesToBucketSubDir(self): 653 def testFlatCopyingObjsAndFilesToBucketSubDir(self):
695 """Tests copying flatly listed objects and files to bucket subdir.""" 654 """Tests copying flatly listed objects and files to bucket subdir."""
696 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'd0/f1', 'd1/d2/f2']) 655 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'd0/f1', 'd1/d2/f2'])
697 src_dir = self.CreateTempDir(test_files=['f3', ('d3', 'f4'), 656 src_dir = self.CreateTempDir(test_files=['f3', ('d3', 'f4'),
698 ('d4', 'd5', 'f5')]) 657 ('d4', 'd5', 'f5')])
699 dst_bucket_uri = self.CreateBucket(test_objects=['dst_subdir0/existing', 658 dst_bucket_uri = self.CreateBucket(test_objects=['dst_subdir0/existing',
700 'dst_subdir1/existing']) 659 'dst_subdir1/existing'])
701 # Test with and without final slash on dest subdir. 660 # Test with and without final slash on dest subdir.
702 for i, final_char in enumerate(('/', '')): 661 for i, final_char in enumerate(('/', '')):
703 self.RunCommand( 662 self.RunCommand(
704 'cp', ['-R', suri(src_bucket_uri, '**'), os.path.join(src_dir, '**'), 663 'cp', ['-R', suri(src_bucket_uri, '**'), os.path.join(src_dir, '**'),
705 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char]) 664 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char])
706 665
707 actual = set(str(u) for u in self._test_wildcard_iterator( 666 actual = set(str(u) for u in self._test_wildcard_iterator(
708 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 667 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
709 expected = set() 668 expected = set()
710 for i in range(2): 669 for i in range(2):
711 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing')) 670 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing'))
712 for j in range(6): 671 for j in range(6):
713 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'f%d' % j)) 672 expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'f%d' % j))
714 self.assertEqual(expected, actual) 673 self.assertEqual(expected, actual)
715 674
716 # @PerformsFileToObjectUpload 675 # @SequentialAndParallelTransfer
717 def testRecursiveCopyObjsAndFilesToExistingBucketSubDir(self): 676 def testRecursiveCopyObjsAndFilesToExistingBucketSubDir(self):
718 """Tests recursive copy of objects and files to existing bucket subdir.""" 677 """Tests recursive copy of objects and files to existing bucket subdir."""
719 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1']) 678 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1'])
720 dst_bucket_uri = self.CreateBucket(test_objects=[ 679 dst_bucket_uri = self.CreateBucket(test_objects=[
721 'dst_subdir0/existing_obj', 'dst_subdir1/existing_obj']) 680 'dst_subdir0/existing_obj', 'dst_subdir1/existing_obj'])
722 src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')]) 681 src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')])
723 # Test with and without final slash on dest subdir. 682 # Test with and without final slash on dest subdir.
724 for i, final_char in enumerate(('/', '')): 683 for i, final_char in enumerate(('/', '')):
725 self.RunCommand( 684 self.RunCommand(
726 'cp', ['-R', suri(src_bucket_uri), src_dir, 685 'cp', ['-R', suri(src_bucket_uri), src_dir,
727 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char]) 686 suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char])
728 actual = set(str(u) for u in self._test_wildcard_iterator( 687 actual = set(str(u) for u in self._test_wildcard_iterator(
729 suri(dst_bucket_uri, 'dst_subdir%d' % i, '**')).IterAll( 688 suri(dst_bucket_uri, 'dst_subdir%d' % i, '**')).IterAll(
730 expand_top_level_buckets=True)) 689 expand_top_level_buckets=True))
731 tmp_dirname = os.path.split(src_dir)[1] 690 tmp_dirname = os.path.split(src_dir)[1]
732 bucketname = src_bucket_uri.bucket_name 691 bucketname = src_bucket_uri.bucket_name
733 expected = set([ 692 expected = set([
734 suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing_obj'), 693 suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing_obj'),
735 suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'f0'), 694 suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'f0'),
736 suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'nested', 'f1'), 695 suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'nested', 'f1'),
737 suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'f2'), 696 suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'f2'),
738 suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'nested', 'f3') 697 suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'nested', 'f3')
739 ]) 698 ])
740 self.assertEqual(expected, actual) 699 self.assertEqual(expected, actual)
741 700
742 # @PerformsFileToObjectUpload 701 # @SequentialAndParallelTransfer
743 def testRecursiveCopyObjsAndFilesToNonExistentBucketSubDir(self): 702 def testRecursiveCopyObjsAndFilesToNonExistentBucketSubDir(self):
744 """Tests recursive copy of objs + files to non-existent bucket subdir.""" 703 """Tests recursive copy of objs + files to non-existent bucket subdir."""
745 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1']) 704 src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1'])
746 src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')]) 705 src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')])
747 dst_bucket_uri = self.CreateBucket() 706 dst_bucket_uri = self.CreateBucket()
748 self.RunCommand('cp', ['-R', src_dir, suri(src_bucket_uri), 707 self.RunCommand('cp', ['-R', src_dir, suri(src_bucket_uri),
749 suri(dst_bucket_uri, 'dst_subdir')]) 708 suri(dst_bucket_uri, 'dst_subdir')])
750 actual = set(str(u) for u in self._test_wildcard_iterator( 709 actual = set(str(u) for u in self._test_wildcard_iterator(
751 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) 710 suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
752 expected = set([suri(dst_bucket_uri, 'dst_subdir', 'f0'), 711 expected = set([suri(dst_bucket_uri, 'dst_subdir', 'f0'),
(...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after
884 self.RunCommand('cp', [ 843 self.RunCommand('cp', [
885 suri(src_bucket_uri, 'obj0'), 844 suri(src_bucket_uri, 'obj0'),
886 suri(dst_bucket_uri, 'dir%d' % i) + final_dst_char]) 845 suri(dst_bucket_uri, 'dir%d' % i) + final_dst_char])
887 actual = set(str(u) for u in self._test_wildcard_iterator( 846 actual = set(str(u) for u in self._test_wildcard_iterator(
888 suri(dst_bucket_uri, 'dir%d' % i, '**')).IterAll( 847 suri(dst_bucket_uri, 'dir%d' % i, '**')).IterAll(
889 expand_top_level_buckets=True)) 848 expand_top_level_buckets=True))
890 expected = set([suri(dst_bucket_uri, 'dir%d' % i, 'obj0'), 849 expected = set([suri(dst_bucket_uri, 'dir%d' % i, 'obj0'),
891 suri(dst_bucket_uri, 'dir%d' % i, 'existing')]) 850 suri(dst_bucket_uri, 'dir%d' % i, 'existing')])
892 self.assertEqual(expected, actual) 851 self.assertEqual(expected, actual)
893 852
894 # @PerformsFileToObjectUpload 853 # @SequentialAndParallelTransfer
895 def testCopyingWildcardedFilesToBucketSubDir(self): 854 def testCopyingWildcardedFilesToBucketSubDir(self):
896 """Tests copying wildcarded files to a bucket subdir.""" 855 """Tests copying wildcarded files to a bucket subdir."""
897 dst_bucket_uri = self.CreateBucket(test_objects=['subdir0/existing', 856 dst_bucket_uri = self.CreateBucket(test_objects=['subdir0/existing',
898 'subdir1/existing']) 857 'subdir1/existing'])
899 src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2']) 858 src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2'])
900 # Test with and without final slash on dest subdir. 859 # Test with and without final slash on dest subdir.
901 for i, final_dst_char in enumerate(('', '/')): 860 for i, final_dst_char in enumerate(('', '/')):
902 self.RunCommand( 861 self.RunCommand(
903 'cp', [os.path.join(src_dir, 'f?'), 862 'cp', [os.path.join(src_dir, 'f?'),
904 suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char]) 863 suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char])
905 actual = set(str(u) for u in self._test_wildcard_iterator( 864 actual = set(str(u) for u in self._test_wildcard_iterator(
906 suri(dst_bucket_uri, 'subdir%d' % i, '**')).IterAll( 865 suri(dst_bucket_uri, 'subdir%d' % i, '**')).IterAll(
907 expand_top_level_buckets=True)) 866 expand_top_level_buckets=True))
908 expected = set([suri(dst_bucket_uri, 'subdir%d' % i, 'existing'), 867 expected = set([suri(dst_bucket_uri, 'subdir%d' % i, 'existing'),
909 suri(dst_bucket_uri, 'subdir%d' % i, 'f0'), 868 suri(dst_bucket_uri, 'subdir%d' % i, 'f0'),
910 suri(dst_bucket_uri, 'subdir%d' % i, 'f1'), 869 suri(dst_bucket_uri, 'subdir%d' % i, 'f1'),
911 suri(dst_bucket_uri, 'subdir%d' % i, 'f2')]) 870 suri(dst_bucket_uri, 'subdir%d' % i, 'f2')])
912 self.assertEqual(expected, actual) 871 self.assertEqual(expected, actual)
913 872
914 # @PerformsFileToObjectUpload 873 # @SequentialAndParallelTransfer
915 def testCopyingOneNestedFileToBucketSubDir(self): 874 def testCopyingOneNestedFileToBucketSubDir(self):
916 """Tests copying one nested file to a bucket subdir.""" 875 """Tests copying one nested file to a bucket subdir."""
917 dst_bucket_uri = self.CreateBucket(test_objects=['d0/placeholder', 876 dst_bucket_uri = self.CreateBucket(test_objects=['d0/placeholder',
918 'd1/placeholder']) 877 'd1/placeholder'])
919 src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')]) 878 src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')])
920 # Test with and without final slash on dest subdir. 879 # Test with and without final slash on dest subdir.
921 for i, final_dst_char in enumerate(('', '/')): 880 for i, final_dst_char in enumerate(('', '/')):
922 self.RunCommand('cp', ['-r', suri(src_dir, 'd3'), 881 self.RunCommand('cp', ['-r', suri(src_dir, 'd3'),
923 suri(dst_bucket_uri, 'd%d' % i) + final_dst_char]) 882 suri(dst_bucket_uri, 'd%d' % i) + final_dst_char])
924 actual = set(str(u) for u in self._test_wildcard_iterator( 883 actual = set(str(u) for u in self._test_wildcard_iterator(
(...skipping 198 matching lines...) Expand 10 before | Expand all | Expand 10 after
1123 1082
1124 def testMakeBucketsCommand(self): 1083 def testMakeBucketsCommand(self):
1125 """Test mb on existing bucket.""" 1084 """Test mb on existing bucket."""
1126 dst_bucket_uri = self.CreateBucket() 1085 dst_bucket_uri = self.CreateBucket()
1127 try: 1086 try:
1128 self.RunCommand('mb', [suri(dst_bucket_uri)]) 1087 self.RunCommand('mb', [suri(dst_bucket_uri)])
1129 self.fail('Did not get expected StorageCreateError') 1088 self.fail('Did not get expected StorageCreateError')
1130 except ServiceException, e: 1089 except ServiceException, e:
1131 self.assertEqual(e.status, 409) 1090 self.assertEqual(e.status, 409)
1132 1091
1133 def testRemoveBucketsCommand(self):
1134 """Test rb on non-existent bucket."""
1135 dst_bucket_uri = self.CreateBucket()
1136 try:
1137 self.RunCommand(
1138 'rb', ['gs://no_exist_%s' % dst_bucket_uri.bucket_name])
1139 self.fail('Did not get expected NotFoundException')
1140 except NotFoundException, e:
1141 self.assertEqual(e.status, 404)
1142
1143 def testRemoveObjsCommand(self): 1092 def testRemoveObjsCommand(self):
1144 """Test rm command on non-existent object.""" 1093 """Test rm command on non-existent object."""
1145 dst_bucket_uri = self.CreateBucket() 1094 dst_bucket_uri = self.CreateBucket()
1146 try: 1095 try:
1147 self.RunCommand('rm', [suri(dst_bucket_uri, 'non_existent')]) 1096 self.RunCommand('rm', [suri(dst_bucket_uri, 'non_existent')])
1148 self.fail('Did not get expected CommandException') 1097 self.fail('Did not get expected CommandException')
1149 except CommandException, e: 1098 except CommandException, e:
1150 self.assertIn('No URLs matched', e.reason) 1099 self.assertIn('No URLs matched', e.reason)
1151 1100
1152 # Now that gsutil ver computes a checksum it adds 1-3 seconds to test run 1101 # Now that gsutil ver computes a checksum it adds 1-3 seconds to test run
1153 # time (for in memory mocked tests that otherwise take ~ 0.1 seconds). Since 1102 # time (for in memory mocked tests that otherwise take ~ 0.1 seconds). Since
1154 # it provides very little test value, we're leaving this test commented out. 1103 # it provides very little test value, we're leaving this test commented out.
1155 # def testVerCommmandRuns(self): 1104 # def testVerCommmandRuns(self):
1156 # """Test that the Ver command basically runs""" 1105 # """Test that the Ver command basically runs"""
1157 # self.RunCommand('ver', []) 1106 # self.RunCommand('ver', [])
OLDNEW
« no previous file with comments | « third_party/gsutil/gslib/tests/test_mv.py ('k') | third_party/gsutil/gslib/tests/test_parallel_cp.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698