OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 # pylint: disable=W0212 | 5 # pylint: disable=W0212 |
6 | 6 |
7 """Unit tests for download_from_google_storage.py.""" | 7 """Unit tests for download_from_google_storage.py.""" |
8 | 8 |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import shutil | 12 import shutil |
13 import sys | 13 import sys |
14 import tarfile | |
14 import tempfile | 15 import tempfile |
15 import threading | 16 import threading |
16 import unittest | 17 import unittest |
17 | 18 |
18 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | 19 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
19 | 20 |
20 import upload_to_google_storage | 21 import upload_to_google_storage |
21 import download_from_google_storage | 22 import download_from_google_storage |
22 | 23 |
23 # ../third_party/gsutil/gsutil | 24 # ../third_party/gsutil/gsutil |
(...skipping 27 matching lines...) Expand all Loading... | |
51 return 0 | 52 return 0 |
52 | 53 |
53 def check_call(self, *args): | 54 def check_call(self, *args): |
54 with self.lock: | 55 with self.lock: |
55 self.append_history('check_call', args) | 56 self.append_history('check_call', args) |
56 if self.expected: | 57 if self.expected: |
57 return self.expected.pop(0) | 58 return self.expected.pop(0) |
58 else: | 59 else: |
59 return (0, '', '') | 60 return (0, '', '') |
60 | 61 |
62 class ChangedWorkingDirectory(object): | |
63 def __init__(self, working_directory): | |
64 self._old_cwd = '' | |
65 self._working_directory = working_directory | |
66 | |
67 def __enter__(self): | |
68 self._old_cwd = os.getcwd() | |
69 print "Enter directory = ", self._working_directory | |
70 os.chdir(self._working_directory) | |
71 | |
72 def __exit__(self, *_): | |
73 print "Enter directory = ", self._old_cwd | |
74 os.chdir(self._old_cwd) | |
61 | 75 |
62 class GstoolsUnitTests(unittest.TestCase): | 76 class GstoolsUnitTests(unittest.TestCase): |
63 def setUp(self): | 77 def setUp(self): |
64 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') | 78 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') |
65 self.base_path = os.path.join(self.temp_dir, 'test_files') | 79 self.base_path = os.path.join(self.temp_dir, 'test_files') |
66 shutil.copytree(os.path.join(TEST_DIR, 'gstools'), self.base_path) | 80 shutil.copytree(os.path.join(TEST_DIR, 'gstools'), self.base_path) |
67 | 81 |
68 def cleanUp(self): | 82 def cleanUp(self): |
69 shutil.rmtree(self.temp_dir) | 83 shutil.rmtree(self.temp_dir) |
70 | 84 |
85 def test_validate_tar_file(self): | |
86 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') | |
87 with ChangedWorkingDirectory(self.base_path): | |
88 # Sanity ok check. | |
89 tar_dir = 'ok_dir' | |
90 os.makedirs(tar_dir) | |
hinoka
2015/06/24 18:54:38
use absolute paths
os.path.join(self.base_path, t
ricow1
2015/06/25 06:24:05
We already changed into self.base_path, but done
| |
91 tar = 'good.tar.gz' | |
92 lorem_ipsum_copy = os.path.join(tar_dir, 'lorem_ipsum.txt') | |
93 shutil.copyfile(lorem_ipsum, lorem_ipsum_copy) | |
94 with tarfile.open(tar, 'w:gz') as tar: | |
95 tar.add(lorem_ipsum_copy) | |
96 self.assertTrue( | |
97 download_from_google_storage._validate_tar_file(tar, tar_dir)) | |
98 | |
99 # Test no links. | |
100 tar_dir_link = 'for_tar_link' | |
101 os.makedirs(tar_dir_link) | |
102 link = os.path.join(tar_dir_link, 'link') | |
103 os.symlink(lorem_ipsum, link) | |
104 tar_with_links = 'with_links.tar.gz' | |
105 with tarfile.open(tar_with_links, 'w:gz') as tar: | |
106 tar.add(link) | |
107 self.assertFalse( | |
108 download_from_google_storage._validate_tar_file(tar, tar_dir_link)) | |
109 | |
110 # Test not outside. | |
111 tar_dir_outside = 'outside_tar' | |
112 os.makedirs(tar_dir_outside) | |
113 tar_with_outside = 'with_outside.tar.gz' | |
114 with tarfile.open(tar_with_outside, 'w:gz') as tar: | |
115 tar.add(lorem_ipsum) | |
116 self.assertFalse( | |
117 download_from_google_storage._validate_tar_file(tar, | |
118 tar_dir_outside)) | |
119 # Test no .. | |
120 tar_with_dotdot = 'with_dotdot.tar.gz' | |
121 dotdot_file = os.path.join(tar_dir, '..', tar_dir, 'lorem_ipsum.txt') | |
122 with tarfile.open(tar_with_dotdot, 'w:gz') as tar: | |
123 tar.add(dotdot_file) | |
124 self.assertFalse( | |
125 download_from_google_storage._validate_tar_file(tar, | |
126 tar_dir)) | |
127 | |
71 def test_gsutil(self): | 128 def test_gsutil(self): |
72 gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None) | 129 gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None) |
73 self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH) | 130 self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH) |
74 code, _, err = gsutil.check_call() | 131 code, _, err = gsutil.check_call() |
75 self.assertEqual(code, 0) | 132 self.assertEqual(code, 0) |
76 self.assertEqual(err, '') | 133 self.assertEqual(err, '') |
77 | 134 |
78 def test_get_sha1(self): | 135 def test_get_sha1(self): |
79 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') | 136 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') |
80 self.assertEqual( | 137 self.assertEqual( |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
112 | 169 |
113 | 170 |
114 class DownloadTests(unittest.TestCase): | 171 class DownloadTests(unittest.TestCase): |
115 def setUp(self): | 172 def setUp(self): |
116 self.gsutil = GsutilMock(GSUTIL_DEFAULT_PATH, None) | 173 self.gsutil = GsutilMock(GSUTIL_DEFAULT_PATH, None) |
117 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') | 174 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') |
118 self.checkout_test_files = os.path.join( | 175 self.checkout_test_files = os.path.join( |
119 TEST_DIR, 'gstools', 'download_test_data') | 176 TEST_DIR, 'gstools', 'download_test_data') |
120 self.base_path = os.path.join( | 177 self.base_path = os.path.join( |
121 self.temp_dir, 'download_test_data') | 178 self.temp_dir, 'download_test_data') |
179 print "copying %s to %s" % (self.checkout_test_files, self.base_path) | |
122 shutil.copytree(self.checkout_test_files, self.base_path) | 180 shutil.copytree(self.checkout_test_files, self.base_path) |
123 self.base_url = 'gs://sometesturl' | 181 self.base_url = 'gs://sometesturl' |
124 self.parser = optparse.OptionParser() | 182 self.parser = optparse.OptionParser() |
125 self.queue = Queue.Queue() | 183 self.queue = Queue.Queue() |
126 self.ret_codes = Queue.Queue() | 184 self.ret_codes = Queue.Queue() |
127 self.lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') | 185 self.lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') |
128 self.lorem_ipsum_sha1 = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 186 self.lorem_ipsum_sha1 = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
129 self.maxDiff = None | 187 self.maxDiff = None |
130 | 188 |
131 def cleanUp(self): | 189 def cleanUp(self): |
(...skipping 25 matching lines...) Expand all Loading... | |
157 | 215 |
158 def test_download_worker_single_file(self): | 216 def test_download_worker_single_file(self): |
159 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 217 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
160 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 218 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
161 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 219 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
162 self.queue.put((sha1_hash, output_filename)) | 220 self.queue.put((sha1_hash, output_filename)) |
163 self.queue.put((None, None)) | 221 self.queue.put((None, None)) |
164 stdout_queue = Queue.Queue() | 222 stdout_queue = Queue.Queue() |
165 download_from_google_storage._downloader_worker_thread( | 223 download_from_google_storage._downloader_worker_thread( |
166 0, self.queue, False, self.base_url, self.gsutil, | 224 0, self.queue, False, self.base_url, self.gsutil, |
167 stdout_queue, self.ret_codes, True) | 225 stdout_queue, self.ret_codes, True, False) |
168 expected_calls = [ | 226 expected_calls = [ |
169 ('check_call', | 227 ('check_call', |
170 ('ls', input_filename)), | 228 ('ls', input_filename)), |
171 ('check_call', | 229 ('check_call', |
172 ('cp', input_filename, output_filename))] | 230 ('cp', input_filename, output_filename))] |
173 if sys.platform != 'win32': | 231 if sys.platform != 'win32': |
174 expected_calls.append( | 232 expected_calls.append( |
175 ('check_call', | 233 ('check_call', |
176 ('stat', | 234 ('stat', |
177 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 235 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
178 expected_output = [ | 236 expected_output = [ |
179 '0> Downloading %s...' % output_filename] | 237 '0> Downloading %s...' % output_filename] |
180 expected_ret_codes = [] | 238 expected_ret_codes = [] |
181 self.assertEqual(list(stdout_queue.queue), expected_output) | 239 self.assertEqual(list(stdout_queue.queue), expected_output) |
182 self.assertEqual(self.gsutil.history, expected_calls) | 240 self.assertEqual(self.gsutil.history, expected_calls) |
183 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 241 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
184 | 242 |
185 def test_download_worker_skips_file(self): | 243 def test_download_worker_skips_file(self): |
186 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' | 244 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' |
187 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') | 245 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') |
188 self.queue.put((sha1_hash, output_filename)) | 246 self.queue.put((sha1_hash, output_filename)) |
189 self.queue.put((None, None)) | 247 self.queue.put((None, None)) |
190 stdout_queue = Queue.Queue() | 248 stdout_queue = Queue.Queue() |
191 download_from_google_storage._downloader_worker_thread( | 249 download_from_google_storage._downloader_worker_thread( |
192 0, self.queue, False, self.base_url, self.gsutil, | 250 0, self.queue, False, self.base_url, self.gsutil, |
193 stdout_queue, self.ret_codes, True) | 251 stdout_queue, self.ret_codes, True, False) |
194 expected_output = [ | 252 expected_output = [ |
195 '0> File %s exists and SHA1 matches. Skipping.' % output_filename | 253 '0> File %s exists and SHA1 matches. Skipping.' % output_filename |
196 ] | 254 ] |
197 self.assertEqual(list(stdout_queue.queue), expected_output) | 255 self.assertEqual(list(stdout_queue.queue), expected_output) |
198 self.assertEqual(self.gsutil.history, []) | 256 self.assertEqual(self.gsutil.history, []) |
199 | 257 |
258 def test_download_extract_archive(self): | |
259 # By design we make this not match | |
260 sha1_hash = '61223e1ad3d86901a57629fee38313db5ec106ff' | |
261 input_filename = '%s/%s' % (self.base_url, sha1_hash) | |
262 output_filename = os.path.join(self.base_path, 'tarfolder.tar.gz') | |
hinoka
2015/06/24 18:54:38
can you just construct the tarfile here instead of
ricow1
2015/06/25 06:24:05
Done.
| |
263 output_dirname = os.path.join(self.base_path, 'tarfolder') | |
264 extracted_filename = os.path.join(output_dirname, 'subfolder_text.txt') | |
265 self.queue.put((sha1_hash, output_filename)) | |
266 self.queue.put((None, None)) | |
267 stdout_queue = Queue.Queue() | |
268 download_from_google_storage._downloader_worker_thread( | |
269 0, self.queue, False, self.base_url, self.gsutil, | |
270 stdout_queue, self.ret_codes, True, True, delete=False) | |
271 expected_calls = [ | |
272 ('check_call', | |
273 ('ls', input_filename)), | |
274 ('check_call', | |
275 ('cp', input_filename, output_filename))] | |
276 if sys.platform != 'win32': | |
277 expected_calls.append( | |
278 ('check_call', | |
279 ('stat', | |
280 'gs://sometesturl/61223e1ad3d86901a57629fee38313db5ec106ff'))) | |
281 expected_output = [ | |
282 '0> Downloading %s...' % output_filename] | |
283 expected_output.extend([ | |
284 '0> Extracting %s...' % output_dirname]) | |
285 expected_ret_codes = [] | |
286 self.assertTrue(os.path.exists(output_dirname)) | |
287 self.assertTrue(os.path.exists(extracted_filename)) | |
288 self.assertEqual(list(stdout_queue.queue), expected_output) | |
289 self.assertEqual(self.gsutil.history, expected_calls) | |
290 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | |
291 | |
200 def test_download_worker_skips_not_found_file(self): | 292 def test_download_worker_skips_not_found_file(self): |
201 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 293 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
202 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 294 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
203 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 295 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
204 self.queue.put((sha1_hash, output_filename)) | 296 self.queue.put((sha1_hash, output_filename)) |
205 self.queue.put((None, None)) | 297 self.queue.put((None, None)) |
206 stdout_queue = Queue.Queue() | 298 stdout_queue = Queue.Queue() |
207 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. | 299 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. |
208 download_from_google_storage._downloader_worker_thread( | 300 download_from_google_storage._downloader_worker_thread( |
209 0, self.queue, False, self.base_url, self.gsutil, | 301 0, self.queue, False, self.base_url, self.gsutil, |
210 stdout_queue, self.ret_codes, True) | 302 stdout_queue, self.ret_codes, True, False) |
211 expected_output = [ | 303 expected_output = [ |
212 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( | 304 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( |
213 input_filename, output_filename), | 305 input_filename, output_filename), |
214 ] | 306 ] |
215 expected_calls = [ | 307 expected_calls = [ |
216 ('check_call', | 308 ('check_call', |
217 ('ls', input_filename)) | 309 ('ls', input_filename)) |
218 ] | 310 ] |
219 expected_ret_codes = [ | 311 expected_ret_codes = [ |
220 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( | 312 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( |
(...skipping 14 matching lines...) Expand all Loading... | |
235 base_url=self.base_url, | 327 base_url=self.base_url, |
236 gsutil=self.gsutil, | 328 gsutil=self.gsutil, |
237 num_threads=1, | 329 num_threads=1, |
238 directory=False, | 330 directory=False, |
239 recursive=False, | 331 recursive=False, |
240 force=True, | 332 force=True, |
241 output=output_filename, | 333 output=output_filename, |
242 ignore_errors=False, | 334 ignore_errors=False, |
243 sha1_file=False, | 335 sha1_file=False, |
244 verbose=True, | 336 verbose=True, |
245 auto_platform=False) | 337 auto_platform=False, |
338 extract=False) | |
246 expected_calls = [ | 339 expected_calls = [ |
247 ('check_call', | 340 ('check_call', |
248 ('ls', input_filename)), | 341 ('ls', input_filename)), |
249 ('check_call', | 342 ('check_call', |
250 ('cp', input_filename, output_filename)) | 343 ('cp', input_filename, output_filename)) |
251 ] | 344 ] |
252 if sys.platform != 'win32': | 345 if sys.platform != 'win32': |
253 expected_calls.append( | 346 expected_calls.append( |
254 ('check_call', | 347 ('check_call', |
255 ('stat', | 348 ('stat', |
(...skipping 10 matching lines...) Expand all Loading... | |
266 base_url=self.base_url, | 359 base_url=self.base_url, |
267 gsutil=self.gsutil, | 360 gsutil=self.gsutil, |
268 num_threads=1, | 361 num_threads=1, |
269 directory=True, | 362 directory=True, |
270 recursive=False, | 363 recursive=False, |
271 force=False, | 364 force=False, |
272 output=None, | 365 output=None, |
273 ignore_errors=False, | 366 ignore_errors=False, |
274 sha1_file=False, | 367 sha1_file=False, |
275 verbose=True, | 368 verbose=True, |
276 auto_platform=False) | 369 auto_platform=False, |
370 extract=False) | |
277 expected_calls = [ | 371 expected_calls = [ |
278 ('check_call', | 372 ('check_call', |
279 ('ls', input_filename)), | 373 ('ls', input_filename)), |
280 ('check_call', | 374 ('check_call', |
281 ('cp', input_filename, output_filename))] | 375 ('cp', input_filename, output_filename))] |
282 if sys.platform != 'win32': | 376 if sys.platform != 'win32': |
283 expected_calls.append( | 377 expected_calls.append( |
284 ('check_call', | 378 ('check_call', |
285 ('stat', | 379 ('stat', |
286 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 380 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
287 self.assertEqual(self.gsutil.history, expected_calls) | 381 self.assertEqual(self.gsutil.history, expected_calls) |
288 self.assertEqual(code, 0) | 382 self.assertEqual(code, 0) |
289 | 383 |
290 | 384 |
291 if __name__ == '__main__': | 385 if __name__ == '__main__': |
292 unittest.main() | 386 unittest.main() |
OLD | NEW |