OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 # pylint: disable=W0212 | 5 # pylint: disable=W0212 |
6 | 6 |
7 """Unit tests for download_from_google_storage.py.""" | 7 """Unit tests for download_from_google_storage.py.""" |
8 | 8 |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import shutil | 12 import shutil |
13 import sys | 13 import sys |
| 14 import tarfile |
14 import tempfile | 15 import tempfile |
15 import threading | 16 import threading |
16 import unittest | 17 import unittest |
17 | 18 |
18 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | 19 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
19 | 20 |
20 import upload_to_google_storage | 21 import upload_to_google_storage |
21 import download_from_google_storage | 22 import download_from_google_storage |
22 | 23 |
23 # ../third_party/gsutil/gsutil | 24 # ../third_party/gsutil/gsutil |
(...skipping 28 matching lines...) Expand all Loading... |
52 | 53 |
53 def check_call(self, *args): | 54 def check_call(self, *args): |
54 with self.lock: | 55 with self.lock: |
55 self.append_history('check_call', args) | 56 self.append_history('check_call', args) |
56 if self.expected: | 57 if self.expected: |
57 return self.expected.pop(0) | 58 return self.expected.pop(0) |
58 else: | 59 else: |
59 return (0, '', '') | 60 return (0, '', '') |
60 | 61 |
61 | 62 |
| 63 class ChangedWorkingDirectory(object): |
| 64 def __init__(self, working_directory): |
| 65 self._old_cwd = '' |
| 66 self._working_directory = working_directory |
| 67 |
| 68 def __enter__(self): |
| 69 self._old_cwd = os.getcwd() |
| 70 print "Enter directory = ", self._working_directory |
| 71 os.chdir(self._working_directory) |
| 72 |
| 73 def __exit__(self, *_): |
| 74 print "Enter directory = ", self._old_cwd |
| 75 os.chdir(self._old_cwd) |
| 76 |
| 77 |
62 class GstoolsUnitTests(unittest.TestCase): | 78 class GstoolsUnitTests(unittest.TestCase): |
63 def setUp(self): | 79 def setUp(self): |
64 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') | 80 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') |
65 self.base_path = os.path.join(self.temp_dir, 'test_files') | 81 self.base_path = os.path.join(self.temp_dir, 'test_files') |
66 shutil.copytree(os.path.join(TEST_DIR, 'gstools'), self.base_path) | 82 shutil.copytree(os.path.join(TEST_DIR, 'gstools'), self.base_path) |
67 | 83 |
68 def cleanUp(self): | 84 def cleanUp(self): |
69 shutil.rmtree(self.temp_dir) | 85 shutil.rmtree(self.temp_dir) |
70 | 86 |
| 87 def test_validate_tar_file(self): |
| 88 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') |
| 89 with ChangedWorkingDirectory(self.base_path): |
| 90 # Sanity ok check. |
| 91 tar_dir = 'ok_dir' |
| 92 os.makedirs(os.path.join(self.base_path, tar_dir)) |
| 93 tar = 'good.tar.gz' |
| 94 lorem_ipsum_copy = os.path.join(tar_dir, 'lorem_ipsum.txt') |
| 95 shutil.copyfile(lorem_ipsum, lorem_ipsum_copy) |
| 96 with tarfile.open(tar, 'w:gz') as tar: |
| 97 tar.add(lorem_ipsum_copy) |
| 98 self.assertTrue( |
| 99 download_from_google_storage._validate_tar_file(tar, tar_dir)) |
| 100 |
| 101 # Test no links. |
| 102 tar_dir_link = 'for_tar_link' |
| 103 os.makedirs(tar_dir_link) |
| 104 link = os.path.join(tar_dir_link, 'link') |
| 105 os.symlink(lorem_ipsum, link) |
| 106 tar_with_links = 'with_links.tar.gz' |
| 107 with tarfile.open(tar_with_links, 'w:gz') as tar: |
| 108 tar.add(link) |
| 109 self.assertFalse( |
| 110 download_from_google_storage._validate_tar_file(tar, tar_dir_link)) |
| 111 |
| 112 # Test not outside. |
| 113 tar_dir_outside = 'outside_tar' |
| 114 os.makedirs(tar_dir_outside) |
| 115 tar_with_outside = 'with_outside.tar.gz' |
| 116 with tarfile.open(tar_with_outside, 'w:gz') as tar: |
| 117 tar.add(lorem_ipsum) |
| 118 self.assertFalse( |
| 119 download_from_google_storage._validate_tar_file(tar, |
| 120 tar_dir_outside)) |
| 121 # Test no .. |
| 122 tar_with_dotdot = 'with_dotdot.tar.gz' |
| 123 dotdot_file = os.path.join(tar_dir, '..', tar_dir, 'lorem_ipsum.txt') |
| 124 with tarfile.open(tar_with_dotdot, 'w:gz') as tar: |
| 125 tar.add(dotdot_file) |
| 126 self.assertFalse( |
| 127 download_from_google_storage._validate_tar_file(tar, |
| 128 tar_dir)) |
| 129 |
71 def test_gsutil(self): | 130 def test_gsutil(self): |
72 gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None) | 131 gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None) |
73 self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH) | 132 self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH) |
74 code, _, err = gsutil.check_call() | 133 code, _, err = gsutil.check_call() |
75 self.assertEqual(code, 0) | 134 self.assertEqual(code, 0) |
76 self.assertEqual(err, '') | 135 self.assertEqual(err, '') |
77 | 136 |
78 def test_get_sha1(self): | 137 def test_get_sha1(self): |
79 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') | 138 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') |
80 self.assertEqual( | 139 self.assertEqual( |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
157 | 216 |
158 def test_download_worker_single_file(self): | 217 def test_download_worker_single_file(self): |
159 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 218 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
160 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 219 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
161 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 220 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
162 self.queue.put((sha1_hash, output_filename)) | 221 self.queue.put((sha1_hash, output_filename)) |
163 self.queue.put((None, None)) | 222 self.queue.put((None, None)) |
164 stdout_queue = Queue.Queue() | 223 stdout_queue = Queue.Queue() |
165 download_from_google_storage._downloader_worker_thread( | 224 download_from_google_storage._downloader_worker_thread( |
166 0, self.queue, False, self.base_url, self.gsutil, | 225 0, self.queue, False, self.base_url, self.gsutil, |
167 stdout_queue, self.ret_codes, True) | 226 stdout_queue, self.ret_codes, True, False) |
168 expected_calls = [ | 227 expected_calls = [ |
169 ('check_call', | 228 ('check_call', |
170 ('ls', input_filename)), | 229 ('ls', input_filename)), |
171 ('check_call', | 230 ('check_call', |
172 ('cp', input_filename, output_filename))] | 231 ('cp', input_filename, output_filename))] |
173 if sys.platform != 'win32': | 232 if sys.platform != 'win32': |
174 expected_calls.append( | 233 expected_calls.append( |
175 ('check_call', | 234 ('check_call', |
176 ('stat', | 235 ('stat', |
177 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 236 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
178 expected_output = [ | 237 expected_output = [ |
179 '0> Downloading %s...' % output_filename] | 238 '0> Downloading %s...' % output_filename] |
180 expected_ret_codes = [] | 239 expected_ret_codes = [] |
181 self.assertEqual(list(stdout_queue.queue), expected_output) | 240 self.assertEqual(list(stdout_queue.queue), expected_output) |
182 self.assertEqual(self.gsutil.history, expected_calls) | 241 self.assertEqual(self.gsutil.history, expected_calls) |
183 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 242 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
184 | 243 |
185 def test_download_worker_skips_file(self): | 244 def test_download_worker_skips_file(self): |
186 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' | 245 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' |
187 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') | 246 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') |
188 self.queue.put((sha1_hash, output_filename)) | 247 self.queue.put((sha1_hash, output_filename)) |
189 self.queue.put((None, None)) | 248 self.queue.put((None, None)) |
190 stdout_queue = Queue.Queue() | 249 stdout_queue = Queue.Queue() |
191 download_from_google_storage._downloader_worker_thread( | 250 download_from_google_storage._downloader_worker_thread( |
192 0, self.queue, False, self.base_url, self.gsutil, | 251 0, self.queue, False, self.base_url, self.gsutil, |
193 stdout_queue, self.ret_codes, True) | 252 stdout_queue, self.ret_codes, True, False) |
194 expected_output = [ | 253 expected_output = [ |
195 '0> File %s exists and SHA1 matches. Skipping.' % output_filename | 254 '0> File %s exists and SHA1 matches. Skipping.' % output_filename |
196 ] | 255 ] |
197 self.assertEqual(list(stdout_queue.queue), expected_output) | 256 self.assertEqual(list(stdout_queue.queue), expected_output) |
198 self.assertEqual(self.gsutil.history, []) | 257 self.assertEqual(self.gsutil.history, []) |
199 | 258 |
| 259 def test_download_extract_archive(self): |
| 260 # By design we make this not match |
| 261 sha1_hash = '61223e1ad3d86901a57629fee38313db5ec106ff' |
| 262 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
| 263 # Generate a gzipped tarfile |
| 264 output_filename = os.path.join(self.base_path, 'subfolder.tar.gz') |
| 265 output_dirname = os.path.join(self.base_path, 'subfolder') |
| 266 extracted_filename = os.path.join(output_dirname, 'subfolder_text.txt') |
| 267 with tarfile.open(output_filename, 'w:gz') as tar: |
| 268 tar.add(output_dirname, arcname='subfolder') |
| 269 shutil.rmtree(output_dirname) |
| 270 print(output_dirname) |
| 271 self.queue.put((sha1_hash, output_filename)) |
| 272 self.queue.put((None, None)) |
| 273 stdout_queue = Queue.Queue() |
| 274 download_from_google_storage._downloader_worker_thread( |
| 275 0, self.queue, False, self.base_url, self.gsutil, |
| 276 stdout_queue, self.ret_codes, True, True, delete=False) |
| 277 expected_calls = [ |
| 278 ('check_call', |
| 279 ('ls', input_filename)), |
| 280 ('check_call', |
| 281 ('cp', input_filename, output_filename))] |
| 282 if sys.platform != 'win32': |
| 283 expected_calls.append( |
| 284 ('check_call', |
| 285 ('stat', |
| 286 'gs://sometesturl/61223e1ad3d86901a57629fee38313db5ec106ff'))) |
| 287 expected_output = [ |
| 288 '0> Downloading %s...' % output_filename] |
| 289 expected_output.extend([ |
| 290 '0> Extracting 3 entries from %s to %s' % (output_filename, |
| 291 output_dirname)]) |
| 292 expected_ret_codes = [] |
| 293 self.assertEqual(list(stdout_queue.queue), expected_output) |
| 294 self.assertEqual(self.gsutil.history, expected_calls) |
| 295 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
| 296 self.assertTrue(os.path.exists(output_dirname)) |
| 297 self.assertTrue(os.path.exists(extracted_filename)) |
| 298 |
200 def test_download_worker_skips_not_found_file(self): | 299 def test_download_worker_skips_not_found_file(self): |
201 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 300 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
202 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 301 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
203 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 302 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
204 self.queue.put((sha1_hash, output_filename)) | 303 self.queue.put((sha1_hash, output_filename)) |
205 self.queue.put((None, None)) | 304 self.queue.put((None, None)) |
206 stdout_queue = Queue.Queue() | 305 stdout_queue = Queue.Queue() |
207 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. | 306 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. |
208 download_from_google_storage._downloader_worker_thread( | 307 download_from_google_storage._downloader_worker_thread( |
209 0, self.queue, False, self.base_url, self.gsutil, | 308 0, self.queue, False, self.base_url, self.gsutil, |
210 stdout_queue, self.ret_codes, True) | 309 stdout_queue, self.ret_codes, True, False) |
211 expected_output = [ | 310 expected_output = [ |
212 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( | 311 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( |
213 input_filename, output_filename), | 312 input_filename, output_filename), |
214 ] | 313 ] |
215 expected_calls = [ | 314 expected_calls = [ |
216 ('check_call', | 315 ('check_call', |
217 ('ls', input_filename)) | 316 ('ls', input_filename)) |
218 ] | 317 ] |
219 expected_ret_codes = [ | 318 expected_ret_codes = [ |
220 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( | 319 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( |
(...skipping 14 matching lines...) Expand all Loading... |
235 base_url=self.base_url, | 334 base_url=self.base_url, |
236 gsutil=self.gsutil, | 335 gsutil=self.gsutil, |
237 num_threads=1, | 336 num_threads=1, |
238 directory=False, | 337 directory=False, |
239 recursive=False, | 338 recursive=False, |
240 force=True, | 339 force=True, |
241 output=output_filename, | 340 output=output_filename, |
242 ignore_errors=False, | 341 ignore_errors=False, |
243 sha1_file=False, | 342 sha1_file=False, |
244 verbose=True, | 343 verbose=True, |
245 auto_platform=False) | 344 auto_platform=False, |
| 345 extract=False) |
246 expected_calls = [ | 346 expected_calls = [ |
247 ('check_call', | 347 ('check_call', |
248 ('ls', input_filename)), | 348 ('ls', input_filename)), |
249 ('check_call', | 349 ('check_call', |
250 ('cp', input_filename, output_filename)) | 350 ('cp', input_filename, output_filename)) |
251 ] | 351 ] |
252 if sys.platform != 'win32': | 352 if sys.platform != 'win32': |
253 expected_calls.append( | 353 expected_calls.append( |
254 ('check_call', | 354 ('check_call', |
255 ('stat', | 355 ('stat', |
(...skipping 10 matching lines...) Expand all Loading... |
266 base_url=self.base_url, | 366 base_url=self.base_url, |
267 gsutil=self.gsutil, | 367 gsutil=self.gsutil, |
268 num_threads=1, | 368 num_threads=1, |
269 directory=True, | 369 directory=True, |
270 recursive=False, | 370 recursive=False, |
271 force=False, | 371 force=False, |
272 output=None, | 372 output=None, |
273 ignore_errors=False, | 373 ignore_errors=False, |
274 sha1_file=False, | 374 sha1_file=False, |
275 verbose=True, | 375 verbose=True, |
276 auto_platform=False) | 376 auto_platform=False, |
| 377 extract=False) |
277 expected_calls = [ | 378 expected_calls = [ |
278 ('check_call', | 379 ('check_call', |
279 ('ls', input_filename)), | 380 ('ls', input_filename)), |
280 ('check_call', | 381 ('check_call', |
281 ('cp', input_filename, output_filename))] | 382 ('cp', input_filename, output_filename))] |
282 if sys.platform != 'win32': | 383 if sys.platform != 'win32': |
283 expected_calls.append( | 384 expected_calls.append( |
284 ('check_call', | 385 ('check_call', |
285 ('stat', | 386 ('stat', |
286 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 387 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
287 self.assertEqual(self.gsutil.history, expected_calls) | 388 self.assertEqual(self.gsutil.history, expected_calls) |
288 self.assertEqual(code, 0) | 389 self.assertEqual(code, 0) |
289 | 390 |
290 | 391 |
291 if __name__ == '__main__': | 392 if __name__ == '__main__': |
292 unittest.main() | 393 unittest.main() |
OLD | NEW |