OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 # pylint: disable=W0212 | 5 # pylint: disable=W0212 |
6 | 6 |
7 """Unit tests for download_from_google_storage.py.""" | 7 """Unit tests for download_from_google_storage.py.""" |
8 | 8 |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import shutil | 12 import shutil |
13 import sys | 13 import sys |
14 import tarfile | |
15 import tempfile | 14 import tempfile |
16 import threading | 15 import threading |
17 import unittest | 16 import unittest |
18 | 17 |
19 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | 18 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
20 | 19 |
21 import upload_to_google_storage | 20 import upload_to_google_storage |
22 import download_from_google_storage | 21 import download_from_google_storage |
23 | 22 |
24 # ../third_party/gsutil/gsutil | 23 # ../third_party/gsutil/gsutil |
(...skipping 28 matching lines...) Expand all Loading... |
53 | 52 |
54 def check_call(self, *args): | 53 def check_call(self, *args): |
55 with self.lock: | 54 with self.lock: |
56 self.append_history('check_call', args) | 55 self.append_history('check_call', args) |
57 if self.expected: | 56 if self.expected: |
58 return self.expected.pop(0) | 57 return self.expected.pop(0) |
59 else: | 58 else: |
60 return (0, '', '') | 59 return (0, '', '') |
61 | 60 |
62 | 61 |
63 class ChangedWorkingDirectory(object): | |
64 def __init__(self, working_directory): | |
65 self._old_cwd = '' | |
66 self._working_directory = working_directory | |
67 | |
68 def __enter__(self): | |
69 self._old_cwd = os.getcwd() | |
70 print "Enter directory = ", self._working_directory | |
71 os.chdir(self._working_directory) | |
72 | |
73 def __exit__(self, *_): | |
74 print "Enter directory = ", self._old_cwd | |
75 os.chdir(self._old_cwd) | |
76 | |
77 | |
78 class GstoolsUnitTests(unittest.TestCase): | 62 class GstoolsUnitTests(unittest.TestCase): |
79 def setUp(self): | 63 def setUp(self): |
80 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') | 64 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') |
81 self.base_path = os.path.join(self.temp_dir, 'test_files') | 65 self.base_path = os.path.join(self.temp_dir, 'test_files') |
82 shutil.copytree(os.path.join(TEST_DIR, 'gstools'), self.base_path) | 66 shutil.copytree(os.path.join(TEST_DIR, 'gstools'), self.base_path) |
83 | 67 |
84 def cleanUp(self): | 68 def cleanUp(self): |
85 shutil.rmtree(self.temp_dir) | 69 shutil.rmtree(self.temp_dir) |
86 | 70 |
87 def test_validate_tar_file(self): | |
88 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') | |
89 with ChangedWorkingDirectory(self.base_path): | |
90 # Sanity ok check. | |
91 tar_dir = 'ok_dir' | |
92 os.makedirs(os.path.join(self.base_path, tar_dir)) | |
93 tar = 'good.tar.gz' | |
94 lorem_ipsum_copy = os.path.join(tar_dir, 'lorem_ipsum.txt') | |
95 shutil.copyfile(lorem_ipsum, lorem_ipsum_copy) | |
96 with tarfile.open(tar, 'w:gz') as tar: | |
97 tar.add(lorem_ipsum_copy) | |
98 self.assertTrue( | |
99 download_from_google_storage._validate_tar_file(tar, tar_dir)) | |
100 | |
101 # Test no links. | |
102 tar_dir_link = 'for_tar_link' | |
103 os.makedirs(tar_dir_link) | |
104 link = os.path.join(tar_dir_link, 'link') | |
105 os.symlink(lorem_ipsum, link) | |
106 tar_with_links = 'with_links.tar.gz' | |
107 with tarfile.open(tar_with_links, 'w:gz') as tar: | |
108 tar.add(link) | |
109 self.assertFalse( | |
110 download_from_google_storage._validate_tar_file(tar, tar_dir_link)) | |
111 | |
112 # Test not outside. | |
113 tar_dir_outside = 'outside_tar' | |
114 os.makedirs(tar_dir_outside) | |
115 tar_with_outside = 'with_outside.tar.gz' | |
116 with tarfile.open(tar_with_outside, 'w:gz') as tar: | |
117 tar.add(lorem_ipsum) | |
118 self.assertFalse( | |
119 download_from_google_storage._validate_tar_file(tar, | |
120 tar_dir_outside)) | |
121 # Test no .. | |
122 tar_with_dotdot = 'with_dotdot.tar.gz' | |
123 dotdot_file = os.path.join(tar_dir, '..', tar_dir, 'lorem_ipsum.txt') | |
124 with tarfile.open(tar_with_dotdot, 'w:gz') as tar: | |
125 tar.add(dotdot_file) | |
126 self.assertFalse( | |
127 download_from_google_storage._validate_tar_file(tar, | |
128 tar_dir)) | |
129 | |
130 def test_gsutil(self): | 71 def test_gsutil(self): |
131 gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None) | 72 gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None) |
132 self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH) | 73 self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH) |
133 code, _, err = gsutil.check_call() | 74 code, _, err = gsutil.check_call() |
134 self.assertEqual(code, 0) | 75 self.assertEqual(code, 0) |
135 self.assertEqual(err, '') | 76 self.assertEqual(err, '') |
136 | 77 |
137 def test_get_sha1(self): | 78 def test_get_sha1(self): |
138 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') | 79 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') |
139 self.assertEqual( | 80 self.assertEqual( |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
216 | 157 |
217 def test_download_worker_single_file(self): | 158 def test_download_worker_single_file(self): |
218 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 159 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
219 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 160 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
220 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 161 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
221 self.queue.put((sha1_hash, output_filename)) | 162 self.queue.put((sha1_hash, output_filename)) |
222 self.queue.put((None, None)) | 163 self.queue.put((None, None)) |
223 stdout_queue = Queue.Queue() | 164 stdout_queue = Queue.Queue() |
224 download_from_google_storage._downloader_worker_thread( | 165 download_from_google_storage._downloader_worker_thread( |
225 0, self.queue, False, self.base_url, self.gsutil, | 166 0, self.queue, False, self.base_url, self.gsutil, |
226 stdout_queue, self.ret_codes, True, False) | 167 stdout_queue, self.ret_codes, True) |
227 expected_calls = [ | 168 expected_calls = [ |
228 ('check_call', | 169 ('check_call', |
229 ('ls', input_filename)), | 170 ('ls', input_filename)), |
230 ('check_call', | 171 ('check_call', |
231 ('cp', input_filename, output_filename))] | 172 ('cp', input_filename, output_filename))] |
232 if sys.platform != 'win32': | 173 if sys.platform != 'win32': |
233 expected_calls.append( | 174 expected_calls.append( |
234 ('check_call', | 175 ('check_call', |
235 ('stat', | 176 ('stat', |
236 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 177 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
237 expected_output = [ | 178 expected_output = [ |
238 '0> Downloading %s...' % output_filename] | 179 '0> Downloading %s...' % output_filename] |
239 expected_ret_codes = [] | 180 expected_ret_codes = [] |
240 self.assertEqual(list(stdout_queue.queue), expected_output) | 181 self.assertEqual(list(stdout_queue.queue), expected_output) |
241 self.assertEqual(self.gsutil.history, expected_calls) | 182 self.assertEqual(self.gsutil.history, expected_calls) |
242 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 183 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
243 | 184 |
244 def test_download_worker_skips_file(self): | 185 def test_download_worker_skips_file(self): |
245 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' | 186 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' |
246 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') | 187 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') |
247 self.queue.put((sha1_hash, output_filename)) | 188 self.queue.put((sha1_hash, output_filename)) |
248 self.queue.put((None, None)) | 189 self.queue.put((None, None)) |
249 stdout_queue = Queue.Queue() | 190 stdout_queue = Queue.Queue() |
250 download_from_google_storage._downloader_worker_thread( | 191 download_from_google_storage._downloader_worker_thread( |
251 0, self.queue, False, self.base_url, self.gsutil, | 192 0, self.queue, False, self.base_url, self.gsutil, |
252 stdout_queue, self.ret_codes, True, False) | 193 stdout_queue, self.ret_codes, True) |
253 expected_output = [ | 194 expected_output = [ |
254 '0> File %s exists and SHA1 matches. Skipping.' % output_filename | 195 '0> File %s exists and SHA1 matches. Skipping.' % output_filename |
255 ] | 196 ] |
256 self.assertEqual(list(stdout_queue.queue), expected_output) | 197 self.assertEqual(list(stdout_queue.queue), expected_output) |
257 self.assertEqual(self.gsutil.history, []) | 198 self.assertEqual(self.gsutil.history, []) |
258 | 199 |
259 def test_download_extract_archive(self): | |
260 # By design we make this not match | |
261 sha1_hash = '61223e1ad3d86901a57629fee38313db5ec106ff' | |
262 input_filename = '%s/%s' % (self.base_url, sha1_hash) | |
263 # Generate a gzipped tarfile | |
264 output_filename = os.path.join(self.base_path, 'subfolder.tar.gz') | |
265 output_dirname = os.path.join(self.base_path, 'subfolder') | |
266 extracted_filename = os.path.join(output_dirname, 'subfolder_text.txt') | |
267 with tarfile.open(output_filename, 'w:gz') as tar: | |
268 tar.add(output_dirname, arcname='subfolder') | |
269 shutil.rmtree(output_dirname) | |
270 print(output_dirname) | |
271 self.queue.put((sha1_hash, output_filename)) | |
272 self.queue.put((None, None)) | |
273 stdout_queue = Queue.Queue() | |
274 download_from_google_storage._downloader_worker_thread( | |
275 0, self.queue, False, self.base_url, self.gsutil, | |
276 stdout_queue, self.ret_codes, True, True, delete=False) | |
277 expected_calls = [ | |
278 ('check_call', | |
279 ('ls', input_filename)), | |
280 ('check_call', | |
281 ('cp', input_filename, output_filename))] | |
282 if sys.platform != 'win32': | |
283 expected_calls.append( | |
284 ('check_call', | |
285 ('stat', | |
286 'gs://sometesturl/61223e1ad3d86901a57629fee38313db5ec106ff'))) | |
287 expected_output = [ | |
288 '0> Downloading %s...' % output_filename] | |
289 expected_output.extend([ | |
290 '0> Extracting 3 entries from %s to %s' % (output_filename, | |
291 output_dirname)]) | |
292 expected_ret_codes = [] | |
293 self.assertEqual(list(stdout_queue.queue), expected_output) | |
294 self.assertEqual(self.gsutil.history, expected_calls) | |
295 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | |
296 self.assertTrue(os.path.exists(output_dirname)) | |
297 self.assertTrue(os.path.exists(extracted_filename)) | |
298 | |
299 def test_download_worker_skips_not_found_file(self): | 200 def test_download_worker_skips_not_found_file(self): |
300 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 201 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
301 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 202 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
302 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 203 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
303 self.queue.put((sha1_hash, output_filename)) | 204 self.queue.put((sha1_hash, output_filename)) |
304 self.queue.put((None, None)) | 205 self.queue.put((None, None)) |
305 stdout_queue = Queue.Queue() | 206 stdout_queue = Queue.Queue() |
306 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. | 207 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. |
307 download_from_google_storage._downloader_worker_thread( | 208 download_from_google_storage._downloader_worker_thread( |
308 0, self.queue, False, self.base_url, self.gsutil, | 209 0, self.queue, False, self.base_url, self.gsutil, |
309 stdout_queue, self.ret_codes, True, False) | 210 stdout_queue, self.ret_codes, True) |
310 expected_output = [ | 211 expected_output = [ |
311 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( | 212 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( |
312 input_filename, output_filename), | 213 input_filename, output_filename), |
313 ] | 214 ] |
314 expected_calls = [ | 215 expected_calls = [ |
315 ('check_call', | 216 ('check_call', |
316 ('ls', input_filename)) | 217 ('ls', input_filename)) |
317 ] | 218 ] |
318 expected_ret_codes = [ | 219 expected_ret_codes = [ |
319 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( | 220 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( |
(...skipping 14 matching lines...) Expand all Loading... |
334 base_url=self.base_url, | 235 base_url=self.base_url, |
335 gsutil=self.gsutil, | 236 gsutil=self.gsutil, |
336 num_threads=1, | 237 num_threads=1, |
337 directory=False, | 238 directory=False, |
338 recursive=False, | 239 recursive=False, |
339 force=True, | 240 force=True, |
340 output=output_filename, | 241 output=output_filename, |
341 ignore_errors=False, | 242 ignore_errors=False, |
342 sha1_file=False, | 243 sha1_file=False, |
343 verbose=True, | 244 verbose=True, |
344 auto_platform=False, | 245 auto_platform=False) |
345 extract=False) | |
346 expected_calls = [ | 246 expected_calls = [ |
347 ('check_call', | 247 ('check_call', |
348 ('ls', input_filename)), | 248 ('ls', input_filename)), |
349 ('check_call', | 249 ('check_call', |
350 ('cp', input_filename, output_filename)) | 250 ('cp', input_filename, output_filename)) |
351 ] | 251 ] |
352 if sys.platform != 'win32': | 252 if sys.platform != 'win32': |
353 expected_calls.append( | 253 expected_calls.append( |
354 ('check_call', | 254 ('check_call', |
355 ('stat', | 255 ('stat', |
(...skipping 10 matching lines...) Expand all Loading... |
366 base_url=self.base_url, | 266 base_url=self.base_url, |
367 gsutil=self.gsutil, | 267 gsutil=self.gsutil, |
368 num_threads=1, | 268 num_threads=1, |
369 directory=True, | 269 directory=True, |
370 recursive=False, | 270 recursive=False, |
371 force=False, | 271 force=False, |
372 output=None, | 272 output=None, |
373 ignore_errors=False, | 273 ignore_errors=False, |
374 sha1_file=False, | 274 sha1_file=False, |
375 verbose=True, | 275 verbose=True, |
376 auto_platform=False, | 276 auto_platform=False) |
377 extract=False) | |
378 expected_calls = [ | 277 expected_calls = [ |
379 ('check_call', | 278 ('check_call', |
380 ('ls', input_filename)), | 279 ('ls', input_filename)), |
381 ('check_call', | 280 ('check_call', |
382 ('cp', input_filename, output_filename))] | 281 ('cp', input_filename, output_filename))] |
383 if sys.platform != 'win32': | 282 if sys.platform != 'win32': |
384 expected_calls.append( | 283 expected_calls.append( |
385 ('check_call', | 284 ('check_call', |
386 ('stat', | 285 ('stat', |
387 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 286 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
388 self.assertEqual(self.gsutil.history, expected_calls) | 287 self.assertEqual(self.gsutil.history, expected_calls) |
389 self.assertEqual(code, 0) | 288 self.assertEqual(code, 0) |
390 | 289 |
391 | 290 |
392 if __name__ == '__main__': | 291 if __name__ == '__main__': |
393 unittest.main() | 292 unittest.main() |
OLD | NEW |