Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(693)

Side by Side Diff: tests/download_from_google_storage_unittests.py

Issue 807463005: Add support for tar.gz archive files to download from download_from_google_storage (Closed) Base URL: http://src.chromium.org/svn/trunk/tools/depot_tools/
Patch Set: add some tests Created 5 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 # pylint: disable=W0212 5 # pylint: disable=W0212
6 6
7 """Unit tests for download_from_google_storage.py.""" 7 """Unit tests for download_from_google_storage.py."""
8 8
9 import optparse 9 import optparse
10 import os 10 import os
11 import Queue 11 import Queue
12 import shutil 12 import shutil
13 import sys 13 import sys
14 import tarfile
14 import tempfile 15 import tempfile
15 import threading 16 import threading
16 import unittest 17 import unittest
17 18
18 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 19 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
19 20
20 import upload_to_google_storage 21 import upload_to_google_storage
21 import download_from_google_storage 22 import download_from_google_storage
22 23
23 # ../third_party/gsutil/gsutil 24 # ../third_party/gsutil/gsutil
(...skipping 27 matching lines...) Expand all
51 return 0 52 return 0
52 53
53 def check_call(self, *args): 54 def check_call(self, *args):
54 with self.lock: 55 with self.lock:
55 self.append_history('check_call', args) 56 self.append_history('check_call', args)
56 if self.expected: 57 if self.expected:
57 return self.expected.pop(0) 58 return self.expected.pop(0)
58 else: 59 else:
59 return (0, '', '') 60 return (0, '', '')
60 61
62 class ChangedWorkingDirectory(object):
63 def __init__(self, working_directory):
64 self._old_cwd = ''
65 self._working_directory = working_directory
66
67 def __enter__(self):
68 self._old_cwd = os.getcwd()
69 print "Enter directory = ", self._working_directory
70 os.chdir(self._working_directory)
71
72 def __exit__(self, *_):
73 print "Enter directory = ", self._old_cwd
74 os.chdir(self._old_cwd)
61 75
62 class GstoolsUnitTests(unittest.TestCase): 76 class GstoolsUnitTests(unittest.TestCase):
63 def setUp(self): 77 def setUp(self):
64 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') 78 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test')
65 self.base_path = os.path.join(self.temp_dir, 'test_files') 79 self.base_path = os.path.join(self.temp_dir, 'test_files')
66 shutil.copytree(os.path.join(TEST_DIR, 'gstools'), self.base_path) 80 shutil.copytree(os.path.join(TEST_DIR, 'gstools'), self.base_path)
67 81
68 def cleanUp(self): 82 def cleanUp(self):
69 shutil.rmtree(self.temp_dir) 83 shutil.rmtree(self.temp_dir)
70 84
85 def test_validate_tar_file(self):
86 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt')
87 with ChangedWorkingDirectory(self.base_path):
88 # Sanity ok check.
89 tar_dir = 'ok_dir'
90 os.makedirs(tar_dir)
hinoka 2015/06/24 18:54:38 use absolute paths os.path.join(self.base_path, t
ricow1 2015/06/25 06:24:05 We already changed into self.base_path, but done
91 tar = 'good.tar.gz'
92 lorem_ipsum_copy = os.path.join(tar_dir, 'lorem_ipsum.txt')
93 shutil.copyfile(lorem_ipsum, lorem_ipsum_copy)
94 with tarfile.open(tar, 'w:gz') as tar:
95 tar.add(lorem_ipsum_copy)
96 self.assertTrue(
97 download_from_google_storage._validate_tar_file(tar, tar_dir))
98
99 # Test no links.
100 tar_dir_link = 'for_tar_link'
101 os.makedirs(tar_dir_link)
102 link = os.path.join(tar_dir_link, 'link')
103 os.symlink(lorem_ipsum, link)
104 tar_with_links = 'with_links.tar.gz'
105 with tarfile.open(tar_with_links, 'w:gz') as tar:
106 tar.add(link)
107 self.assertFalse(
108 download_from_google_storage._validate_tar_file(tar, tar_dir_link))
109
110 # Test not outside.
111 tar_dir_outside = 'outside_tar'
112 os.makedirs(tar_dir_outside)
113 tar_with_outside = 'with_outside.tar.gz'
114 with tarfile.open(tar_with_outside, 'w:gz') as tar:
115 tar.add(lorem_ipsum)
116 self.assertFalse(
117 download_from_google_storage._validate_tar_file(tar,
118 tar_dir_outside))
119 # Test no ..
120 tar_with_dotdot = 'with_dotdot.tar.gz'
121 dotdot_file = os.path.join(tar_dir, '..', tar_dir, 'lorem_ipsum.txt')
122 with tarfile.open(tar_with_dotdot, 'w:gz') as tar:
123 tar.add(dotdot_file)
124 self.assertFalse(
125 download_from_google_storage._validate_tar_file(tar,
126 tar_dir))
127
71 def test_gsutil(self): 128 def test_gsutil(self):
72 gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None) 129 gsutil = download_from_google_storage.Gsutil(GSUTIL_DEFAULT_PATH, None)
73 self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH) 130 self.assertEqual(gsutil.path, GSUTIL_DEFAULT_PATH)
74 code, _, err = gsutil.check_call() 131 code, _, err = gsutil.check_call()
75 self.assertEqual(code, 0) 132 self.assertEqual(code, 0)
76 self.assertEqual(err, '') 133 self.assertEqual(err, '')
77 134
78 def test_get_sha1(self): 135 def test_get_sha1(self):
79 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') 136 lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt')
80 self.assertEqual( 137 self.assertEqual(
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
112 169
113 170
114 class DownloadTests(unittest.TestCase): 171 class DownloadTests(unittest.TestCase):
115 def setUp(self): 172 def setUp(self):
116 self.gsutil = GsutilMock(GSUTIL_DEFAULT_PATH, None) 173 self.gsutil = GsutilMock(GSUTIL_DEFAULT_PATH, None)
117 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test') 174 self.temp_dir = tempfile.mkdtemp(prefix='gstools_test')
118 self.checkout_test_files = os.path.join( 175 self.checkout_test_files = os.path.join(
119 TEST_DIR, 'gstools', 'download_test_data') 176 TEST_DIR, 'gstools', 'download_test_data')
120 self.base_path = os.path.join( 177 self.base_path = os.path.join(
121 self.temp_dir, 'download_test_data') 178 self.temp_dir, 'download_test_data')
179 print "copying %s to %s" % (self.checkout_test_files, self.base_path)
122 shutil.copytree(self.checkout_test_files, self.base_path) 180 shutil.copytree(self.checkout_test_files, self.base_path)
123 self.base_url = 'gs://sometesturl' 181 self.base_url = 'gs://sometesturl'
124 self.parser = optparse.OptionParser() 182 self.parser = optparse.OptionParser()
125 self.queue = Queue.Queue() 183 self.queue = Queue.Queue()
126 self.ret_codes = Queue.Queue() 184 self.ret_codes = Queue.Queue()
127 self.lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt') 185 self.lorem_ipsum = os.path.join(self.base_path, 'lorem_ipsum.txt')
128 self.lorem_ipsum_sha1 = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' 186 self.lorem_ipsum_sha1 = '7871c8e24da15bad8b0be2c36edc9dc77e37727f'
129 self.maxDiff = None 187 self.maxDiff = None
130 188
131 def cleanUp(self): 189 def cleanUp(self):
(...skipping 25 matching lines...) Expand all
157 215
158 def test_download_worker_single_file(self): 216 def test_download_worker_single_file(self):
159 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' 217 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f'
160 input_filename = '%s/%s' % (self.base_url, sha1_hash) 218 input_filename = '%s/%s' % (self.base_url, sha1_hash)
161 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') 219 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt')
162 self.queue.put((sha1_hash, output_filename)) 220 self.queue.put((sha1_hash, output_filename))
163 self.queue.put((None, None)) 221 self.queue.put((None, None))
164 stdout_queue = Queue.Queue() 222 stdout_queue = Queue.Queue()
165 download_from_google_storage._downloader_worker_thread( 223 download_from_google_storage._downloader_worker_thread(
166 0, self.queue, False, self.base_url, self.gsutil, 224 0, self.queue, False, self.base_url, self.gsutil,
167 stdout_queue, self.ret_codes, True) 225 stdout_queue, self.ret_codes, True, False)
168 expected_calls = [ 226 expected_calls = [
169 ('check_call', 227 ('check_call',
170 ('ls', input_filename)), 228 ('ls', input_filename)),
171 ('check_call', 229 ('check_call',
172 ('cp', input_filename, output_filename))] 230 ('cp', input_filename, output_filename))]
173 if sys.platform != 'win32': 231 if sys.platform != 'win32':
174 expected_calls.append( 232 expected_calls.append(
175 ('check_call', 233 ('check_call',
176 ('stat', 234 ('stat',
177 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) 235 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f')))
178 expected_output = [ 236 expected_output = [
179 '0> Downloading %s...' % output_filename] 237 '0> Downloading %s...' % output_filename]
180 expected_ret_codes = [] 238 expected_ret_codes = []
181 self.assertEqual(list(stdout_queue.queue), expected_output) 239 self.assertEqual(list(stdout_queue.queue), expected_output)
182 self.assertEqual(self.gsutil.history, expected_calls) 240 self.assertEqual(self.gsutil.history, expected_calls)
183 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) 241 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes)
184 242
185 def test_download_worker_skips_file(self): 243 def test_download_worker_skips_file(self):
186 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' 244 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe'
187 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') 245 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt')
188 self.queue.put((sha1_hash, output_filename)) 246 self.queue.put((sha1_hash, output_filename))
189 self.queue.put((None, None)) 247 self.queue.put((None, None))
190 stdout_queue = Queue.Queue() 248 stdout_queue = Queue.Queue()
191 download_from_google_storage._downloader_worker_thread( 249 download_from_google_storage._downloader_worker_thread(
192 0, self.queue, False, self.base_url, self.gsutil, 250 0, self.queue, False, self.base_url, self.gsutil,
193 stdout_queue, self.ret_codes, True) 251 stdout_queue, self.ret_codes, True, False)
194 expected_output = [ 252 expected_output = [
195 '0> File %s exists and SHA1 matches. Skipping.' % output_filename 253 '0> File %s exists and SHA1 matches. Skipping.' % output_filename
196 ] 254 ]
197 self.assertEqual(list(stdout_queue.queue), expected_output) 255 self.assertEqual(list(stdout_queue.queue), expected_output)
198 self.assertEqual(self.gsutil.history, []) 256 self.assertEqual(self.gsutil.history, [])
199 257
258 def test_download_extract_archive(self):
259 # By design we make this not match
260 sha1_hash = '61223e1ad3d86901a57629fee38313db5ec106ff'
261 input_filename = '%s/%s' % (self.base_url, sha1_hash)
262 output_filename = os.path.join(self.base_path, 'tarfolder.tar.gz')
hinoka 2015/06/24 18:54:38 can you just construct the tarfile here instead of
ricow1 2015/06/25 06:24:05 Done.
263 output_dirname = os.path.join(self.base_path, 'tarfolder')
264 extracted_filename = os.path.join(output_dirname, 'subfolder_text.txt')
265 self.queue.put((sha1_hash, output_filename))
266 self.queue.put((None, None))
267 stdout_queue = Queue.Queue()
268 download_from_google_storage._downloader_worker_thread(
269 0, self.queue, False, self.base_url, self.gsutil,
270 stdout_queue, self.ret_codes, True, True, delete=False)
271 expected_calls = [
272 ('check_call',
273 ('ls', input_filename)),
274 ('check_call',
275 ('cp', input_filename, output_filename))]
276 if sys.platform != 'win32':
277 expected_calls.append(
278 ('check_call',
279 ('stat',
280 'gs://sometesturl/61223e1ad3d86901a57629fee38313db5ec106ff')))
281 expected_output = [
282 '0> Downloading %s...' % output_filename]
283 expected_output.extend([
284 '0> Extracting %s...' % output_dirname])
285 expected_ret_codes = []
286 self.assertTrue(os.path.exists(output_dirname))
287 self.assertTrue(os.path.exists(extracted_filename))
288 self.assertEqual(list(stdout_queue.queue), expected_output)
289 self.assertEqual(self.gsutil.history, expected_calls)
290 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes)
291
200 def test_download_worker_skips_not_found_file(self): 292 def test_download_worker_skips_not_found_file(self):
201 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' 293 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f'
202 input_filename = '%s/%s' % (self.base_url, sha1_hash) 294 input_filename = '%s/%s' % (self.base_url, sha1_hash)
203 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') 295 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt')
204 self.queue.put((sha1_hash, output_filename)) 296 self.queue.put((sha1_hash, output_filename))
205 self.queue.put((None, None)) 297 self.queue.put((None, None))
206 stdout_queue = Queue.Queue() 298 stdout_queue = Queue.Queue()
207 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. 299 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called.
208 download_from_google_storage._downloader_worker_thread( 300 download_from_google_storage._downloader_worker_thread(
209 0, self.queue, False, self.base_url, self.gsutil, 301 0, self.queue, False, self.base_url, self.gsutil,
210 stdout_queue, self.ret_codes, True) 302 stdout_queue, self.ret_codes, True, False)
211 expected_output = [ 303 expected_output = [
212 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( 304 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % (
213 input_filename, output_filename), 305 input_filename, output_filename),
214 ] 306 ]
215 expected_calls = [ 307 expected_calls = [
216 ('check_call', 308 ('check_call',
217 ('ls', input_filename)) 309 ('ls', input_filename))
218 ] 310 ]
219 expected_ret_codes = [ 311 expected_ret_codes = [
220 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( 312 (1, 'Failed to fetch file %s for %s. [Err: ]' % (
(...skipping 14 matching lines...) Expand all
235 base_url=self.base_url, 327 base_url=self.base_url,
236 gsutil=self.gsutil, 328 gsutil=self.gsutil,
237 num_threads=1, 329 num_threads=1,
238 directory=False, 330 directory=False,
239 recursive=False, 331 recursive=False,
240 force=True, 332 force=True,
241 output=output_filename, 333 output=output_filename,
242 ignore_errors=False, 334 ignore_errors=False,
243 sha1_file=False, 335 sha1_file=False,
244 verbose=True, 336 verbose=True,
245 auto_platform=False) 337 auto_platform=False,
338 extract=False)
246 expected_calls = [ 339 expected_calls = [
247 ('check_call', 340 ('check_call',
248 ('ls', input_filename)), 341 ('ls', input_filename)),
249 ('check_call', 342 ('check_call',
250 ('cp', input_filename, output_filename)) 343 ('cp', input_filename, output_filename))
251 ] 344 ]
252 if sys.platform != 'win32': 345 if sys.platform != 'win32':
253 expected_calls.append( 346 expected_calls.append(
254 ('check_call', 347 ('check_call',
255 ('stat', 348 ('stat',
(...skipping 10 matching lines...) Expand all
266 base_url=self.base_url, 359 base_url=self.base_url,
267 gsutil=self.gsutil, 360 gsutil=self.gsutil,
268 num_threads=1, 361 num_threads=1,
269 directory=True, 362 directory=True,
270 recursive=False, 363 recursive=False,
271 force=False, 364 force=False,
272 output=None, 365 output=None,
273 ignore_errors=False, 366 ignore_errors=False,
274 sha1_file=False, 367 sha1_file=False,
275 verbose=True, 368 verbose=True,
276 auto_platform=False) 369 auto_platform=False,
370 extract=False)
277 expected_calls = [ 371 expected_calls = [
278 ('check_call', 372 ('check_call',
279 ('ls', input_filename)), 373 ('ls', input_filename)),
280 ('check_call', 374 ('check_call',
281 ('cp', input_filename, output_filename))] 375 ('cp', input_filename, output_filename))]
282 if sys.platform != 'win32': 376 if sys.platform != 'win32':
283 expected_calls.append( 377 expected_calls.append(
284 ('check_call', 378 ('check_call',
285 ('stat', 379 ('stat',
286 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) 380 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f')))
287 self.assertEqual(self.gsutil.history, expected_calls) 381 self.assertEqual(self.gsutil.history, expected_calls)
288 self.assertEqual(code, 0) 382 self.assertEqual(code, 0)
289 383
290 384
291 if __name__ == '__main__': 385 if __name__ == '__main__':
292 unittest.main() 386 unittest.main()
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698