OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 # pylint: disable=W0212 | 5 # pylint: disable=W0212 |
6 | 6 |
7 """Unit tests for download_from_google_storage.py.""" | 7 """Unit tests for download_from_google_storage.py.""" |
8 | 8 |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 import Queue | 11 import Queue |
12 import shutil | 12 import shutil |
13 import sys | 13 import sys |
14 import tempfile | 14 import tempfile |
15 import threading | 15 import threading |
16 import unittest | 16 import unittest |
17 | 17 |
18 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | 18 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
19 | 19 |
20 import upload_to_google_storage | 20 import upload_to_google_storage |
21 import download_from_google_storage | 21 import download_from_google_storage |
22 | 22 |
23 # ../third_party/gsutil/gsutil | 23 # ../third_party/gsutil/gsutil |
24 GSUTIL_DEFAULT_PATH = os.path.join( | 24 GSUTIL_DEFAULT_PATH = os.path.join( |
25 os.path.dirname(os.path.dirname(os.path.abspath(__file__))), | 25 os.path.dirname(os.path.dirname(os.path.abspath(__file__))), |
26 'third_party', 'gsutil', 'gsutil') | 26 'gsutil.py') |
27 TEST_DIR = os.path.dirname(os.path.abspath(__file__)) | 27 TEST_DIR = os.path.dirname(os.path.abspath(__file__)) |
28 | 28 |
29 | 29 |
30 class GsutilMock(object): | 30 class GsutilMock(object): |
31 def __init__(self, path, boto_path, timeout=None): | 31 def __init__(self, path, boto_path, timeout=None): |
32 self.path = path | 32 self.path = path |
33 self.timeout = timeout | 33 self.timeout = timeout |
34 self.boto_path = boto_path | 34 self.boto_path = boto_path |
35 self.expected = [] | 35 self.expected = [] |
36 self.history = [] | 36 self.history = [] |
(...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
162 self.queue.put((sha1_hash, output_filename)) | 162 self.queue.put((sha1_hash, output_filename)) |
163 self.queue.put((None, None)) | 163 self.queue.put((None, None)) |
164 stdout_queue = Queue.Queue() | 164 stdout_queue = Queue.Queue() |
165 download_from_google_storage._downloader_worker_thread( | 165 download_from_google_storage._downloader_worker_thread( |
166 0, self.queue, False, self.base_url, self.gsutil, | 166 0, self.queue, False, self.base_url, self.gsutil, |
167 stdout_queue, self.ret_codes, True) | 167 stdout_queue, self.ret_codes, True) |
168 expected_calls = [ | 168 expected_calls = [ |
169 ('check_call', | 169 ('check_call', |
170 ('ls', input_filename)), | 170 ('ls', input_filename)), |
171 ('check_call', | 171 ('check_call', |
172 ('cp', '-q', input_filename, output_filename))] | 172 ('cp', input_filename, output_filename))] |
173 if sys.platform != 'win32': | 173 if sys.platform != 'win32': |
174 expected_calls.append( | 174 expected_calls.append( |
175 ('check_call', | 175 ('check_call', |
176 ('ls', | 176 ('stat', |
177 '-L', | |
178 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 177 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
179 expected_output = [ | 178 expected_output = [ |
180 '0> Downloading %s...' % output_filename] | 179 '0> Downloading %s...' % output_filename] |
181 expected_ret_codes = [] | 180 expected_ret_codes = [] |
182 self.assertEqual(list(stdout_queue.queue), expected_output) | 181 self.assertEqual(list(stdout_queue.queue), expected_output) |
183 self.assertEqual(self.gsutil.history, expected_calls) | 182 self.assertEqual(self.gsutil.history, expected_calls) |
184 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 183 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
185 | 184 |
186 def test_download_worker_skips_file(self): | 185 def test_download_worker_skips_file(self): |
187 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' | 186 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' |
(...skipping 15 matching lines...) Expand all Loading... |
203 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 202 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
204 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 203 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
205 self.queue.put((sha1_hash, output_filename)) | 204 self.queue.put((sha1_hash, output_filename)) |
206 self.queue.put((None, None)) | 205 self.queue.put((None, None)) |
207 stdout_queue = Queue.Queue() | 206 stdout_queue = Queue.Queue() |
208 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. | 207 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. |
209 download_from_google_storage._downloader_worker_thread( | 208 download_from_google_storage._downloader_worker_thread( |
210 0, self.queue, False, self.base_url, self.gsutil, | 209 0, self.queue, False, self.base_url, self.gsutil, |
211 stdout_queue, self.ret_codes, True) | 210 stdout_queue, self.ret_codes, True) |
212 expected_output = [ | 211 expected_output = [ |
213 '0> File %s for %s does not exist, skipping.' % ( | 212 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( |
214 input_filename, output_filename), | 213 input_filename, output_filename), |
215 ] | 214 ] |
216 expected_calls = [ | 215 expected_calls = [ |
217 ('check_call', | 216 ('check_call', |
218 ('ls', input_filename)) | 217 ('ls', input_filename)) |
219 ] | 218 ] |
220 expected_ret_codes = [ | 219 expected_ret_codes = [ |
221 (1, 'File %s for %s does not exist.' % ( | 220 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( |
222 input_filename, output_filename)) | 221 input_filename, output_filename)) |
223 ] | 222 ] |
224 self.assertEqual(list(stdout_queue.queue), expected_output) | 223 self.assertEqual(list(stdout_queue.queue), expected_output) |
225 self.assertEqual(self.gsutil.history, expected_calls) | 224 self.assertEqual(self.gsutil.history, expected_calls) |
226 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 225 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
227 | 226 |
228 def test_download_cp_fails(self): | 227 def test_download_cp_fails(self): |
229 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 228 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
230 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 229 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
231 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 230 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
232 self.gsutil.add_expected(0, '', '') | 231 self.gsutil.add_expected(0, '', '') |
233 self.gsutil.add_expected(101, '', 'Test error message.') | 232 self.gsutil.add_expected(101, '', 'Test error message.') |
234 code = download_from_google_storage.download_from_google_storage( | 233 code = download_from_google_storage.download_from_google_storage( |
235 input_filename=sha1_hash, | 234 input_filename=sha1_hash, |
236 base_url=self.base_url, | 235 base_url=self.base_url, |
237 gsutil=self.gsutil, | 236 gsutil=self.gsutil, |
238 num_threads=1, | 237 num_threads=1, |
239 directory=False, | 238 directory=False, |
240 recursive=False, | 239 recursive=False, |
241 force=True, | 240 force=True, |
242 output=output_filename, | 241 output=output_filename, |
243 ignore_errors=False, | 242 ignore_errors=False, |
244 sha1_file=False, | 243 sha1_file=False, |
245 verbose=True, | 244 verbose=True, |
246 auto_platform=False) | 245 auto_platform=False) |
247 expected_calls = [ | 246 expected_calls = [ |
248 ('check_call', | 247 ('check_call', |
249 ('ls', input_filename)), | 248 ('ls', input_filename)), |
250 ('check_call', | 249 ('check_call', |
251 ('cp', '-q', input_filename, output_filename)) | 250 ('cp', input_filename, output_filename)) |
252 ] | 251 ] |
253 if sys.platform != 'win32': | 252 if sys.platform != 'win32': |
254 expected_calls.append( | 253 expected_calls.append( |
255 ('check_call', | 254 ('check_call', |
256 ('ls', | 255 ('stat', |
257 '-L', | |
258 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 256 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
259 self.assertEqual(self.gsutil.history, expected_calls) | 257 self.assertEqual(self.gsutil.history, expected_calls) |
260 self.assertEqual(code, 101) | 258 self.assertEqual(code, 101) |
261 | 259 |
262 def test_download_directory_no_recursive_non_force(self): | 260 def test_download_directory_no_recursive_non_force(self): |
263 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 261 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
264 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 262 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
265 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 263 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
266 code = download_from_google_storage.download_from_google_storage( | 264 code = download_from_google_storage.download_from_google_storage( |
267 input_filename=self.base_path, | 265 input_filename=self.base_path, |
268 base_url=self.base_url, | 266 base_url=self.base_url, |
269 gsutil=self.gsutil, | 267 gsutil=self.gsutil, |
270 num_threads=1, | 268 num_threads=1, |
271 directory=True, | 269 directory=True, |
272 recursive=False, | 270 recursive=False, |
273 force=False, | 271 force=False, |
274 output=None, | 272 output=None, |
275 ignore_errors=False, | 273 ignore_errors=False, |
276 sha1_file=False, | 274 sha1_file=False, |
277 verbose=True, | 275 verbose=True, |
278 auto_platform=False) | 276 auto_platform=False) |
279 expected_calls = [ | 277 expected_calls = [ |
280 ('check_call', | 278 ('check_call', |
281 ('ls', input_filename)), | 279 ('ls', input_filename)), |
282 ('check_call', | 280 ('check_call', |
283 ('cp', '-q', input_filename, output_filename))] | 281 ('cp', input_filename, output_filename))] |
284 if sys.platform != 'win32': | 282 if sys.platform != 'win32': |
285 expected_calls.append( | 283 expected_calls.append( |
286 ('check_call', | 284 ('check_call', |
287 ('ls', | 285 ('stat', |
288 '-L', | |
289 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 286 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
290 self.assertEqual(self.gsutil.history, expected_calls) | 287 self.assertEqual(self.gsutil.history, expected_calls) |
291 self.assertEqual(code, 0) | 288 self.assertEqual(code, 0) |
292 | 289 |
293 | 290 |
294 if __name__ == '__main__': | 291 if __name__ == '__main__': |
295 unittest.main() | 292 unittest.main() |
OLD | NEW |