| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 # pylint: disable=W0212 | 5 # pylint: disable=W0212 |
| 6 | 6 |
| 7 """Unit tests for download_from_google_storage.py.""" | 7 """Unit tests for download_from_google_storage.py.""" |
| 8 | 8 |
| 9 import optparse | 9 import optparse |
| 10 import os | 10 import os |
| 11 import Queue | 11 import Queue |
| 12 import shutil | 12 import shutil |
| 13 import sys | 13 import sys |
| 14 import tempfile | 14 import tempfile |
| 15 import threading | 15 import threading |
| 16 import unittest | 16 import unittest |
| 17 | 17 |
| 18 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | 18 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| 19 | 19 |
| 20 import upload_to_google_storage | 20 import upload_to_google_storage |
| 21 import download_from_google_storage | 21 import download_from_google_storage |
| 22 | 22 |
| 23 # ../third_party/gsutil/gsutil | 23 # ../third_party/gsutil/gsutil |
| 24 GSUTIL_DEFAULT_PATH = os.path.join( | 24 GSUTIL_DEFAULT_PATH = os.path.join( |
| 25 os.path.dirname(os.path.dirname(os.path.abspath(__file__))), | 25 os.path.dirname(os.path.dirname(os.path.abspath(__file__))), |
| 26 'third_party', 'gsutil', 'gsutil') | 26 'gsutil.py') |
| 27 TEST_DIR = os.path.dirname(os.path.abspath(__file__)) | 27 TEST_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 28 | 28 |
| 29 | 29 |
| 30 class GsutilMock(object): | 30 class GsutilMock(object): |
| 31 def __init__(self, path, boto_path, timeout=None): | 31 def __init__(self, path, boto_path, timeout=None): |
| 32 self.path = path | 32 self.path = path |
| 33 self.timeout = timeout | 33 self.timeout = timeout |
| 34 self.boto_path = boto_path | 34 self.boto_path = boto_path |
| 35 self.expected = [] | 35 self.expected = [] |
| 36 self.history = [] | 36 self.history = [] |
| (...skipping 125 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 162 self.queue.put((sha1_hash, output_filename)) | 162 self.queue.put((sha1_hash, output_filename)) |
| 163 self.queue.put((None, None)) | 163 self.queue.put((None, None)) |
| 164 stdout_queue = Queue.Queue() | 164 stdout_queue = Queue.Queue() |
| 165 download_from_google_storage._downloader_worker_thread( | 165 download_from_google_storage._downloader_worker_thread( |
| 166 0, self.queue, False, self.base_url, self.gsutil, | 166 0, self.queue, False, self.base_url, self.gsutil, |
| 167 stdout_queue, self.ret_codes, True) | 167 stdout_queue, self.ret_codes, True) |
| 168 expected_calls = [ | 168 expected_calls = [ |
| 169 ('check_call', | 169 ('check_call', |
| 170 ('ls', input_filename)), | 170 ('ls', input_filename)), |
| 171 ('check_call', | 171 ('check_call', |
| 172 ('cp', '-q', input_filename, output_filename))] | 172 ('cp', input_filename, output_filename))] |
| 173 if sys.platform != 'win32': | 173 if sys.platform != 'win32': |
| 174 expected_calls.append( | 174 expected_calls.append( |
| 175 ('check_call', | 175 ('check_call', |
| 176 ('ls', | 176 ('stat', |
| 177 '-L', | |
| 178 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 177 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
| 179 expected_output = [ | 178 expected_output = [ |
| 180 '0> Downloading %s...' % output_filename] | 179 '0> Downloading %s...' % output_filename] |
| 181 expected_ret_codes = [] | 180 expected_ret_codes = [] |
| 182 self.assertEqual(list(stdout_queue.queue), expected_output) | 181 self.assertEqual(list(stdout_queue.queue), expected_output) |
| 183 self.assertEqual(self.gsutil.history, expected_calls) | 182 self.assertEqual(self.gsutil.history, expected_calls) |
| 184 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 183 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
| 185 | 184 |
| 186 def test_download_worker_skips_file(self): | 185 def test_download_worker_skips_file(self): |
| 187 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' | 186 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' |
| (...skipping 15 matching lines...) Expand all Loading... |
| 203 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 202 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
| 204 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 203 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
| 205 self.queue.put((sha1_hash, output_filename)) | 204 self.queue.put((sha1_hash, output_filename)) |
| 206 self.queue.put((None, None)) | 205 self.queue.put((None, None)) |
| 207 stdout_queue = Queue.Queue() | 206 stdout_queue = Queue.Queue() |
| 208 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. | 207 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. |
| 209 download_from_google_storage._downloader_worker_thread( | 208 download_from_google_storage._downloader_worker_thread( |
| 210 0, self.queue, False, self.base_url, self.gsutil, | 209 0, self.queue, False, self.base_url, self.gsutil, |
| 211 stdout_queue, self.ret_codes, True) | 210 stdout_queue, self.ret_codes, True) |
| 212 expected_output = [ | 211 expected_output = [ |
| 213 '0> File %s for %s does not exist, skipping.' % ( | 212 '0> Failed to fetch file %s for %s, skipping. [Err: ]' % ( |
| 214 input_filename, output_filename), | 213 input_filename, output_filename), |
| 215 ] | 214 ] |
| 216 expected_calls = [ | 215 expected_calls = [ |
| 217 ('check_call', | 216 ('check_call', |
| 218 ('ls', input_filename)) | 217 ('ls', input_filename)) |
| 219 ] | 218 ] |
| 220 expected_ret_codes = [ | 219 expected_ret_codes = [ |
| 221 (1, 'File %s for %s does not exist.' % ( | 220 (1, 'Failed to fetch file %s for %s. [Err: ]' % ( |
| 222 input_filename, output_filename)) | 221 input_filename, output_filename)) |
| 223 ] | 222 ] |
| 224 self.assertEqual(list(stdout_queue.queue), expected_output) | 223 self.assertEqual(list(stdout_queue.queue), expected_output) |
| 225 self.assertEqual(self.gsutil.history, expected_calls) | 224 self.assertEqual(self.gsutil.history, expected_calls) |
| 226 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 225 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
| 227 | 226 |
| 228 def test_download_cp_fails(self): | 227 def test_download_cp_fails(self): |
| 229 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 228 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
| 230 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 229 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
| 231 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 230 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
| 232 self.gsutil.add_expected(0, '', '') | 231 self.gsutil.add_expected(0, '', '') |
| 233 self.gsutil.add_expected(101, '', 'Test error message.') | 232 self.gsutil.add_expected(101, '', 'Test error message.') |
| 234 code = download_from_google_storage.download_from_google_storage( | 233 code = download_from_google_storage.download_from_google_storage( |
| 235 input_filename=sha1_hash, | 234 input_filename=sha1_hash, |
| 236 base_url=self.base_url, | 235 base_url=self.base_url, |
| 237 gsutil=self.gsutil, | 236 gsutil=self.gsutil, |
| 238 num_threads=1, | 237 num_threads=1, |
| 239 directory=False, | 238 directory=False, |
| 240 recursive=False, | 239 recursive=False, |
| 241 force=True, | 240 force=True, |
| 242 output=output_filename, | 241 output=output_filename, |
| 243 ignore_errors=False, | 242 ignore_errors=False, |
| 244 sha1_file=False, | 243 sha1_file=False, |
| 245 verbose=True, | 244 verbose=True, |
| 246 auto_platform=False) | 245 auto_platform=False) |
| 247 expected_calls = [ | 246 expected_calls = [ |
| 248 ('check_call', | 247 ('check_call', |
| 249 ('ls', input_filename)), | 248 ('ls', input_filename)), |
| 250 ('check_call', | 249 ('check_call', |
| 251 ('cp', '-q', input_filename, output_filename)) | 250 ('cp', input_filename, output_filename)) |
| 252 ] | 251 ] |
| 253 if sys.platform != 'win32': | 252 if sys.platform != 'win32': |
| 254 expected_calls.append( | 253 expected_calls.append( |
| 255 ('check_call', | 254 ('check_call', |
| 256 ('ls', | 255 ('stat', |
| 257 '-L', | |
| 258 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 256 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
| 259 self.assertEqual(self.gsutil.history, expected_calls) | 257 self.assertEqual(self.gsutil.history, expected_calls) |
| 260 self.assertEqual(code, 101) | 258 self.assertEqual(code, 101) |
| 261 | 259 |
| 262 def test_download_directory_no_recursive_non_force(self): | 260 def test_download_directory_no_recursive_non_force(self): |
| 263 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 261 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
| 264 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 262 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
| 265 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 263 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
| 266 code = download_from_google_storage.download_from_google_storage( | 264 code = download_from_google_storage.download_from_google_storage( |
| 267 input_filename=self.base_path, | 265 input_filename=self.base_path, |
| 268 base_url=self.base_url, | 266 base_url=self.base_url, |
| 269 gsutil=self.gsutil, | 267 gsutil=self.gsutil, |
| 270 num_threads=1, | 268 num_threads=1, |
| 271 directory=True, | 269 directory=True, |
| 272 recursive=False, | 270 recursive=False, |
| 273 force=False, | 271 force=False, |
| 274 output=None, | 272 output=None, |
| 275 ignore_errors=False, | 273 ignore_errors=False, |
| 276 sha1_file=False, | 274 sha1_file=False, |
| 277 verbose=True, | 275 verbose=True, |
| 278 auto_platform=False) | 276 auto_platform=False) |
| 279 expected_calls = [ | 277 expected_calls = [ |
| 280 ('check_call', | 278 ('check_call', |
| 281 ('ls', input_filename)), | 279 ('ls', input_filename)), |
| 282 ('check_call', | 280 ('check_call', |
| 283 ('cp', '-q', input_filename, output_filename))] | 281 ('cp', input_filename, output_filename))] |
| 284 if sys.platform != 'win32': | 282 if sys.platform != 'win32': |
| 285 expected_calls.append( | 283 expected_calls.append( |
| 286 ('check_call', | 284 ('check_call', |
| 287 ('ls', | 285 ('stat', |
| 288 '-L', | |
| 289 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 286 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
| 290 self.assertEqual(self.gsutil.history, expected_calls) | 287 self.assertEqual(self.gsutil.history, expected_calls) |
| 291 self.assertEqual(code, 0) | 288 self.assertEqual(code, 0) |
| 292 | 289 |
| 293 | 290 |
| 294 if __name__ == '__main__': | 291 if __name__ == '__main__': |
| 295 unittest.main() | 292 unittest.main() |
| OLD | NEW |