| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 # pylint: disable=W0212 | 5 # pylint: disable=W0212 |
| 6 | 6 |
| 7 """Unit tests for download_from_google_storage.py.""" | 7 """Unit tests for download_from_google_storage.py.""" |
| 8 | 8 |
| 9 import optparse | 9 import optparse |
| 10 import os | 10 import os |
| (...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 166 | 166 |
| 167 def test_download_worker_single_file(self): | 167 def test_download_worker_single_file(self): |
| 168 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 168 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
| 169 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 169 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
| 170 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 170 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
| 171 self.queue.put((sha1_hash, output_filename)) | 171 self.queue.put((sha1_hash, output_filename)) |
| 172 self.queue.put((None, None)) | 172 self.queue.put((None, None)) |
| 173 stdout_queue = Queue.Queue() | 173 stdout_queue = Queue.Queue() |
| 174 download_from_google_storage._downloader_worker_thread( | 174 download_from_google_storage._downloader_worker_thread( |
| 175 0, self.queue, False, self.base_url, self.gsutil, | 175 0, self.queue, False, self.base_url, self.gsutil, |
| 176 stdout_queue, self.ret_codes) | 176 stdout_queue, self.ret_codes, True) |
| 177 expected_calls = [ | 177 expected_calls = [ |
| 178 ('check_call', | 178 ('check_call', |
| 179 ('ls', input_filename)), | 179 ('ls', input_filename)), |
| 180 ('check_call', | 180 ('check_call', |
| 181 ('cp', '-q', input_filename, output_filename))] | 181 ('cp', '-q', input_filename, output_filename))] |
| 182 if sys.platform != 'win32': | 182 if sys.platform != 'win32': |
| 183 expected_calls.append( | 183 expected_calls.append( |
| 184 ('check_call', | 184 ('check_call', |
| 185 ('ls', | 185 ('ls', |
| 186 '-L', | 186 '-L', |
| 187 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 187 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
| 188 expected_output = [ | 188 expected_output = [ |
| 189 '0> Downloading %s...' % output_filename] | 189 '0> Downloading %s...' % output_filename] |
| 190 expected_ret_codes = [] | 190 expected_ret_codes = [] |
| 191 self.assertEqual(list(stdout_queue.queue), expected_output) | 191 self.assertEqual(list(stdout_queue.queue), expected_output) |
| 192 self.assertEqual(self.gsutil.history, expected_calls) | 192 self.assertEqual(self.gsutil.history, expected_calls) |
| 193 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 193 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
| 194 | 194 |
| 195 def test_download_worker_skips_file(self): | 195 def test_download_worker_skips_file(self): |
| 196 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' | 196 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' |
| 197 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') | 197 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') |
| 198 self.queue.put((sha1_hash, output_filename)) | 198 self.queue.put((sha1_hash, output_filename)) |
| 199 self.queue.put((None, None)) | 199 self.queue.put((None, None)) |
| 200 stdout_queue = Queue.Queue() | 200 stdout_queue = Queue.Queue() |
| 201 download_from_google_storage._downloader_worker_thread( | 201 download_from_google_storage._downloader_worker_thread( |
| 202 0, self.queue, False, self.base_url, self.gsutil, | 202 0, self.queue, False, self.base_url, self.gsutil, |
| 203 stdout_queue, self.ret_codes) | 203 stdout_queue, self.ret_codes, True) |
| 204 expected_output = [ | 204 expected_output = [ |
| 205 '0> File %s exists and SHA1 matches. Skipping.' % output_filename | 205 '0> File %s exists and SHA1 matches. Skipping.' % output_filename |
| 206 ] | 206 ] |
| 207 self.assertEqual(list(stdout_queue.queue), expected_output) | 207 self.assertEqual(list(stdout_queue.queue), expected_output) |
| 208 self.assertEqual(self.gsutil.history, []) | 208 self.assertEqual(self.gsutil.history, []) |
| 209 | 209 |
| 210 def test_download_worker_skips_not_found_file(self): | 210 def test_download_worker_skips_not_found_file(self): |
| 211 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 211 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
| 212 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 212 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
| 213 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 213 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
| 214 self.queue.put((sha1_hash, output_filename)) | 214 self.queue.put((sha1_hash, output_filename)) |
| 215 self.queue.put((None, None)) | 215 self.queue.put((None, None)) |
| 216 stdout_queue = Queue.Queue() | 216 stdout_queue = Queue.Queue() |
| 217 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. | 217 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. |
| 218 download_from_google_storage._downloader_worker_thread( | 218 download_from_google_storage._downloader_worker_thread( |
| 219 0, self.queue, False, self.base_url, self.gsutil, | 219 0, self.queue, False, self.base_url, self.gsutil, |
| 220 stdout_queue, self.ret_codes) | 220 stdout_queue, self.ret_codes, True) |
| 221 expected_output = [ | 221 expected_output = [ |
| 222 '0> File %s for %s does not exist, skipping.' % ( | 222 '0> File %s for %s does not exist, skipping.' % ( |
| 223 input_filename, output_filename), | 223 input_filename, output_filename), |
| 224 ] | 224 ] |
| 225 expected_calls = [ | 225 expected_calls = [ |
| 226 ('check_call', | 226 ('check_call', |
| 227 ('ls', input_filename)) | 227 ('ls', input_filename)) |
| 228 ] | 228 ] |
| 229 expected_ret_codes = [ | 229 expected_ret_codes = [ |
| 230 (1, 'File %s for %s does not exist.' % ( | 230 (1, 'File %s for %s does not exist.' % ( |
| (...skipping 12 matching lines...) Expand all Loading... |
| 243 code = download_from_google_storage.download_from_google_storage( | 243 code = download_from_google_storage.download_from_google_storage( |
| 244 input_filename=sha1_hash, | 244 input_filename=sha1_hash, |
| 245 base_url=self.base_url, | 245 base_url=self.base_url, |
| 246 gsutil=self.gsutil, | 246 gsutil=self.gsutil, |
| 247 num_threads=1, | 247 num_threads=1, |
| 248 directory=False, | 248 directory=False, |
| 249 recursive=False, | 249 recursive=False, |
| 250 force=True, | 250 force=True, |
| 251 output=output_filename, | 251 output=output_filename, |
| 252 ignore_errors=False, | 252 ignore_errors=False, |
| 253 sha1_file=False) | 253 sha1_file=False, |
| 254 verbose=True) |
| 254 expected_calls = [ | 255 expected_calls = [ |
| 255 ('check_call', | 256 ('check_call', |
| 256 ('ls', input_filename)), | 257 ('ls', input_filename)), |
| 257 ('check_call', | 258 ('check_call', |
| 258 ('cp', '-q', input_filename, output_filename)) | 259 ('cp', '-q', input_filename, output_filename)) |
| 259 ] | 260 ] |
| 260 if sys.platform != 'win32': | 261 if sys.platform != 'win32': |
| 261 expected_calls.append( | 262 expected_calls.append( |
| 262 ('check_call', | 263 ('check_call', |
| 263 ('ls', | 264 ('ls', |
| 264 '-L', | 265 '-L', |
| 265 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 266 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
| 266 self.assertEqual(self.gsutil.history, expected_calls) | 267 self.assertEqual(self.gsutil.history, expected_calls) |
| 267 self.assertEqual(code, 101) | 268 self.assertEqual(code, 101) |
| 268 | 269 |
| 269 def test_download_directory_no_recursive_non_force(self): | 270 def test_download_directory_no_recursive_non_force(self): |
| 270 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 271 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
| 271 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 272 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
| 272 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 273 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
| 273 code = download_from_google_storage.download_from_google_storage( | 274 code = download_from_google_storage.download_from_google_storage( |
| 274 input_filename=self.base_path, | 275 input_filename=self.base_path, |
| 275 base_url=self.base_url, | 276 base_url=self.base_url, |
| 276 gsutil=self.gsutil, | 277 gsutil=self.gsutil, |
| 277 num_threads=1, | 278 num_threads=1, |
| 278 directory=True, | 279 directory=True, |
| 279 recursive=False, | 280 recursive=False, |
| 280 force=False, | 281 force=False, |
| 281 output=None, | 282 output=None, |
| 282 ignore_errors=False, | 283 ignore_errors=False, |
| 283 sha1_file=False) | 284 sha1_file=False, |
| 285 verbose=True) |
| 284 expected_calls = [ | 286 expected_calls = [ |
| 285 ('check_call', | 287 ('check_call', |
| 286 ('ls', input_filename)), | 288 ('ls', input_filename)), |
| 287 ('check_call', | 289 ('check_call', |
| 288 ('cp', '-q', input_filename, output_filename))] | 290 ('cp', '-q', input_filename, output_filename))] |
| 289 if sys.platform != 'win32': | 291 if sys.platform != 'win32': |
| 290 expected_calls.append( | 292 expected_calls.append( |
| 291 ('check_call', | 293 ('check_call', |
| 292 ('ls', | 294 ('ls', |
| 293 '-L', | 295 '-L', |
| 294 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 296 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
| 295 self.assertEqual(self.gsutil.history, expected_calls) | 297 self.assertEqual(self.gsutil.history, expected_calls) |
| 296 self.assertEqual(code, 0) | 298 self.assertEqual(code, 0) |
| 297 | 299 |
| 298 | 300 |
| 299 if __name__ == '__main__': | 301 if __name__ == '__main__': |
| 300 unittest.main() | 302 unittest.main() |
| OLD | NEW |