OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 # pylint: disable=W0212 | 5 # pylint: disable=W0212 |
6 | 6 |
7 """Unit tests for download_from_google_storage.py.""" | 7 """Unit tests for download_from_google_storage.py.""" |
8 | 8 |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
166 | 166 |
167 def test_download_worker_single_file(self): | 167 def test_download_worker_single_file(self): |
168 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 168 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
169 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 169 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
170 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 170 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
171 self.queue.put((sha1_hash, output_filename)) | 171 self.queue.put((sha1_hash, output_filename)) |
172 self.queue.put((None, None)) | 172 self.queue.put((None, None)) |
173 stdout_queue = Queue.Queue() | 173 stdout_queue = Queue.Queue() |
174 download_from_google_storage._downloader_worker_thread( | 174 download_from_google_storage._downloader_worker_thread( |
175 0, self.queue, False, self.base_url, self.gsutil, | 175 0, self.queue, False, self.base_url, self.gsutil, |
176 stdout_queue, self.ret_codes) | 176 stdout_queue, self.ret_codes, True) |
177 expected_calls = [ | 177 expected_calls = [ |
178 ('check_call', | 178 ('check_call', |
179 ('ls', input_filename)), | 179 ('ls', input_filename)), |
180 ('check_call', | 180 ('check_call', |
181 ('cp', '-q', input_filename, output_filename))] | 181 ('cp', '-q', input_filename, output_filename))] |
182 if sys.platform != 'win32': | 182 if sys.platform != 'win32': |
183 expected_calls.append( | 183 expected_calls.append( |
184 ('check_call', | 184 ('check_call', |
185 ('ls', | 185 ('ls', |
186 '-L', | 186 '-L', |
187 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 187 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
188 expected_output = [ | 188 expected_output = [ |
189 '0> Downloading %s...' % output_filename] | 189 '0> Downloading %s...' % output_filename] |
190 expected_ret_codes = [] | 190 expected_ret_codes = [] |
191 self.assertEqual(list(stdout_queue.queue), expected_output) | 191 self.assertEqual(list(stdout_queue.queue), expected_output) |
192 self.assertEqual(self.gsutil.history, expected_calls) | 192 self.assertEqual(self.gsutil.history, expected_calls) |
193 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) | 193 self.assertEqual(list(self.ret_codes.queue), expected_ret_codes) |
194 | 194 |
195 def test_download_worker_skips_file(self): | 195 def test_download_worker_skips_file(self): |
196 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' | 196 sha1_hash = 'e6c4fbd4fe7607f3e6ebf68b2ea4ef694da7b4fe' |
197 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') | 197 output_filename = os.path.join(self.base_path, 'rootfolder_text.txt') |
198 self.queue.put((sha1_hash, output_filename)) | 198 self.queue.put((sha1_hash, output_filename)) |
199 self.queue.put((None, None)) | 199 self.queue.put((None, None)) |
200 stdout_queue = Queue.Queue() | 200 stdout_queue = Queue.Queue() |
201 download_from_google_storage._downloader_worker_thread( | 201 download_from_google_storage._downloader_worker_thread( |
202 0, self.queue, False, self.base_url, self.gsutil, | 202 0, self.queue, False, self.base_url, self.gsutil, |
203 stdout_queue, self.ret_codes) | 203 stdout_queue, self.ret_codes, True) |
204 expected_output = [ | 204 expected_output = [ |
205 '0> File %s exists and SHA1 matches. Skipping.' % output_filename | 205 '0> File %s exists and SHA1 matches. Skipping.' % output_filename |
206 ] | 206 ] |
207 self.assertEqual(list(stdout_queue.queue), expected_output) | 207 self.assertEqual(list(stdout_queue.queue), expected_output) |
208 self.assertEqual(self.gsutil.history, []) | 208 self.assertEqual(self.gsutil.history, []) |
209 | 209 |
210 def test_download_worker_skips_not_found_file(self): | 210 def test_download_worker_skips_not_found_file(self): |
211 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 211 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
212 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 212 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
213 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 213 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
214 self.queue.put((sha1_hash, output_filename)) | 214 self.queue.put((sha1_hash, output_filename)) |
215 self.queue.put((None, None)) | 215 self.queue.put((None, None)) |
216 stdout_queue = Queue.Queue() | 216 stdout_queue = Queue.Queue() |
217 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. | 217 self.gsutil.add_expected(1, '', '') # Return error when 'ls' is called. |
218 download_from_google_storage._downloader_worker_thread( | 218 download_from_google_storage._downloader_worker_thread( |
219 0, self.queue, False, self.base_url, self.gsutil, | 219 0, self.queue, False, self.base_url, self.gsutil, |
220 stdout_queue, self.ret_codes) | 220 stdout_queue, self.ret_codes, True) |
221 expected_output = [ | 221 expected_output = [ |
222 '0> File %s for %s does not exist, skipping.' % ( | 222 '0> File %s for %s does not exist, skipping.' % ( |
223 input_filename, output_filename), | 223 input_filename, output_filename), |
224 ] | 224 ] |
225 expected_calls = [ | 225 expected_calls = [ |
226 ('check_call', | 226 ('check_call', |
227 ('ls', input_filename)) | 227 ('ls', input_filename)) |
228 ] | 228 ] |
229 expected_ret_codes = [ | 229 expected_ret_codes = [ |
230 (1, 'File %s for %s does not exist.' % ( | 230 (1, 'File %s for %s does not exist.' % ( |
(...skipping 12 matching lines...) Expand all Loading... |
243 code = download_from_google_storage.download_from_google_storage( | 243 code = download_from_google_storage.download_from_google_storage( |
244 input_filename=sha1_hash, | 244 input_filename=sha1_hash, |
245 base_url=self.base_url, | 245 base_url=self.base_url, |
246 gsutil=self.gsutil, | 246 gsutil=self.gsutil, |
247 num_threads=1, | 247 num_threads=1, |
248 directory=False, | 248 directory=False, |
249 recursive=False, | 249 recursive=False, |
250 force=True, | 250 force=True, |
251 output=output_filename, | 251 output=output_filename, |
252 ignore_errors=False, | 252 ignore_errors=False, |
253 sha1_file=False) | 253 sha1_file=False, |
| 254 verbose=True) |
254 expected_calls = [ | 255 expected_calls = [ |
255 ('check_call', | 256 ('check_call', |
256 ('ls', input_filename)), | 257 ('ls', input_filename)), |
257 ('check_call', | 258 ('check_call', |
258 ('cp', '-q', input_filename, output_filename)) | 259 ('cp', '-q', input_filename, output_filename)) |
259 ] | 260 ] |
260 if sys.platform != 'win32': | 261 if sys.platform != 'win32': |
261 expected_calls.append( | 262 expected_calls.append( |
262 ('check_call', | 263 ('check_call', |
263 ('ls', | 264 ('ls', |
264 '-L', | 265 '-L', |
265 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 266 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
266 self.assertEqual(self.gsutil.history, expected_calls) | 267 self.assertEqual(self.gsutil.history, expected_calls) |
267 self.assertEqual(code, 101) | 268 self.assertEqual(code, 101) |
268 | 269 |
269 def test_download_directory_no_recursive_non_force(self): | 270 def test_download_directory_no_recursive_non_force(self): |
270 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' | 271 sha1_hash = '7871c8e24da15bad8b0be2c36edc9dc77e37727f' |
271 input_filename = '%s/%s' % (self.base_url, sha1_hash) | 272 input_filename = '%s/%s' % (self.base_url, sha1_hash) |
272 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') | 273 output_filename = os.path.join(self.base_path, 'uploaded_lorem_ipsum.txt') |
273 code = download_from_google_storage.download_from_google_storage( | 274 code = download_from_google_storage.download_from_google_storage( |
274 input_filename=self.base_path, | 275 input_filename=self.base_path, |
275 base_url=self.base_url, | 276 base_url=self.base_url, |
276 gsutil=self.gsutil, | 277 gsutil=self.gsutil, |
277 num_threads=1, | 278 num_threads=1, |
278 directory=True, | 279 directory=True, |
279 recursive=False, | 280 recursive=False, |
280 force=False, | 281 force=False, |
281 output=None, | 282 output=None, |
282 ignore_errors=False, | 283 ignore_errors=False, |
283 sha1_file=False) | 284 sha1_file=False, |
| 285 verbose=True) |
284 expected_calls = [ | 286 expected_calls = [ |
285 ('check_call', | 287 ('check_call', |
286 ('ls', input_filename)), | 288 ('ls', input_filename)), |
287 ('check_call', | 289 ('check_call', |
288 ('cp', '-q', input_filename, output_filename))] | 290 ('cp', '-q', input_filename, output_filename))] |
289 if sys.platform != 'win32': | 291 if sys.platform != 'win32': |
290 expected_calls.append( | 292 expected_calls.append( |
291 ('check_call', | 293 ('check_call', |
292 ('ls', | 294 ('ls', |
293 '-L', | 295 '-L', |
294 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) | 296 'gs://sometesturl/7871c8e24da15bad8b0be2c36edc9dc77e37727f'))) |
295 self.assertEqual(self.gsutil.history, expected_calls) | 297 self.assertEqual(self.gsutil.history, expected_calls) |
296 self.assertEqual(code, 0) | 298 self.assertEqual(code, 0) |
297 | 299 |
298 | 300 |
299 if __name__ == '__main__': | 301 if __name__ == '__main__': |
300 unittest.main() | 302 unittest.main() |
OLD | NEW |