OLD | NEW |
1 # Copyright 2010 Google Inc. | 1 # Copyright 2010 Google Inc. |
2 # | 2 # |
3 # Permission is hereby granted, free of charge, to any person obtaining a | 3 # Permission is hereby granted, free of charge, to any person obtaining a |
4 # copy of this software and associated documentation files (the | 4 # copy of this software and associated documentation files (the |
5 # "Software"), to deal in the Software without restriction, including | 5 # "Software"), to deal in the Software without restriction, including |
6 # without limitation the rights to use, copy, modify, merge, publish, dis- | 6 # without limitation the rights to use, copy, modify, merge, publish, dis- |
7 # tribute, sublicense, and/or sell copies of the Software, and to permit | 7 # tribute, sublicense, and/or sell copies of the Software, and to permit |
8 # persons to whom the Software is furnished to do so, subject to the fol- | 8 # persons to whom the Software is furnished to do so, subject to the fol- |
9 # lowing conditions: | 9 # lowing conditions: |
10 # | 10 # |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
58 Proxy class that translates progress callbacks made by | 58 Proxy class that translates progress callbacks made by |
59 boto.s3.Key.get_file(), taking into account that we're resuming | 59 boto.s3.Key.get_file(), taking into account that we're resuming |
60 a download. | 60 a download. |
61 """ | 61 """ |
62 def __init__(self, proxied_cb, download_start_point): | 62 def __init__(self, proxied_cb, download_start_point): |
63 self.proxied_cb = proxied_cb | 63 self.proxied_cb = proxied_cb |
64 self.download_start_point = download_start_point | 64 self.download_start_point = download_start_point |
65 | 65 |
66 def call(self, total_bytes_uploaded, total_size): | 66 def call(self, total_bytes_uploaded, total_size): |
67 self.proxied_cb(self.download_start_point + total_bytes_uploaded, | 67 self.proxied_cb(self.download_start_point + total_bytes_uploaded, |
68 self.download_start_point + total_size) | 68 total_size) |
69 | 69 |
70 | 70 |
71 def get_cur_file_size(fp, position_to_eof=False): | 71 def get_cur_file_size(fp, position_to_eof=False): |
72 """ | 72 """ |
73 Returns size of file, optionally leaving fp positioned at EOF. | 73 Returns size of file, optionally leaving fp positioned at EOF. |
74 """ | 74 """ |
75 if not position_to_eof: | 75 if not position_to_eof: |
76 cur_pos = fp.tell() | 76 cur_pos = fp.tell() |
77 fp.seek(0, os.SEEK_END) | 77 fp.seek(0, os.SEEK_END) |
78 cur_file_size = fp.tell() | 78 cur_file_size = fp.tell() |
(...skipping 208 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
287 torrent, version_id) | 287 torrent, version_id) |
288 # Download succceded, so remove the tracker file (if have one). | 288 # Download succceded, so remove the tracker file (if have one). |
289 self._remove_tracker_file() | 289 self._remove_tracker_file() |
290 self._check_final_md5(key, fp.name) | 290 self._check_final_md5(key, fp.name) |
291 if debug >= 1: | 291 if debug >= 1: |
292 print 'Resumable download complete.' | 292 print 'Resumable download complete.' |
293 return | 293 return |
294 except self.RETRYABLE_EXCEPTIONS, e: | 294 except self.RETRYABLE_EXCEPTIONS, e: |
295 if debug >= 1: | 295 if debug >= 1: |
296 print('Caught exception (%s)' % e.__repr__()) | 296 print('Caught exception (%s)' % e.__repr__()) |
| 297 if isinstance(e, IOError) and e.errno == errno.EPIPE: |
| 298 # Broken pipe error causes httplib to immediately |
| 299 # close the socket (http://bugs.python.org/issue5542), |
| 300 # so we need to close and reopen the key before resuming |
| 301 # the download. |
| 302 key.get_file(fp, headers, cb, num_cb, torrent, version_id, |
| 303 override_num_retries=0) |
297 except ResumableDownloadException, e: | 304 except ResumableDownloadException, e: |
298 if e.disposition == ResumableTransferDisposition.ABORT: | 305 if (e.disposition == |
| 306 ResumableTransferDisposition.ABORT_CUR_PROCESS): |
299 if debug >= 1: | 307 if debug >= 1: |
300 print('Caught non-retryable ResumableDownloadException ' | 308 print('Caught non-retryable ResumableDownloadException ' |
301 '(%s)' % e.message) | 309 '(%s)' % e.message) |
302 raise | 310 raise |
| 311 elif (e.disposition == |
| 312 ResumableTransferDisposition.ABORT): |
| 313 if debug >= 1: |
| 314 print('Caught non-retryable ResumableDownloadException ' |
| 315 '(%s); aborting and removing tracker file' % |
| 316 e.message) |
| 317 self._remove_tracker_file() |
| 318 raise |
303 else: | 319 else: |
304 if debug >= 1: | 320 if debug >= 1: |
305 print('Caught ResumableDownloadException (%s) - will ' | 321 print('Caught ResumableDownloadException (%s) - will ' |
306 'retry' % e.message) | 322 'retry' % e.message) |
307 | 323 |
308 # At this point we had a re-tryable failure; see if made progress. | 324 # At this point we had a re-tryable failure; see if made progress. |
309 if get_cur_file_size(fp) > had_file_bytes_before_attempt: | 325 if get_cur_file_size(fp) > had_file_bytes_before_attempt: |
310 progress_less_iterations = 0 | 326 progress_less_iterations = 0 |
311 else: | 327 else: |
312 progress_less_iterations += 1 | 328 progress_less_iterations += 1 |
313 | 329 |
314 if progress_less_iterations > self.num_retries: | 330 if progress_less_iterations > self.num_retries: |
315 # Don't retry any longer in the current process. | 331 # Don't retry any longer in the current process. |
316 raise ResumableDownloadException( | 332 raise ResumableDownloadException( |
317 'Too many resumable download attempts failed without ' | 333 'Too many resumable download attempts failed without ' |
318 'progress. You might try this download again later', | 334 'progress. You might try this download again later', |
319 ResumableTransferDisposition.ABORT) | 335 ResumableTransferDisposition.ABORT_CUR_PROCESS) |
320 | 336 |
321 # Close the key, in case a previous download died partway | 337 # Close the key, in case a previous download died partway |
322 # through and left data in the underlying key HTTP buffer. | 338 # through and left data in the underlying key HTTP buffer. |
323 key.close() | 339 # Do this within a try/except block in case the connection is |
| 340 # closed (since key.close() attempts to do a final read, in which |
| 341 # case this read attempt would get an IncompleteRead exception, |
| 342 # which we can safely ignore. |
| 343 try: |
| 344 key.close() |
| 345 except httplib.IncompleteRead: |
| 346 pass |
324 | 347 |
325 sleep_time_secs = 2**progress_less_iterations | 348 sleep_time_secs = 2**progress_less_iterations |
326 if debug >= 1: | 349 if debug >= 1: |
327 print('Got retryable failure (%d progress-less in a row).\n' | 350 print('Got retryable failure (%d progress-less in a row).\n' |
328 'Sleeping %d seconds before re-trying' % | 351 'Sleeping %d seconds before re-trying' % |
329 (progress_less_iterations, sleep_time_secs)) | 352 (progress_less_iterations, sleep_time_secs)) |
330 time.sleep(sleep_time_secs) | 353 time.sleep(sleep_time_secs) |
OLD | NEW |