OLD | NEW |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import base64 | 5 import base64 |
6 import logging | 6 import logging |
7 import posixpath | 7 import posixpath |
8 import time | 8 import time |
9 | 9 |
10 from appengine_wrappers import urlfetch | 10 from appengine_wrappers import urlfetch |
(...skipping 24 matching lines...) Expand all Loading... |
35 """ | 35 """ |
36 def __init__(self, base_path=None): | 36 def __init__(self, base_path=None): |
37 assert base_path is None or not base_path.endswith('/'), base_path | 37 assert base_path is None or not base_path.endswith('/'), base_path |
38 self._base_path = base_path | 38 self._base_path = base_path |
39 self._retries_left = _MAX_RETRIES | 39 self._retries_left = _MAX_RETRIES |
40 | 40 |
41 def Fetch(self, url, username=None, password=None, access_token=None): | 41 def Fetch(self, url, username=None, password=None, access_token=None): |
42 """Fetches a file synchronously. | 42 """Fetches a file synchronously. |
43 """ | 43 """ |
44 return urlfetch.fetch(self._FromBasePath(url), | 44 return urlfetch.fetch(self._FromBasePath(url), |
45 deadline=20, | 45 deadline=40, |
46 headers=_MakeHeaders(username, | 46 headers=_MakeHeaders(username, |
47 password, | 47 password, |
48 access_token)) | 48 access_token)) |
49 | 49 |
50 def FetchAsync(self, url, username=None, password=None, access_token=None): | 50 def FetchAsync(self, url, username=None, password=None, access_token=None): |
51 """Fetches a file asynchronously, and returns a Future with the result. | 51 """Fetches a file asynchronously, and returns a Future with the result. |
52 """ | 52 """ |
53 def process_result(result): | 53 def process_result(result): |
54 if result.status_code == 429: | 54 if result.status_code == 429: |
55 if self._retries_left == 0: | 55 if self._retries_left == 0: |
56 logging.error('Still throttled. Giving up.') | 56 logging.error('Still throttled. Giving up.') |
57 return result | 57 return result |
58 self._retries_left -= 1 | 58 self._retries_left -= 1 |
59 logging.info('Throttled. Trying again in %s seconds.' % | 59 logging.info('Throttled. Trying again in %s seconds.' % |
60 _RETRY_DELAY_SECONDS) | 60 _RETRY_DELAY_SECONDS) |
61 time.sleep(_RETRY_DELAY_SECONDS) | 61 time.sleep(_RETRY_DELAY_SECONDS) |
62 return self.FetchAsync(url, username, password, access_token).Get() | 62 return self.FetchAsync(url, username, password, access_token).Get() |
63 return result | 63 return result |
64 | 64 |
65 rpc = urlfetch.create_rpc(deadline=20) | 65 rpc = urlfetch.create_rpc(deadline=40) |
66 urlfetch.make_fetch_call(rpc, | 66 urlfetch.make_fetch_call(rpc, |
67 self._FromBasePath(url), | 67 self._FromBasePath(url), |
68 headers=_MakeHeaders(username, | 68 headers=_MakeHeaders(username, |
69 password, | 69 password, |
70 access_token)) | 70 access_token)) |
71 return Future(callback=lambda: process_result(rpc.get_result())) | 71 return Future(callback=lambda: process_result(rpc.get_result())) |
72 | 72 |
73 def _FromBasePath(self, url): | 73 def _FromBasePath(self, url): |
74 assert not url.startswith('/'), url | 74 assert not url.startswith('/'), url |
75 if self._base_path is not None: | 75 if self._base_path is not None: |
76 url = posixpath.join(self._base_path, url) if url else self._base_path | 76 url = posixpath.join(self._base_path, url) if url else self._base_path |
77 return url | 77 return url |
OLD | NEW |