Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1)

Side by Side Diff: chrome/common/extensions/docs/server2/appengine_url_fetcher.py

Issue 660383002: Docserver: Persist stat cache for versioned file systems (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: change caching strategy, better refresh cycle synchronization Created 6 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import base64 5 import base64
6 import logging 6 import logging
7 import posixpath 7 import posixpath
8 import time 8 import time
9 9
10 from appengine_wrappers import urlfetch 10 from appengine_wrappers import urlfetch
(...skipping 24 matching lines...) Expand all
35 """ 35 """
36 def __init__(self, base_path=None): 36 def __init__(self, base_path=None):
37 assert base_path is None or not base_path.endswith('/'), base_path 37 assert base_path is None or not base_path.endswith('/'), base_path
38 self._base_path = base_path 38 self._base_path = base_path
39 self._retries_left = _MAX_RETRIES 39 self._retries_left = _MAX_RETRIES
40 40
41 def Fetch(self, url, username=None, password=None, access_token=None): 41 def Fetch(self, url, username=None, password=None, access_token=None):
42 """Fetches a file synchronously. 42 """Fetches a file synchronously.
43 """ 43 """
44 return urlfetch.fetch(self._FromBasePath(url), 44 return urlfetch.fetch(self._FromBasePath(url),
45 deadline=20, 45 deadline=40,
46 headers=_MakeHeaders(username, 46 headers=_MakeHeaders(username,
47 password, 47 password,
48 access_token)) 48 access_token))
49 49
50 def FetchAsync(self, url, username=None, password=None, access_token=None): 50 def FetchAsync(self, url, username=None, password=None, access_token=None):
51 """Fetches a file asynchronously, and returns a Future with the result. 51 """Fetches a file asynchronously, and returns a Future with the result.
52 """ 52 """
53 def process_result(result): 53 def process_result(result):
54 if result.status_code == 429: 54 if result.status_code == 429:
55 if self._retries_left == 0: 55 if self._retries_left == 0:
56 logging.error('Still throttled. Giving up.') 56 logging.error('Still throttled. Giving up.')
57 return result 57 return result
58 self._retries_left -= 1 58 self._retries_left -= 1
59 logging.info('Throttled. Trying again in %s seconds.' % 59 logging.info('Throttled. Trying again in %s seconds.' %
60 _RETRY_DELAY_SECONDS) 60 _RETRY_DELAY_SECONDS)
61 time.sleep(_RETRY_DELAY_SECONDS) 61 time.sleep(_RETRY_DELAY_SECONDS)
62 return self.FetchAsync(url, username, password, access_token).Get() 62 return self.FetchAsync(url, username, password, access_token).Get()
63 return result 63 return result
64 64
65 rpc = urlfetch.create_rpc(deadline=20) 65 rpc = urlfetch.create_rpc(deadline=40)
66 urlfetch.make_fetch_call(rpc, 66 urlfetch.make_fetch_call(rpc,
67 self._FromBasePath(url), 67 self._FromBasePath(url),
68 headers=_MakeHeaders(username, 68 headers=_MakeHeaders(username,
69 password, 69 password,
70 access_token)) 70 access_token))
71 return Future(callback=lambda: process_result(rpc.get_result())) 71 return Future(callback=lambda: process_result(rpc.get_result()))
72 72
73 def _FromBasePath(self, url): 73 def _FromBasePath(self, url):
74 assert not url.startswith('/'), url 74 assert not url.startswith('/'), url
75 if self._base_path is not None: 75 if self._base_path is not None:
76 url = posixpath.join(self._base_path, url) if url else self._base_path 76 url = posixpath.join(self._base_path, url) if url else self._base_path
77 return url 77 return url
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698