Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(182)

Side by Side Diff: infra_libs/httplib2_utils.py

Issue 2128413002: Make RetriableHttp to be composable (Closed) Base URL: https://chromium.googlesource.com/infra/infra.git@master
Patch Set: Fix Created 4 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | infra_libs/test/httplib2_utils_test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2015 The Chromium Authors. All rights reserved. 1 # Copyright 2015 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import collections 5 import collections
6 import copy 6 import copy
7 import json 7 import json
8 import logging 8 import logging
9 import os 9 import os
10 import re 10 import re
(...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after
155 credentials_filename, 155 credentials_filename,
156 scope=scope, 156 scope=scope,
157 service_accounts_creds_root=service_accounts_creds_root) 157 service_accounts_creds_root=service_accounts_creds_root)
158 158
159 if http_identifier: 159 if http_identifier:
160 http = InstrumentedHttp(http_identifier, timeout=timeout) 160 http = InstrumentedHttp(http_identifier, timeout=timeout)
161 else: 161 else:
162 http = httplib2.Http(timeout=timeout) 162 http = httplib2.Http(timeout=timeout)
163 return creds.authorize(http) 163 return creds.authorize(http)
164 164
165 class RetriableHttp(httplib2.Http): 165 class RetriableHttp(object):
166 """A httplib2.Http object that retries on failure.""" 166 """A httplib2.Http object that retries on failure."""
167 167
168 def __init__(self, max_tries=5, retrying_statuses_fn=None, **kwargs): 168 def __init__(self, http, max_tries=5, retrying_statuses_fn=None):
169 """ 169 """
170 Args: 170 Args:
171 http_obj: an httplib2.Http instance 171 http: an httplib2.Http instance
172 max_tries: a number of maximum tries 172 max_tries: a number of maximum tries
173 retrying_statuses_fn: a function that returns True if a given status 173 retrying_statuses_fn: a function that returns True if a given status
174 should be retried 174 should be retried
175 """ 175 """
176 super(RetriableHttp, self).__init__(**kwargs) 176 self._http = http
177 self._max_tries = max_tries 177 self._max_tries = max_tries
178 self._retrying_statuses_fn = retrying_statuses_fn or \ 178 self._retrying_statuses_fn = retrying_statuses_fn or \
179 set(range(500,599)).__contains__ 179 set(range(500,599)).__contains__
180 180
181 def request(self, uri, method='GET', body=None, *args, **kwargs): 181 def request(self, uri, method='GET', body=None, *args, **kwargs):
182 for i in range(1, self._max_tries + 1): 182 for i in range(1, self._max_tries + 1):
183 try: 183 try:
184 response, content = super(RetriableHttp, self).request( 184 response, content = self._http.request(uri, method, body, *args,
185 uri, method, body, *args, **kwargs) 185 **kwargs)
186 186
187 if self._retrying_statuses_fn(response.status): 187 if self._retrying_statuses_fn(response.status):
188 logging.info('RetriableHttp: attempt %d receiving status %d, %s', 188 logging.info('RetriableHttp: attempt %d receiving status %d, %s',
189 i, response.status, 189 i, response.status,
190 'final attempt' if i == self._max_tries else \ 190 'final attempt' if i == self._max_tries else \
191 'will retry') 191 'will retry')
192 else: 192 else:
193 break 193 break
194 except (ValueError, errors.Error, 194 except (ValueError, errors.Error,
195 socket.timeout, socket.error, socket.herror, socket.gaierror, 195 socket.timeout, socket.error, socket.herror, socket.gaierror,
196 httplib2.HttpLib2Error) as error: 196 httplib2.HttpLib2Error) as error:
197 logging.info('RetriableHttp: attempt %d received exception: %s, %s', 197 logging.info('RetriableHttp: attempt %d received exception: %s, %s',
198 i, error, 'final attempt' if i == self._max_tries else \ 198 i, error, 'final attempt' if i == self._max_tries else \
199 'will retry') 199 'will retry')
200 if i == self._max_tries: 200 if i == self._max_tries:
201 raise 201 raise
202 202
203 return response, content 203 return response, content
204 204
205 def __getattr__(self, name):
206 return getattr(self._http, name)
207
208 def __setattr__(self, name, value):
209 if name in ('_http', '_max_tries', '_retrying_statuses_fn'):
210 self.__dict__[name] = value
211 else:
212 setattr(self._http, name, value)
205 213
206 class InstrumentedHttp(httplib2.Http): 214 class InstrumentedHttp(httplib2.Http):
207 """A httplib2.Http object that reports ts_mon metrics about its requests.""" 215 """A httplib2.Http object that reports ts_mon metrics about its requests."""
208 216
209 def __init__(self, name, time_fn=time.time, timeout=DEFAULT_TIMEOUT, 217 def __init__(self, name, time_fn=time.time, timeout=DEFAULT_TIMEOUT,
210 **kwargs): 218 **kwargs):
211 """ 219 """
212 Args: 220 Args:
213 name: An identifier for the HTTP requests made by this object. 221 name: An identifier for the HTTP requests made by this object.
214 time_fn: Function returning the current time in seconds. Use for testing 222 time_fn: Function returning the current time in seconds. Use for testing
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after
298 self.requests_made.append(self.HttpCall(uri, method, body, headers)) 306 self.requests_made.append(self.HttpCall(uri, method, body, headers))
299 headers = None 307 headers = None
300 body = None 308 body = None
301 for candidate in self._uris: 309 for candidate in self._uris:
302 if candidate[0].match(uri): 310 if candidate[0].match(uri):
303 _, headers, body = candidate 311 _, headers, body = candidate
304 break 312 break
305 if not headers: 313 if not headers:
306 raise AssertionError("Unexpected request to %s" % uri) 314 raise AssertionError("Unexpected request to %s" % uri)
307 return httplib2.Response(headers), body 315 return httplib2.Response(headers), body
OLDNEW
« no previous file with comments | « no previous file | infra_libs/test/httplib2_utils_test.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698