OLD | NEW |
| (Empty) |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | |
2 # Use of this source code is governed by a BSD-style license that can be | |
3 # found in the LICENSE file. | |
4 | |
5 """Classes and functions for generic network communication over HTTP.""" | |
6 | |
7 import cookielib | |
8 import cStringIO as StringIO | |
9 import httplib | |
10 import itertools | |
11 import logging | |
12 import math | |
13 import os | |
14 import random | |
15 import re | |
16 import socket | |
17 import ssl | |
18 import threading | |
19 import time | |
20 import urllib | |
21 import urllib2 | |
22 import urlparse | |
23 | |
24 from third_party import requests | |
25 from third_party.requests import adapters | |
26 from third_party.rietveld import upload | |
27 | |
28 from utils import zip_package | |
29 | |
30 # Hack out upload logging.info() | |
31 upload.logging = logging.getLogger('upload') | |
32 # Mac pylint choke on this line. | |
33 upload.logging.setLevel(logging.WARNING) # pylint: disable=E1103 | |
34 | |
35 | |
36 # TODO(vadimsh): Remove this once we don't have to support python 2.6 anymore. | |
37 def monkey_patch_httplib(): | |
38 """Patch httplib.HTTPConnection to have '_tunnel_host' attribute. | |
39 | |
40 'requests' library (>= v2) accesses 'HTTPConnection._tunnel_host' attribute | |
41 added only in python 2.6.3. This function patches HTTPConnection to have it | |
42 on python 2.6.2 as well. | |
43 """ | |
44 conn = httplib.HTTPConnection('example.com') | |
45 if not hasattr(conn, '_tunnel_host'): | |
46 httplib.HTTPConnection._tunnel_host = None | |
47 monkey_patch_httplib() | |
48 | |
49 | |
50 # Big switch that controls what API to use to make HTTP requests. | |
51 # It's temporary here to simplify benchmarking of old vs new implementation. | |
52 USE_REQUESTS_LIB = True | |
53 | |
54 # The name of the key to store the count of url attempts. | |
55 COUNT_KEY = 'UrlOpenAttempt' | |
56 | |
57 # Default maximum number of attempts to trying opening a url before aborting. | |
58 URL_OPEN_MAX_ATTEMPTS = 30 | |
59 | |
60 # Default timeout when retrying. | |
61 URL_OPEN_TIMEOUT = 6*60. | |
62 | |
63 # Content type for url encoded POST body. | |
64 URL_ENCODED_FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded' | |
65 | |
66 # Default content type for POST body. | |
67 DEFAULT_CONTENT_TYPE = URL_ENCODED_FORM_CONTENT_TYPE | |
68 | |
69 # Content type -> function that encodes a request body. | |
70 CONTENT_ENCODERS = { | |
71 URL_ENCODED_FORM_CONTENT_TYPE: urllib.urlencode, | |
72 } | |
73 | |
74 # File to use to store all auth cookies. | |
75 COOKIE_FILE = os.path.join(os.path.expanduser('~'), '.isolated_cookies') | |
76 | |
77 # Google Storage URL regular expression. | |
78 GS_STORAGE_HOST_URL_RE = re.compile(r'https://.*\.storage\.googleapis\.com') | |
79 | |
80 | |
81 # Global (for now) map: server URL (http://example.com) -> HttpService instance. | |
82 # Used by get_http_service to cache HttpService instances. | |
83 _http_services = {} | |
84 _http_services_lock = threading.Lock() | |
85 | |
86 # CookieJar reused by all services + lock that protects its instantiation. | |
87 _cookie_jar = None | |
88 _cookie_jar_lock = threading.Lock() | |
89 | |
90 # Path to cacert.pem bundle file reused by all services. | |
91 _ca_certs = None | |
92 _ca_certs_lock = threading.Lock() | |
93 | |
94 | |
95 class NetError(IOError): | |
96 """Generic network related error.""" | |
97 | |
98 def __init__(self, inner_exc=None): | |
99 super(NetError, self).__init__(str(inner_exc or self.__doc__)) | |
100 self.inner_exc = inner_exc | |
101 | |
102 def format(self, verbose=False): | |
103 """Human readable description with detailed information about the error.""" | |
104 out = ['Exception: %s' % (self.inner_exc,)] | |
105 if verbose: | |
106 headers = None | |
107 body = None | |
108 if isinstance(self.inner_exc, urllib2.HTTPError): | |
109 headers = self.inner_exc.hdrs.items() | |
110 body = self.inner_exc.read() | |
111 elif isinstance(self.inner_exc, requests.HTTPError): | |
112 headers = self.inner_exc.response.headers.items() | |
113 body = self.inner_exc.response.content | |
114 if headers or body: | |
115 out.append('----------') | |
116 if headers: | |
117 for header, value in headers: | |
118 if not header.startswith('x-'): | |
119 out.append('%s: %s' % (header.capitalize(), value)) | |
120 out.append('') | |
121 out.append(body or '<empty body>') | |
122 out.append('----------') | |
123 return '\n'.join(out) | |
124 | |
125 | |
126 class TimeoutError(NetError): | |
127 """Timeout while reading HTTP response.""" | |
128 | |
129 | |
130 class ConnectionError(NetError): | |
131 """Failed to connect to the server.""" | |
132 | |
133 | |
134 class HttpError(NetError): | |
135 """Server returned HTTP error code.""" | |
136 | |
137 def __init__(self, code, inner_exc=None): | |
138 super(HttpError, self).__init__(inner_exc) | |
139 self.code = code | |
140 | |
141 | |
142 def url_open(url, **kwargs): | |
143 """Attempts to open the given url multiple times. | |
144 | |
145 |data| can be either: | |
146 - None for a GET request | |
147 - str for pre-encoded data | |
148 - list for data to be encoded | |
149 - dict for data to be encoded | |
150 | |
151 See HttpService.request for a full list of arguments. | |
152 | |
153 Returns HttpResponse object, where the response may be read from, or None | |
154 if it was unable to connect. | |
155 """ | |
156 urlhost, urlpath = split_server_request_url(url) | |
157 service = get_http_service(urlhost) | |
158 return service.request(urlpath, **kwargs) | |
159 | |
160 | |
161 def url_read(url, **kwargs): | |
162 """Attempts to open the given url multiple times and read all data from it. | |
163 | |
164 Accepts same arguments as url_open function. | |
165 | |
166 Returns all data read or None if it was unable to connect or read the data. | |
167 """ | |
168 kwargs['stream'] = False | |
169 response = url_open(url, **kwargs) | |
170 if not response: | |
171 return None | |
172 try: | |
173 return response.read() | |
174 except TimeoutError: | |
175 return None | |
176 | |
177 | |
178 def split_server_request_url(url): | |
179 """Splits the url into scheme+netloc and path+params+query+fragment.""" | |
180 url_parts = list(urlparse.urlparse(url)) | |
181 urlhost = '%s://%s' % (url_parts[0], url_parts[1]) | |
182 urlpath = urlparse.urlunparse(['', ''] + url_parts[2:]) | |
183 return urlhost, urlpath | |
184 | |
185 | |
186 def get_http_service(urlhost): | |
187 """Returns existing or creates new instance of HttpService that can send | |
188 requests to given base urlhost. | |
189 """ | |
190 # Ensure consistency. | |
191 urlhost = str(urlhost).lower().rstrip('/') | |
192 with _http_services_lock: | |
193 service = _http_services.get(urlhost) | |
194 if not service: | |
195 if GS_STORAGE_HOST_URL_RE.match(urlhost): | |
196 # For Google Storage URL create a dumber HttpService that doesn't modify | |
197 # requests with COUNT_KEY (since it breaks a signature) and doesn't try | |
198 # to 'login' into Google Storage (since it's impossible). | |
199 service = HttpService( | |
200 urlhost, | |
201 engine=create_request_engine(None), | |
202 authenticator=None, | |
203 use_count_key=False) | |
204 else: | |
205 # For other URLs (presumably App Engine), create a fancier HttpService | |
206 # with cookies, authentication and COUNT_KEY query parameter in retries. | |
207 cookie_jar = get_cookie_jar() | |
208 service = HttpService( | |
209 urlhost, | |
210 engine=create_request_engine(cookie_jar), | |
211 authenticator=AppEngineAuthenticator(urlhost, cookie_jar), | |
212 use_count_key=True) | |
213 _http_services[urlhost] = service | |
214 return service | |
215 | |
216 | |
217 def create_request_engine(cookie_jar): | |
218 """Returns a new instance of RequestEngine subclass. | |
219 | |
220 |cookie_jar| is an instance of ThreadSafeCookieJar class that holds all | |
221 cookies. It is optional and may be None (in that case cookies are not saved | |
222 on disk). | |
223 """ | |
224 if USE_REQUESTS_LIB: | |
225 return RequestsLibEngine(cookie_jar, get_cacerts_bundle()) | |
226 return Urllib2Engine(cookie_jar) | |
227 | |
228 | |
229 def get_cookie_jar(): | |
230 """Returns global CoookieJar object that stores cookies in the file.""" | |
231 global _cookie_jar | |
232 with _cookie_jar_lock: | |
233 if _cookie_jar is not None: | |
234 return _cookie_jar | |
235 jar = ThreadSafeCookieJar(COOKIE_FILE) | |
236 jar.load() | |
237 _cookie_jar = jar | |
238 return jar | |
239 | |
240 | |
241 def get_cacerts_bundle(): | |
242 """Returns path to a file with CA root certificates bundle.""" | |
243 global _ca_certs | |
244 with _ca_certs_lock: | |
245 if _ca_certs is not None and os.path.exists(_ca_certs): | |
246 return _ca_certs | |
247 _ca_certs = zip_package.extract_resource(requests, 'cacert.pem') | |
248 return _ca_certs | |
249 | |
250 | |
251 class HttpService(object): | |
252 """Base class for a class that provides an API to HTTP based service: | |
253 - Provides 'request' method. | |
254 - Supports automatic request retries. | |
255 - Supports persistent cookies. | |
256 - Thread safe. | |
257 """ | |
258 def __init__(self, urlhost, engine, authenticator=None, use_count_key=True): | |
259 self.urlhost = urlhost | |
260 self.engine = engine | |
261 self.authenticator = authenticator | |
262 self.use_count_key = use_count_key | |
263 | |
264 @staticmethod | |
265 def is_transient_http_error(code, retry_404, retry_50x): | |
266 """Returns True if given HTTP response code is a transient error.""" | |
267 # Google Storage can return this and it should be retried. | |
268 if code == 408: | |
269 return True | |
270 # Retry 404 only if allowed by the caller. | |
271 if code == 404: | |
272 return retry_404 | |
273 # All other 4** errors are fatal. | |
274 if code < 500: | |
275 return False | |
276 # Retry >= 500 error only if allowed by the caller. | |
277 return retry_50x | |
278 | |
279 @staticmethod | |
280 def encode_request_body(body, content_type): | |
281 """Returns request body encoded according to its content type.""" | |
282 # No body or it is already encoded. | |
283 if body is None or isinstance(body, str): | |
284 return body | |
285 # Any body should have content type set. | |
286 assert content_type, 'Request has body, but no content type' | |
287 encoder = CONTENT_ENCODERS.get(content_type) | |
288 assert encoder, ('Unknown content type %s' % content_type) | |
289 return encoder(body) | |
290 | |
291 def request( | |
292 self, | |
293 urlpath, | |
294 data=None, | |
295 content_type=None, | |
296 max_attempts=URL_OPEN_MAX_ATTEMPTS, | |
297 retry_404=False, | |
298 retry_50x=True, | |
299 timeout=URL_OPEN_TIMEOUT, | |
300 read_timeout=None, | |
301 stream=True, | |
302 method=None): | |
303 """Attempts to open the given url multiple times. | |
304 | |
305 |urlpath| is relative to the server root, i.e. '/some/request?param=1'. | |
306 | |
307 |data| can be either: | |
308 - None for a GET request | |
309 - str for pre-encoded data | |
310 - list for data to be form-encoded | |
311 - dict for data to be form-encoded | |
312 | |
313 - Optionally retries HTTP 404 and 50x. | |
314 - Retries up to |max_attempts| times. If None or 0, there's no limit in the | |
315 number of retries. | |
316 - Retries up to |timeout| duration in seconds. If None or 0, there's no | |
317 limit in the time taken to do retries. | |
318 - If both |max_attempts| and |timeout| are None or 0, this functions retries | |
319 indefinitely. | |
320 | |
321 If |method| is given it can be 'GET', 'POST' or 'PUT' and it will be used | |
322 when performing the request. By default it's GET if |data| is None and POST | |
323 if |data| is not None. | |
324 | |
325 If |read_timeout| is not None will configure underlying socket to | |
326 raise TimeoutError exception whenever there's no response from the server | |
327 for more than |read_timeout| seconds. It can happen during any read | |
328 operation so once you pass non-None |read_timeout| be prepared to handle | |
329 these exceptions in subsequent reads from the stream. | |
330 | |
331 Returns a file-like object, where the response may be read from, or None | |
332 if it was unable to connect. If |stream| is False will read whole response | |
333 into memory buffer before returning file-like object that reads from this | |
334 memory buffer. | |
335 """ | |
336 assert urlpath and urlpath[0] == '/', urlpath | |
337 | |
338 if data is not None: | |
339 assert method in (None, 'POST', 'PUT') | |
340 method = method or 'POST' | |
341 content_type = content_type or DEFAULT_CONTENT_TYPE | |
342 body = self.encode_request_body(data, content_type) | |
343 else: | |
344 assert method in (None, 'GET') | |
345 method = method or 'GET' | |
346 body = None | |
347 assert not content_type, 'Can\'t use content_type on GET' | |
348 | |
349 # Prepare request info. | |
350 parsed = urlparse.urlparse('/' + urlpath.lstrip('/')) | |
351 resource_url = urlparse.urljoin(self.urlhost, parsed.path) | |
352 query_params = urlparse.parse_qsl(parsed.query) | |
353 | |
354 # Prepare headers. | |
355 headers = {} | |
356 if body is not None: | |
357 headers['Content-Length'] = len(body) | |
358 if content_type: | |
359 headers['Content-Type'] = content_type | |
360 | |
361 last_error = None | |
362 auth_attempted = False | |
363 | |
364 for attempt in retry_loop(max_attempts, timeout): | |
365 # Log non-first attempt. | |
366 if attempt.attempt: | |
367 logging.warning( | |
368 'Retrying request %s, attempt %d/%d...', | |
369 resource_url, attempt.attempt, max_attempts) | |
370 | |
371 try: | |
372 # Prepare and send a new request. | |
373 request = HttpRequest(method, resource_url, query_params, body, | |
374 headers, read_timeout, stream) | |
375 self.prepare_request(request, attempt.attempt) | |
376 response = self.engine.perform_request(request) | |
377 logging.debug('Request %s succeeded', request.get_full_url()) | |
378 return response | |
379 | |
380 except (ConnectionError, TimeoutError) as e: | |
381 last_error = e | |
382 logging.warning( | |
383 'Unable to open url %s on attempt %d.\n%s', | |
384 request.get_full_url(), attempt.attempt, e.format()) | |
385 continue | |
386 | |
387 except HttpError as e: | |
388 last_error = e | |
389 | |
390 # Access denied -> authenticate. | |
391 if e.code in (401, 403): | |
392 logging.error( | |
393 'Authentication is required for %s on attempt %d.\n%s', | |
394 request.get_full_url(), attempt.attempt, e.format()) | |
395 # Try to authenticate only once. If it doesn't help, then server does | |
396 # not support app engine authentication. | |
397 if not auth_attempted: | |
398 auth_attempted = True | |
399 if self.authenticator and self.authenticator.authenticate(): | |
400 # Success! Run request again immediately. | |
401 attempt.skip_sleep = True | |
402 # Also refresh cookies used by request engine. | |
403 self.engine.reload_cookies() | |
404 continue | |
405 # Authentication attempt was unsuccessful. | |
406 logging.error( | |
407 'Unable to authenticate to %s.\n%s', | |
408 request.get_full_url(), e.format(verbose=True)) | |
409 return None | |
410 | |
411 # Hit a error that can not be retried -> stop retry loop. | |
412 if not self.is_transient_http_error(e.code, retry_404, retry_50x): | |
413 # This HttpError means we reached the server and there was a problem | |
414 # with the request, so don't retry. | |
415 logging.error( | |
416 'Able to connect to %s but an exception was thrown.\n%s', | |
417 request.get_full_url(), e.format(verbose=True)) | |
418 return None | |
419 | |
420 # Retry all other errors. | |
421 logging.warning( | |
422 'Server responded with error on %s on attempt %d.\n%s', | |
423 request.get_full_url(), attempt.attempt, e.format()) | |
424 continue | |
425 | |
426 logging.error( | |
427 'Unable to open given url, %s, after %d attempts.\n%s', | |
428 request.get_full_url(), max_attempts, last_error.format(verbose=True)) | |
429 return None | |
430 | |
431 def prepare_request(self, request, attempt): # pylint: disable=R0201 | |
432 """Modify HttpRequest before sending it by adding COUNT_KEY parameter.""" | |
433 # Add COUNT_KEY only on retries. | |
434 if self.use_count_key and attempt: | |
435 request.params += [(COUNT_KEY, attempt)] | |
436 | |
437 | |
438 class HttpRequest(object): | |
439 """Request to HttpService.""" | |
440 | |
441 def __init__(self, method, url, params, body, headers, timeout, stream): | |
442 """Arguments: | |
443 |method| - HTTP method to use | |
444 |url| - relative URL to the resource, without query parameters | |
445 |params| - list of (key, value) pairs to put into GET parameters | |
446 |body| - encoded body of the request (None or str) | |
447 |headers| - dict with request headers | |
448 |timeout| - socket read timeout (None to disable) | |
449 |stream| - True to stream response from socket | |
450 """ | |
451 self.method = method | |
452 self.url = url | |
453 self.params = params[:] | |
454 self.body = body | |
455 self.headers = headers.copy() | |
456 self.timeout = timeout | |
457 self.stream = stream | |
458 | |
459 def get_full_url(self): | |
460 """Resource URL with url-encoded GET parameters.""" | |
461 if not self.params: | |
462 return self.url | |
463 else: | |
464 return '%s?%s' % (self.url, urllib.urlencode(self.params)) | |
465 | |
466 def make_fake_response(self, content=''): | |
467 """Makes new fake HttpResponse to this request, useful in tests.""" | |
468 return HttpResponse.get_fake_response(content, self.get_full_url()) | |
469 | |
470 | |
471 class HttpResponse(object): | |
472 """Response from HttpService.""" | |
473 | |
474 def __init__(self, stream, url, headers): | |
475 self._stream = stream | |
476 self._url = url | |
477 self._headers = headers | |
478 self._read = 0 | |
479 | |
480 @property | |
481 def content_length(self): | |
482 """Total length to the response or None if not known in advance.""" | |
483 length = self._headers.get('Content-Length') | |
484 return int(length) if length is not None else None | |
485 | |
486 def read(self, size=None): | |
487 """Reads up to |size| bytes from the stream and returns them. | |
488 | |
489 If |size| is None reads all available bytes. | |
490 | |
491 Raises TimeoutError on read timeout. | |
492 """ | |
493 try: | |
494 # cStringIO has a bug: stream.read(None) is not the same as stream.read(). | |
495 data = self._stream.read() if size is None else self._stream.read(size) | |
496 self._read += len(data) | |
497 return data | |
498 except (socket.timeout, ssl.SSLError, requests.Timeout) as e: | |
499 logging.error('Timeout while reading from %s, read %d of %s: %s', | |
500 self._url, self._read, self.content_length, e) | |
501 raise TimeoutError(e) | |
502 | |
503 @classmethod | |
504 def get_fake_response(cls, content, url): | |
505 """Returns HttpResponse with predefined content, useful in tests.""" | |
506 return cls(StringIO.StringIO(content), | |
507 url, {'content-length': len(content)}) | |
508 | |
509 | |
510 class RequestEngine(object): | |
511 """Base class for objects that know how to execute HttpRequests.""" | |
512 | |
513 def perform_request(self, request): | |
514 """Sends a HttpRequest to the server and reads back the response. | |
515 | |
516 Returns HttpResponse. | |
517 | |
518 Raises: | |
519 ConnectionError - failed to establish connection to the server. | |
520 TimeoutError - timeout while connecting or reading response. | |
521 HttpError - server responded with >= 400 error code. | |
522 """ | |
523 raise NotImplementedError() | |
524 | |
525 def reload_cookies(self): | |
526 """Reloads cookies from original cookie jar.""" | |
527 # This method is optional. | |
528 pass | |
529 | |
530 | |
531 class Authenticator(object): | |
532 """Base class for objects that know how to authenticate into http services.""" | |
533 | |
534 def authenticate(self): | |
535 """Authenticates in the app engine service.""" | |
536 raise NotImplementedError() | |
537 | |
538 | |
539 class Urllib2Engine(RequestEngine): | |
540 """Class that knows how to execute HttpRequests via urllib2.""" | |
541 | |
542 def __init__(self, cookie_jar): | |
543 super(Urllib2Engine, self).__init__() | |
544 self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) | |
545 | |
546 def perform_request(self, request): | |
547 try: | |
548 req = self.make_urllib2_request(request) | |
549 if request.timeout: | |
550 resp = self.opener.open(req, timeout=request.timeout) | |
551 else: | |
552 resp = self.opener.open(req) | |
553 return HttpResponse(resp, req.get_full_url(), resp.headers) | |
554 except urllib2.HTTPError as e: | |
555 raise HttpError(e.code, e) | |
556 except (urllib2.URLError, httplib.HTTPException, | |
557 socket.timeout, ssl.SSLError) as e: | |
558 raise ConnectionError(e) | |
559 | |
560 @staticmethod | |
561 def make_urllib2_request(request): | |
562 """Converts HttpRequest to urllib2.Request.""" | |
563 result = urllib2.Request(request.get_full_url(), data=request.body) | |
564 for header, value in request.headers.iteritems(): | |
565 result.add_header(header, value) | |
566 return result | |
567 | |
568 | |
569 class RequestsLibEngine(RequestEngine): | |
570 """Class that knows how to execute HttpRequests via requests library.""" | |
571 | |
572 # Preferred number of connections in a connection pool. | |
573 CONNECTION_POOL_SIZE = 64 | |
574 # If True will not open more than CONNECTION_POOL_SIZE connections. | |
575 CONNECTION_POOL_BLOCK = False | |
576 # Maximum number of internal connection retries in a connection pool. | |
577 CONNECTION_RETRIES = 0 | |
578 | |
579 def __init__(self, cookie_jar, ca_certs): | |
580 super(RequestsLibEngine, self).__init__() | |
581 self.session = requests.Session() | |
582 self.cookie_jar = cookie_jar | |
583 # Configure session. | |
584 self.session.trust_env = False | |
585 if cookie_jar: | |
586 self.session.cookies = cookie_jar | |
587 self.session.verify = ca_certs | |
588 # Configure connection pools. | |
589 for protocol in ('https://', 'http://'): | |
590 self.session.mount(protocol, adapters.HTTPAdapter( | |
591 pool_connections=self.CONNECTION_POOL_SIZE, | |
592 pool_maxsize=self.CONNECTION_POOL_SIZE, | |
593 max_retries=self.CONNECTION_RETRIES, | |
594 pool_block=self.CONNECTION_POOL_BLOCK)) | |
595 | |
596 def perform_request(self, request): | |
597 try: | |
598 response = self.session.request( | |
599 method=request.method, | |
600 url=request.url, | |
601 params=request.params, | |
602 data=request.body, | |
603 headers=request.headers, | |
604 timeout=request.timeout, | |
605 stream=request.stream) | |
606 response.raise_for_status() | |
607 if request.stream: | |
608 stream = response.raw | |
609 else: | |
610 stream = StringIO.StringIO(response.content) | |
611 return HttpResponse(stream, request.get_full_url(), response.headers) | |
612 except requests.Timeout as e: | |
613 raise TimeoutError(e) | |
614 except requests.HTTPError as e: | |
615 raise HttpError(e.response.status_code, e) | |
616 except (requests.ConnectionError, socket.timeout, ssl.SSLError) as e: | |
617 raise ConnectionError(e) | |
618 | |
619 def reload_cookies(self): | |
620 if self.cookie_jar: | |
621 self.session.cookies = self.cookie_jar | |
622 | |
623 | |
624 class AppEngineAuthenticator(Authenticator): | |
625 """Helper class to perform AppEngine authentication dance via upload.py.""" | |
626 | |
627 # This lock ensures that user won't be confused with multiple concurrent | |
628 # login prompts. | |
629 _auth_lock = threading.Lock() | |
630 | |
631 def __init__(self, urlhost, cookie_jar, email=None, password=None): | |
632 super(AppEngineAuthenticator, self).__init__() | |
633 self.urlhost = urlhost | |
634 self.cookie_jar = cookie_jar | |
635 self.email = email | |
636 self.password = password | |
637 self._keyring = None | |
638 | |
639 def authenticate(self): | |
640 """Authenticates in the app engine application. | |
641 | |
642 Mutates |self.cookie_jar| in place by adding all required cookies. | |
643 | |
644 Returns True on success. | |
645 """ | |
646 # To be used from inside AuthServer. | |
647 cookie_jar = self.cookie_jar | |
648 # RPC server that uses AuthenticationSupport's cookie jar. | |
649 class AuthServer(upload.AbstractRpcServer): | |
650 def _GetOpener(self): | |
651 # Authentication code needs to know about 302 response. | |
652 # So make OpenerDirector without HTTPRedirectHandler. | |
653 opener = urllib2.OpenerDirector() | |
654 opener.add_handler(urllib2.ProxyHandler()) | |
655 opener.add_handler(urllib2.UnknownHandler()) | |
656 opener.add_handler(urllib2.HTTPHandler()) | |
657 opener.add_handler(urllib2.HTTPDefaultErrorHandler()) | |
658 opener.add_handler(urllib2.HTTPSHandler()) | |
659 opener.add_handler(urllib2.HTTPErrorProcessor()) | |
660 opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar)) | |
661 return opener | |
662 def PerformAuthentication(self): | |
663 self._Authenticate() | |
664 return self.authenticated | |
665 with cookie_jar: | |
666 with self._auth_lock: | |
667 rpc_server = AuthServer(self.urlhost, self.get_credentials) | |
668 return rpc_server.PerformAuthentication() | |
669 | |
670 def get_credentials(self): | |
671 """Called during authentication process to get the credentials. | |
672 May be called multiple times if authentication fails. | |
673 Returns tuple (email, password). | |
674 """ | |
675 if self.email and self.password: | |
676 return (self.email, self.password) | |
677 self._keyring = self._keyring or upload.KeyringCreds(self.urlhost, | |
678 self.urlhost, self.email) | |
679 return self._keyring.GetUserCredentials() | |
680 | |
681 | |
682 class ThreadSafeCookieJar(cookielib.MozillaCookieJar): | |
683 """MozillaCookieJar with thread safe load and save.""" | |
684 | |
685 def __enter__(self): | |
686 """Context manager interface.""" | |
687 return self | |
688 | |
689 def __exit__(self, *_args): | |
690 """Saves cookie jar when exiting the block.""" | |
691 self.save() | |
692 return False | |
693 | |
694 def load(self, filename=None, ignore_discard=False, ignore_expires=False): | |
695 """Loads cookies from the file if it exists.""" | |
696 filename = os.path.expanduser(filename or self.filename) | |
697 with self._cookies_lock: | |
698 if os.path.exists(filename): | |
699 try: | |
700 cookielib.MozillaCookieJar.load( | |
701 self, filename, ignore_discard, ignore_expires) | |
702 logging.debug('Loaded cookies from %s', filename) | |
703 except (cookielib.LoadError, IOError): | |
704 pass | |
705 else: | |
706 try: | |
707 fd = os.open(filename, os.O_CREAT, 0600) | |
708 os.close(fd) | |
709 except OSError: | |
710 logging.debug('Failed to create %s', filename) | |
711 try: | |
712 os.chmod(filename, 0600) | |
713 except OSError: | |
714 logging.debug('Failed to fix mode for %s', filename) | |
715 | |
716 def save(self, filename=None, ignore_discard=False, ignore_expires=False): | |
717 """Saves cookies to the file, completely overwriting it.""" | |
718 logging.debug('Saving cookies to %s', filename or self.filename) | |
719 with self._cookies_lock: | |
720 try: | |
721 cookielib.MozillaCookieJar.save( | |
722 self, filename, ignore_discard, ignore_expires) | |
723 except OSError: | |
724 logging.error('Failed to save %s', filename) | |
725 | |
726 | |
727 class RetryAttempt(object): | |
728 """Contains information about current retry attempt. | |
729 | |
730 Yielded from retry_loop. | |
731 """ | |
732 | |
733 def __init__(self, attempt, remaining): | |
734 """Information about current attempt in retry loop: | |
735 |attempt| - zero based index of attempt. | |
736 |remaining| - how much time is left before retry loop finishes retries. | |
737 """ | |
738 self.attempt = attempt | |
739 self.remaining = remaining | |
740 self.skip_sleep = False | |
741 | |
742 | |
743 def calculate_sleep_before_retry(attempt, max_duration): | |
744 """How long to sleep before retrying an attempt in retry_loop.""" | |
745 # Maximum sleeping time. We're hammering a cloud-distributed service, it'll | |
746 # survive. | |
747 MAX_SLEEP = 10. | |
748 # random.random() returns [0.0, 1.0). Starts with relatively short waiting | |
749 # time by starting with 1.5/2+1.5^-1 median offset. | |
750 duration = (random.random() * 1.5) + math.pow(1.5, (attempt - 1)) | |
751 assert duration > 0.1 | |
752 duration = min(MAX_SLEEP, duration) | |
753 if max_duration: | |
754 duration = min(max_duration, duration) | |
755 return duration | |
756 | |
757 | |
758 def sleep_before_retry(attempt, max_duration): | |
759 """Sleeps for some amount of time when retrying the attempt in retry_loop. | |
760 | |
761 To be mocked in tests. | |
762 """ | |
763 time.sleep(calculate_sleep_before_retry(attempt, max_duration)) | |
764 | |
765 | |
766 def current_time(): | |
767 """Used by retry loop to get current time. | |
768 | |
769 To be mocked in tests. | |
770 """ | |
771 return time.time() | |
772 | |
773 | |
774 def retry_loop(max_attempts=None, timeout=None): | |
775 """Yields whenever new attempt to perform some action is needed. | |
776 | |
777 Yields instances of RetryAttempt class that contains information about current | |
778 attempt. Setting |skip_sleep| attribute of RetryAttempt to True will cause | |
779 retry loop to run next attempt immediately. | |
780 """ | |
781 start = current_time() | |
782 for attempt in itertools.count(): | |
783 # Too many attempts? | |
784 if max_attempts and attempt == max_attempts: | |
785 break | |
786 # Retried for too long? | |
787 remaining = (timeout - (current_time() - start)) if timeout else None | |
788 if remaining is not None and remaining < 0: | |
789 break | |
790 # Kick next iteration. | |
791 attemp_obj = RetryAttempt(attempt, remaining) | |
792 yield attemp_obj | |
793 if attemp_obj.skip_sleep: | |
794 continue | |
795 # Only sleep if we are going to try again. | |
796 if max_attempts and attempt != max_attempts - 1: | |
797 remaining = (timeout - (current_time() - start)) if timeout else None | |
798 if remaining is not None and remaining < 0: | |
799 break | |
800 sleep_before_retry(attempt, remaining) | |
OLD | NEW |