| OLD | NEW |
|---|
| (Empty) |
| 1 # urllib3/connectionpool.py | |
| 2 # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) | |
| 3 # | |
| 4 # This module is part of urllib3 and is released under | |
| 5 # the MIT License: http://www.opensource.org/licenses/mit-license.php | |
| 6 | |
| 7 import errno | |
| 8 import logging | |
| 9 | |
| 10 from socket import error as SocketError, timeout as SocketTimeout | |
| 11 import socket | |
| 12 | |
| 13 try: # Python 3 | |
| 14 from http.client import HTTPConnection, HTTPException | |
| 15 from http.client import HTTP_PORT, HTTPS_PORT | |
| 16 except ImportError: | |
| 17 from httplib import HTTPConnection, HTTPException | |
| 18 from httplib import HTTP_PORT, HTTPS_PORT | |
| 19 | |
| 20 try: # Python 3 | |
| 21 from queue import LifoQueue, Empty, Full | |
| 22 except ImportError: | |
| 23 from Queue import LifoQueue, Empty, Full | |
| 24 import Queue as _ # Platform-specific: Windows | |
| 25 | |
| 26 | |
| 27 try: # Compiled with SSL? | |
| 28 HTTPSConnection = object | |
| 29 | |
| 30 class BaseSSLError(BaseException): | |
| 31 pass | |
| 32 | |
| 33 ssl = None | |
| 34 | |
| 35 try: # Python 3 | |
| 36 from http.client import HTTPSConnection | |
| 37 except ImportError: | |
| 38 from httplib import HTTPSConnection | |
| 39 | |
| 40 import ssl | |
| 41 BaseSSLError = ssl.SSLError | |
| 42 | |
| 43 except (ImportError, AttributeError): # Platform-specific: No SSL. | |
| 44 pass | |
| 45 | |
| 46 | |
| 47 from .exceptions import ( | |
| 48 ClosedPoolError, | |
| 49 ConnectTimeoutError, | |
| 50 EmptyPoolError, | |
| 51 HostChangedError, | |
| 52 MaxRetryError, | |
| 53 SSLError, | |
| 54 ReadTimeoutError, | |
| 55 ProxyError, | |
| 56 ) | |
| 57 from .packages.ssl_match_hostname import CertificateError, match_hostname | |
| 58 from .packages import six | |
| 59 from .request import RequestMethods | |
| 60 from .response import HTTPResponse | |
| 61 from .util import ( | |
| 62 assert_fingerprint, | |
| 63 get_host, | |
| 64 is_connection_dropped, | |
| 65 resolve_cert_reqs, | |
| 66 resolve_ssl_version, | |
| 67 ssl_wrap_socket, | |
| 68 Timeout, | |
| 69 ) | |
| 70 | |
| 71 xrange = six.moves.xrange | |
| 72 | |
| 73 log = logging.getLogger(__name__) | |
| 74 | |
| 75 _Default = object() | |
| 76 | |
| 77 port_by_scheme = { | |
| 78 'http': HTTP_PORT, | |
| 79 'https': HTTPS_PORT, | |
| 80 } | |
| 81 | |
| 82 | |
| 83 ## Connection objects (extension of httplib) | |
| 84 | |
| 85 class VerifiedHTTPSConnection(HTTPSConnection): | |
| 86 """ | |
| 87 Based on httplib.HTTPSConnection but wraps the socket with | |
| 88 SSL certification. | |
| 89 """ | |
| 90 cert_reqs = None | |
| 91 ca_certs = None | |
| 92 ssl_version = None | |
| 93 | |
| 94 def set_cert(self, key_file=None, cert_file=None, | |
| 95 cert_reqs=None, ca_certs=None, | |
| 96 assert_hostname=None, assert_fingerprint=None): | |
| 97 | |
| 98 self.key_file = key_file | |
| 99 self.cert_file = cert_file | |
| 100 self.cert_reqs = cert_reqs | |
| 101 self.ca_certs = ca_certs | |
| 102 self.assert_hostname = assert_hostname | |
| 103 self.assert_fingerprint = assert_fingerprint | |
| 104 | |
| 105 def connect(self): | |
| 106 # Add certificate verification | |
| 107 try: | |
| 108 sock = socket.create_connection( | |
| 109 address=(self.host, self.port), | |
| 110 timeout=self.timeout) | |
| 111 except SocketTimeout: | |
| 112 raise ConnectTimeoutError( | |
| 113 self, "Connection to %s timed out. (connect timeout=%s)" % | |
| 114 (self.host, self.timeout)) | |
| 115 | |
| 116 resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) | |
| 117 resolved_ssl_version = resolve_ssl_version(self.ssl_version) | |
| 118 | |
| 119 if self._tunnel_host: | |
| 120 self.sock = sock | |
| 121 # Calls self._set_hostport(), so self.host is | |
| 122 # self._tunnel_host below. | |
| 123 self._tunnel() | |
| 124 | |
| 125 # Wrap socket using verification with the root certs in | |
| 126 # trusted_root_certs | |
| 127 self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file, | |
| 128 cert_reqs=resolved_cert_reqs, | |
| 129 ca_certs=self.ca_certs, | |
| 130 server_hostname=self.host, | |
| 131 ssl_version=resolved_ssl_version) | |
| 132 | |
| 133 if resolved_cert_reqs != ssl.CERT_NONE: | |
| 134 if self.assert_fingerprint: | |
| 135 assert_fingerprint(self.sock.getpeercert(binary_form=True), | |
| 136 self.assert_fingerprint) | |
| 137 elif self.assert_hostname is not False: | |
| 138 match_hostname(self.sock.getpeercert(), | |
| 139 self.assert_hostname or self.host) | |
| 140 | |
| 141 | |
| 142 ## Pool objects | |
| 143 | |
| 144 class ConnectionPool(object): | |
| 145 """ | |
| 146 Base class for all connection pools, such as | |
| 147 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. | |
| 148 """ | |
| 149 | |
| 150 scheme = None | |
| 151 QueueCls = LifoQueue | |
| 152 | |
| 153 def __init__(self, host, port=None): | |
| 154 # httplib doesn't like it when we include brackets in ipv6 addresses | |
| 155 host = host.strip('[]') | |
| 156 | |
| 157 self.host = host | |
| 158 self.port = port | |
| 159 | |
| 160 def __str__(self): | |
| 161 return '%s(host=%r, port=%r)' % (type(self).__name__, | |
| 162 self.host, self.port) | |
| 163 | |
| 164 # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.p
y#l252 | |
| 165 _blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) | |
| 166 | |
| 167 class HTTPConnectionPool(ConnectionPool, RequestMethods): | |
| 168 """ | |
| 169 Thread-safe connection pool for one host. | |
| 170 | |
| 171 :param host: | |
| 172 Host used for this HTTP Connection (e.g. "localhost"), passed into | |
| 173 :class:`httplib.HTTPConnection`. | |
| 174 | |
| 175 :param port: | |
| 176 Port used for this HTTP Connection (None is equivalent to 80), passed | |
| 177 into :class:`httplib.HTTPConnection`. | |
| 178 | |
| 179 :param strict: | |
| 180 Causes BadStatusLine to be raised if the status line can't be parsed | |
| 181 as a valid HTTP/1.0 or 1.1 status line, passed into | |
| 182 :class:`httplib.HTTPConnection`. | |
| 183 | |
| 184 .. note:: | |
| 185 Only works in Python 2. This parameter is ignored in Python 3. | |
| 186 | |
| 187 :param timeout: | |
| 188 Socket timeout in seconds for each individual connection. This can | |
| 189 be a float or integer, which sets the timeout for the HTTP request, | |
| 190 or an instance of :class:`urllib3.util.Timeout` which gives you more | |
| 191 fine-grained control over request timeouts. After the constructor has | |
| 192 been parsed, this is always a `urllib3.util.Timeout` object. | |
| 193 | |
| 194 :param maxsize: | |
| 195 Number of connections to save that can be reused. More than 1 is useful | |
| 196 in multithreaded situations. If ``block`` is set to false, more | |
| 197 connections will be created but they will not be saved once they've | |
| 198 been used. | |
| 199 | |
| 200 :param block: | |
| 201 If set to True, no more than ``maxsize`` connections will be used at | |
| 202 a time. When no free connections are available, the call will block | |
| 203 until a connection has been released. This is a useful side effect for | |
| 204 particular multithreaded situations where one does not want to use more | |
| 205 than maxsize connections per host to prevent flooding. | |
| 206 | |
| 207 :param headers: | |
| 208 Headers to include with all requests, unless other headers are given | |
| 209 explicitly. | |
| 210 | |
| 211 :param _proxy: | |
| 212 Parsed proxy URL, should not be used directly, instead, see | |
| 213 :class:`urllib3.connectionpool.ProxyManager`" | |
| 214 | |
| 215 :param _proxy_headers: | |
| 216 A dictionary with proxy headers, should not be used directly, | |
| 217 instead, see :class:`urllib3.connectionpool.ProxyManager`" | |
| 218 """ | |
| 219 | |
| 220 scheme = 'http' | |
| 221 | |
| 222 def __init__(self, host, port=None, strict=False, | |
| 223 timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False, | |
| 224 headers=None, _proxy=None, _proxy_headers=None): | |
| 225 ConnectionPool.__init__(self, host, port) | |
| 226 RequestMethods.__init__(self, headers) | |
| 227 | |
| 228 self.strict = strict | |
| 229 | |
| 230 # This is for backwards compatibility and can be removed once a timeout | |
| 231 # can only be set to a Timeout object | |
| 232 if not isinstance(timeout, Timeout): | |
| 233 timeout = Timeout.from_float(timeout) | |
| 234 | |
| 235 self.timeout = timeout | |
| 236 | |
| 237 self.pool = self.QueueCls(maxsize) | |
| 238 self.block = block | |
| 239 | |
| 240 self.proxy = _proxy | |
| 241 self.proxy_headers = _proxy_headers or {} | |
| 242 | |
| 243 # Fill the queue up so that doing get() on it will block properly | |
| 244 for _ in xrange(maxsize): | |
| 245 self.pool.put(None) | |
| 246 | |
| 247 # These are mostly for testing and debugging purposes. | |
| 248 self.num_connections = 0 | |
| 249 self.num_requests = 0 | |
| 250 | |
| 251 def _new_conn(self): | |
| 252 """ | |
| 253 Return a fresh :class:`httplib.HTTPConnection`. | |
| 254 """ | |
| 255 self.num_connections += 1 | |
| 256 log.info("Starting new HTTP connection (%d): %s" % | |
| 257 (self.num_connections, self.host)) | |
| 258 extra_params = {} | |
| 259 if not six.PY3: # Python 2 | |
| 260 extra_params['strict'] = self.strict | |
| 261 | |
| 262 return HTTPConnection(host=self.host, port=self.port, | |
| 263 timeout=self.timeout.connect_timeout, | |
| 264 **extra_params) | |
| 265 | |
| 266 | |
| 267 def _get_conn(self, timeout=None): | |
| 268 """ | |
| 269 Get a connection. Will return a pooled connection if one is available. | |
| 270 | |
| 271 If no connections are available and :prop:`.block` is ``False``, then a | |
| 272 fresh connection is returned. | |
| 273 | |
| 274 :param timeout: | |
| 275 Seconds to wait before giving up and raising | |
| 276 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and | |
| 277 :prop:`.block` is ``True``. | |
| 278 """ | |
| 279 conn = None | |
| 280 try: | |
| 281 conn = self.pool.get(block=self.block, timeout=timeout) | |
| 282 | |
| 283 except AttributeError: # self.pool is None | |
| 284 raise ClosedPoolError(self, "Pool is closed.") | |
| 285 | |
| 286 except Empty: | |
| 287 if self.block: | |
| 288 raise EmptyPoolError(self, | |
| 289 "Pool reached maximum size and no more " | |
| 290 "connections are allowed.") | |
| 291 pass # Oh well, we'll create a new connection then | |
| 292 | |
| 293 # If this is a persistent connection, check if it got disconnected | |
| 294 if conn and is_connection_dropped(conn): | |
| 295 log.info("Resetting dropped connection: %s" % self.host) | |
| 296 conn.close() | |
| 297 | |
| 298 return conn or self._new_conn() | |
| 299 | |
| 300 def _put_conn(self, conn): | |
| 301 """ | |
| 302 Put a connection back into the pool. | |
| 303 | |
| 304 :param conn: | |
| 305 Connection object for the current host and port as returned by | |
| 306 :meth:`._new_conn` or :meth:`._get_conn`. | |
| 307 | |
| 308 If the pool is already full, the connection is closed and discarded | |
| 309 because we exceeded maxsize. If connections are discarded frequently, | |
| 310 then maxsize should be increased. | |
| 311 | |
| 312 If the pool is closed, then the connection will be closed and discarded. | |
| 313 """ | |
| 314 try: | |
| 315 self.pool.put(conn, block=False) | |
| 316 return # Everything is dandy, done. | |
| 317 except AttributeError: | |
| 318 # self.pool is None. | |
| 319 pass | |
| 320 except Full: | |
| 321 # This should never happen if self.block == True | |
| 322 log.warning("HttpConnectionPool is full, discarding connection: %s" | |
| 323 % self.host) | |
| 324 | |
| 325 # Connection never got put back into the pool, close it. | |
| 326 if conn: | |
| 327 conn.close() | |
| 328 | |
| 329 def _get_timeout(self, timeout): | |
| 330 """ Helper that always returns a :class:`urllib3.util.Timeout` """ | |
| 331 if timeout is _Default: | |
| 332 return self.timeout.clone() | |
| 333 | |
| 334 if isinstance(timeout, Timeout): | |
| 335 return timeout.clone() | |
| 336 else: | |
| 337 # User passed us an int/float. This is for backwards compatibility, | |
| 338 # can be removed later | |
| 339 return Timeout.from_float(timeout) | |
| 340 | |
| 341 def _make_request(self, conn, method, url, timeout=_Default, | |
| 342 **httplib_request_kw): | |
| 343 """ | |
| 344 Perform a request on a given httplib connection object taken from our | |
| 345 pool. | |
| 346 | |
| 347 :param conn: | |
| 348 a connection from one of our connection pools | |
| 349 | |
| 350 :param timeout: | |
| 351 Socket timeout in seconds for the request. This can be a | |
| 352 float or integer, which will set the same timeout value for | |
| 353 the socket connect and the socket read, or an instance of | |
| 354 :class:`urllib3.util.Timeout`, which gives you more fine-grained | |
| 355 control over your timeouts. | |
| 356 """ | |
| 357 self.num_requests += 1 | |
| 358 | |
| 359 timeout_obj = self._get_timeout(timeout) | |
| 360 | |
| 361 try: | |
| 362 timeout_obj.start_connect() | |
| 363 conn.timeout = timeout_obj.connect_timeout | |
| 364 # conn.request() calls httplib.*.request, not the method in | |
| 365 # request.py. It also calls makefile (recv) on the socket | |
| 366 conn.request(method, url, **httplib_request_kw) | |
| 367 except SocketTimeout: | |
| 368 raise ConnectTimeoutError( | |
| 369 self, "Connection to %s timed out. (connect timeout=%s)" % | |
| 370 (self.host, timeout_obj.connect_timeout)) | |
| 371 | |
| 372 # Reset the timeout for the recv() on the socket | |
| 373 read_timeout = timeout_obj.read_timeout | |
| 374 log.debug("Setting read timeout to %s" % read_timeout) | |
| 375 # App Engine doesn't have a sock attr | |
| 376 if hasattr(conn, 'sock') and \ | |
| 377 read_timeout is not None and \ | |
| 378 read_timeout is not Timeout.DEFAULT_TIMEOUT: | |
| 379 # In Python 3 socket.py will catch EAGAIN and return None when you | |
| 380 # try and read into the file pointer created by http.client, which | |
| 381 # instead raises a BadStatusLine exception. Instead of catching | |
| 382 # the exception and assuming all BadStatusLine exceptions are read | |
| 383 # timeouts, check for a zero timeout before making the request. | |
| 384 if read_timeout == 0: | |
| 385 raise ReadTimeoutError( | |
| 386 self, url, | |
| 387 "Read timed out. (read timeout=%s)" % read_timeout) | |
| 388 conn.sock.settimeout(read_timeout) | |
| 389 | |
| 390 # Receive the response from the server | |
| 391 try: | |
| 392 try: # Python 2.7+, use buffering of HTTP responses | |
| 393 httplib_response = conn.getresponse(buffering=True) | |
| 394 except TypeError: # Python 2.6 and older | |
| 395 httplib_response = conn.getresponse() | |
| 396 except SocketTimeout: | |
| 397 raise ReadTimeoutError( | |
| 398 self, url, "Read timed out. (read timeout=%s)" % read_timeout) | |
| 399 | |
| 400 except SocketError as e: # Platform-specific: Python 2 | |
| 401 # See the above comment about EAGAIN in Python 3. In Python 2 we | |
| 402 # have to specifically catch it and throw the timeout error | |
| 403 if e.errno in _blocking_errnos: | |
| 404 raise ReadTimeoutError( | |
| 405 self, url, | |
| 406 "Read timed out. (read timeout=%s)" % read_timeout) | |
| 407 raise | |
| 408 | |
| 409 | |
| 410 # AppEngine doesn't have a version attr. | |
| 411 http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') | |
| 412 log.debug("\"%s %s %s\" %s %s" % (method, url, http_version, | |
| 413 httplib_response.status, | |
| 414 httplib_response.length)) | |
| 415 return httplib_response | |
| 416 | |
| 417 def close(self): | |
| 418 """ | |
| 419 Close all pooled connections and disable the pool. | |
| 420 """ | |
| 421 # Disable access to the pool | |
| 422 old_pool, self.pool = self.pool, None | |
| 423 | |
| 424 try: | |
| 425 while True: | |
| 426 conn = old_pool.get(block=False) | |
| 427 if conn: | |
| 428 conn.close() | |
| 429 | |
| 430 except Empty: | |
| 431 pass # Done. | |
| 432 | |
| 433 def is_same_host(self, url): | |
| 434 """ | |
| 435 Check if the given ``url`` is a member of the same host as this | |
| 436 connection pool. | |
| 437 """ | |
| 438 if url.startswith('/'): | |
| 439 return True | |
| 440 | |
| 441 # TODO: Add optional support for socket.gethostbyname checking. | |
| 442 scheme, host, port = get_host(url) | |
| 443 | |
| 444 if self.port and not port: | |
| 445 # Use explicit default port for comparison when none is given. | |
| 446 port = port_by_scheme.get(scheme) | |
| 447 | |
| 448 return (scheme, host, port) == (self.scheme, self.host, self.port) | |
| 449 | |
| 450 def urlopen(self, method, url, body=None, headers=None, retries=3, | |
| 451 redirect=True, assert_same_host=True, timeout=_Default, | |
| 452 pool_timeout=None, release_conn=None, **response_kw): | |
| 453 """ | |
| 454 Get a connection from the pool and perform an HTTP request. This is the | |
| 455 lowest level call for making a request, so you'll need to specify all | |
| 456 the raw details. | |
| 457 | |
| 458 .. note:: | |
| 459 | |
| 460 More commonly, it's appropriate to use a convenience method provided | |
| 461 by :class:`.RequestMethods`, such as :meth:`request`. | |
| 462 | |
| 463 .. note:: | |
| 464 | |
| 465 `release_conn` will only behave as expected if | |
| 466 `preload_content=False` because we want to make | |
| 467 `preload_content=False` the default behaviour someday soon without | |
| 468 breaking backwards compatibility. | |
| 469 | |
| 470 :param method: | |
| 471 HTTP request method (such as GET, POST, PUT, etc.) | |
| 472 | |
| 473 :param body: | |
| 474 Data to send in the request body (useful for creating | |
| 475 POST requests, see HTTPConnectionPool.post_url for | |
| 476 more convenience). | |
| 477 | |
| 478 :param headers: | |
| 479 Dictionary of custom headers to send, such as User-Agent, | |
| 480 If-None-Match, etc. If None, pool headers are used. If provided, | |
| 481 these headers completely replace any pool-specific headers. | |
| 482 | |
| 483 :param retries: | |
| 484 Number of retries to allow before raising a MaxRetryError exception. | |
| 485 | |
| 486 :param redirect: | |
| 487 If True, automatically handle redirects (status codes 301, 302, | |
| 488 303, 307, 308). Each redirect counts as a retry. | |
| 489 | |
| 490 :param assert_same_host: | |
| 491 If ``True``, will make sure that the host of the pool requests is | |
| 492 consistent else will raise HostChangedError. When False, you can | |
| 493 use the pool on an HTTP proxy and request foreign hosts. | |
| 494 | |
| 495 :param timeout: | |
| 496 If specified, overrides the default timeout for this one | |
| 497 request. It may be a float (in seconds) or an instance of | |
| 498 :class:`urllib3.util.Timeout`. | |
| 499 | |
| 500 :param pool_timeout: | |
| 501 If set and the pool is set to block=True, then this method will | |
| 502 block for ``pool_timeout`` seconds and raise EmptyPoolError if no | |
| 503 connection is available within the time period. | |
| 504 | |
| 505 :param release_conn: | |
| 506 If False, then the urlopen call will not release the connection | |
| 507 back into the pool once a response is received (but will release if | |
| 508 you read the entire contents of the response such as when | |
| 509 `preload_content=True`). This is useful if you're not preloading | |
| 510 the response's content immediately. You will need to call | |
| 511 ``r.release_conn()`` on the response ``r`` to return the connection | |
| 512 back into the pool. If None, it takes the value of | |
| 513 ``response_kw.get('preload_content', True)``. | |
| 514 | |
| 515 :param \**response_kw: | |
| 516 Additional parameters are passed to | |
| 517 :meth:`urllib3.response.HTTPResponse.from_httplib` | |
| 518 """ | |
| 519 if headers is None: | |
| 520 headers = self.headers | |
| 521 | |
| 522 if retries < 0: | |
| 523 raise MaxRetryError(self, url) | |
| 524 | |
| 525 if release_conn is None: | |
| 526 release_conn = response_kw.get('preload_content', True) | |
| 527 | |
| 528 # Check host | |
| 529 if assert_same_host and not self.is_same_host(url): | |
| 530 raise HostChangedError(self, url, retries - 1) | |
| 531 | |
| 532 conn = None | |
| 533 | |
| 534 try: | |
| 535 # Request a connection from the queue | |
| 536 conn = self._get_conn(timeout=pool_timeout) | |
| 537 | |
| 538 # Make the request on the httplib connection object | |
| 539 httplib_response = self._make_request(conn, method, url, | |
| 540 timeout=timeout, | |
| 541 body=body, headers=headers) | |
| 542 | |
| 543 # If we're going to release the connection in ``finally:``, then | |
| 544 # the request doesn't need to know about the connection. Otherwise | |
| 545 # it will also try to release it and we'll have a double-release | |
| 546 # mess. | |
| 547 response_conn = not release_conn and conn | |
| 548 | |
| 549 # Import httplib's response into our own wrapper object | |
| 550 response = HTTPResponse.from_httplib(httplib_response, | |
| 551 pool=self, | |
| 552 connection=response_conn, | |
| 553 **response_kw) | |
| 554 | |
| 555 # else: | |
| 556 # The connection will be put back into the pool when | |
| 557 # ``response.release_conn()`` is called (implicitly by | |
| 558 # ``response.read()``) | |
| 559 | |
| 560 except Empty: | |
| 561 # Timed out by queue | |
| 562 raise ReadTimeoutError( | |
| 563 self, url, "Read timed out, no pool connections are available.") | |
| 564 | |
| 565 except SocketTimeout: | |
| 566 # Timed out by socket | |
| 567 raise ReadTimeoutError(self, url, "Read timed out.") | |
| 568 | |
| 569 except BaseSSLError as e: | |
| 570 # SSL certificate error | |
| 571 if 'timed out' in str(e) or \ | |
| 572 'did not complete (read)' in str(e): # Platform-specific: Python
2.6 | |
| 573 raise ReadTimeoutError(self, url, "Read timed out.") | |
| 574 raise SSLError(e) | |
| 575 | |
| 576 except CertificateError as e: | |
| 577 # Name mismatch | |
| 578 raise SSLError(e) | |
| 579 | |
| 580 except (HTTPException, SocketError) as e: | |
| 581 if isinstance(e, SocketError) and self.proxy is not None: | |
| 582 raise ProxyError('Cannot connect to proxy. ' | |
| 583 'Socket error: %s.' % e) | |
| 584 | |
| 585 # Connection broken, discard. It will be replaced next _get_conn(). | |
| 586 conn = None | |
| 587 # This is necessary so we can access e below | |
| 588 err = e | |
| 589 | |
| 590 if retries == 0: | |
| 591 raise MaxRetryError(self, url, e) | |
| 592 | |
| 593 finally: | |
| 594 if release_conn: | |
| 595 # Put the connection back to be reused. If the connection is | |
| 596 # expired then it will be None, which will get replaced with a | |
| 597 # fresh connection during _get_conn. | |
| 598 self._put_conn(conn) | |
| 599 | |
| 600 if not conn: | |
| 601 # Try again | |
| 602 log.warn("Retrying (%d attempts remain) after connection " | |
| 603 "broken by '%r': %s" % (retries, err, url)) | |
| 604 return self.urlopen(method, url, body, headers, retries - 1, | |
| 605 redirect, assert_same_host, | |
| 606 timeout=timeout, pool_timeout=pool_timeout, | |
| 607 release_conn=release_conn, **response_kw) | |
| 608 | |
| 609 # Handle redirect? | |
| 610 redirect_location = redirect and response.get_redirect_location() | |
| 611 if redirect_location: | |
| 612 if response.status == 303: | |
| 613 method = 'GET' | |
| 614 log.info("Redirecting %s -> %s" % (url, redirect_location)) | |
| 615 return self.urlopen(method, redirect_location, body, headers, | |
| 616 retries - 1, redirect, assert_same_host, | |
| 617 timeout=timeout, pool_timeout=pool_timeout, | |
| 618 release_conn=release_conn, **response_kw) | |
| 619 | |
| 620 return response | |
| 621 | |
| 622 | |
| 623 class HTTPSConnectionPool(HTTPConnectionPool): | |
| 624 """ | |
| 625 Same as :class:`.HTTPConnectionPool`, but HTTPS. | |
| 626 | |
| 627 When Python is compiled with the :mod:`ssl` module, then | |
| 628 :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates, | |
| 629 instead of :class:`httplib.HTTPSConnection`. | |
| 630 | |
| 631 :class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``, | |
| 632 ``assert_hostname`` and ``host`` in this order to verify connections. | |
| 633 If ``assert_hostname`` is False, no verification is done. | |
| 634 | |
| 635 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs`` and | |
| 636 ``ssl_version`` are only used if :mod:`ssl` is available and are fed into | |
| 637 :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket | |
| 638 into an SSL socket. | |
| 639 """ | |
| 640 | |
| 641 scheme = 'https' | |
| 642 | |
| 643 def __init__(self, host, port=None, | |
| 644 strict=False, timeout=None, maxsize=1, | |
| 645 block=False, headers=None, | |
| 646 _proxy=None, _proxy_headers=None, | |
| 647 key_file=None, cert_file=None, cert_reqs=None, | |
| 648 ca_certs=None, ssl_version=None, | |
| 649 assert_hostname=None, assert_fingerprint=None): | |
| 650 | |
| 651 HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, | |
| 652 block, headers, _proxy, _proxy_headers) | |
| 653 self.key_file = key_file | |
| 654 self.cert_file = cert_file | |
| 655 self.cert_reqs = cert_reqs | |
| 656 self.ca_certs = ca_certs | |
| 657 self.ssl_version = ssl_version | |
| 658 self.assert_hostname = assert_hostname | |
| 659 self.assert_fingerprint = assert_fingerprint | |
| 660 | |
| 661 def _prepare_conn(self, connection): | |
| 662 """ | |
| 663 Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket` | |
| 664 and establish the tunnel if proxy is used. | |
| 665 """ | |
| 666 | |
| 667 if isinstance(connection, VerifiedHTTPSConnection): | |
| 668 connection.set_cert(key_file=self.key_file, | |
| 669 cert_file=self.cert_file, | |
| 670 cert_reqs=self.cert_reqs, | |
| 671 ca_certs=self.ca_certs, | |
| 672 assert_hostname=self.assert_hostname, | |
| 673 assert_fingerprint=self.assert_fingerprint) | |
| 674 connection.ssl_version = self.ssl_version | |
| 675 | |
| 676 if self.proxy is not None: | |
| 677 # Python 2.7+ | |
| 678 try: | |
| 679 set_tunnel = connection.set_tunnel | |
| 680 except AttributeError: # Platform-specific: Python 2.6 | |
| 681 set_tunnel = connection._set_tunnel | |
| 682 set_tunnel(self.host, self.port, self.proxy_headers) | |
| 683 # Establish tunnel connection early, because otherwise httplib | |
| 684 # would improperly set Host: header to proxy's IP:port. | |
| 685 connection.connect() | |
| 686 | |
| 687 return connection | |
| 688 | |
| 689 def _new_conn(self): | |
| 690 """ | |
| 691 Return a fresh :class:`httplib.HTTPSConnection`. | |
| 692 """ | |
| 693 self.num_connections += 1 | |
| 694 log.info("Starting new HTTPS connection (%d): %s" | |
| 695 % (self.num_connections, self.host)) | |
| 696 | |
| 697 actual_host = self.host | |
| 698 actual_port = self.port | |
| 699 if self.proxy is not None: | |
| 700 actual_host = self.proxy.host | |
| 701 actual_port = self.proxy.port | |
| 702 | |
| 703 if not ssl: # Platform-specific: Python compiled without +ssl | |
| 704 if not HTTPSConnection or HTTPSConnection is object: | |
| 705 raise SSLError("Can't connect to HTTPS URL because the SSL " | |
| 706 "module is not available.") | |
| 707 connection_class = HTTPSConnection | |
| 708 else: | |
| 709 connection_class = VerifiedHTTPSConnection | |
| 710 | |
| 711 extra_params = {} | |
| 712 if not six.PY3: # Python 2 | |
| 713 extra_params['strict'] = self.strict | |
| 714 connection = connection_class(host=actual_host, port=actual_port, | |
| 715 timeout=self.timeout.connect_timeout, | |
| 716 **extra_params) | |
| 717 | |
| 718 return self._prepare_conn(connection) | |
| 719 | |
| 720 | |
| 721 def connection_from_url(url, **kw): | |
| 722 """ | |
| 723 Given a url, return an :class:`.ConnectionPool` instance of its host. | |
| 724 | |
| 725 This is a shortcut for not having to parse out the scheme, host, and port | |
| 726 of the url before creating an :class:`.ConnectionPool` instance. | |
| 727 | |
| 728 :param url: | |
| 729 Absolute URL string that must include the scheme. Port is optional. | |
| 730 | |
| 731 :param \**kw: | |
| 732 Passes additional parameters to the constructor of the appropriate | |
| 733 :class:`.ConnectionPool`. Useful for specifying things like | |
| 734 timeout, maxsize, headers, etc. | |
| 735 | |
| 736 Example: :: | |
| 737 | |
| 738 >>> conn = connection_from_url('http://google.com/') | |
| 739 >>> r = conn.request('GET', '/') | |
| 740 """ | |
| 741 scheme, host, port = get_host(url) | |
| 742 if scheme == 'https': | |
| 743 return HTTPSConnectionPool(host, port=port, **kw) | |
| 744 else: | |
| 745 return HTTPConnectionPool(host, port=port, **kw) | |
| OLD | NEW |
|---|
Chromium Code Reviews
