Add 'requests' library to third_party.

third_party/requests/packages/urllib3/connectionpool.py

+# urllib3/connectionpool.py
+# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+#
+# This module is part of urllib3 and is released under
+# the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+import logging
+import socket
+import errno
+
+from socket import error as SocketError, timeout as SocketTimeout
+from .util import resolve_cert_reqs, resolve_ssl_version, assert_fingerprint
+
+try: # Python 3
+    from http.client import HTTPConnection, HTTPException
+    from http.client import HTTP_PORT, HTTPS_PORT
+except ImportError:
+    from httplib import HTTPConnection, HTTPException
+    from httplib import HTTP_PORT, HTTPS_PORT
+
+try: # Python 3
+    from queue import LifoQueue, Empty, Full
+except ImportError:
+    from Queue import LifoQueue, Empty, Full
+
+
+try: # Compiled with SSL?
+    HTTPSConnection = object
+    BaseSSLError = None
+    ssl = None
+
+    try: # Python 3
+        from http.client import HTTPSConnection
+    except ImportError:
+        from httplib import HTTPSConnection
+
+    import ssl
+    BaseSSLError = ssl.SSLError
+
+except (ImportError, AttributeError): # Platform-specific: No SSL.
+    pass
+
+
+from .request import RequestMethods
+from .response import HTTPResponse
+from .util import get_host, is_connection_dropped, ssl_wrap_socket
+from .exceptions import (
+    ClosedPoolError,
+    EmptyPoolError,
+    HostChangedError,
+    MaxRetryError,
+    SSLError,
+    TimeoutError,
+)
+
+from .packages.ssl_match_hostname import match_hostname, CertificateError
+from .packages import six
+
+
+xrange = six.moves.xrange
+
+log = logging.getLogger(__name__)
+
+_Default = object()
+
+port_by_scheme = {
+    'http': HTTP_PORT,
+    'https': HTTPS_PORT,
+}
+
+
+## Connection objects (extension of httplib)
+
+class VerifiedHTTPSConnection(HTTPSConnection):
+    """
+    Based on httplib.HTTPSConnection but wraps the socket with
+    SSL certification.
+    """
+    cert_reqs = None
+    ca_certs = None
+    ssl_version = None
+
+    def set_cert(self, key_file=None, cert_file=None,
+                 cert_reqs=None, ca_certs=None,
+                 assert_hostname=None, assert_fingerprint=None):
+
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.cert_reqs = cert_reqs
+        self.ca_certs = ca_certs
+        self.assert_hostname = assert_hostname
+        self.assert_fingerprint = assert_fingerprint
+
+    def connect(self):
+        # Add certificate verification
+        sock = socket.create_connection((self.host, self.port), self.timeout)
+
+        resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
+        resolved_ssl_version = resolve_ssl_version(self.ssl_version)
+
+        # Wrap socket using verification with the root certs in
+        # trusted_root_certs
+        self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file,
+                                    cert_reqs=resolved_cert_reqs,
+                                    ca_certs=self.ca_certs,
+                                    server_hostname=self.host,
+                                    ssl_version=resolved_ssl_version)
+
+        if resolved_cert_reqs != ssl.CERT_NONE:
+            if self.assert_fingerprint:
+                assert_fingerprint(self.sock.getpeercert(binary_form=True),
+                                   self.assert_fingerprint)
+            else:
+                match_hostname(self.sock.getpeercert(),
+                               self.assert_hostname or self.host)
+
+## Pool objects
+
+class ConnectionPool(object):
+    """
+    Base class for all connection pools, such as
+    :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
+    """
+
+    scheme = None
+    QueueCls = LifoQueue
+
+    def __init__(self, host, port=None):
+        self.host = host
+        self.port = port
+
+    def __str__(self):
+        return '%s(host=%r, port=%r)' % (type(self).__name__,
+                                         self.host, self.port)
+
+
+class HTTPConnectionPool(ConnectionPool, RequestMethods):
+    """
+    Thread-safe connection pool for one host.
+
+    :param host:
+        Host used for this HTTP Connection (e.g. "localhost"), passed into
+        :class:`httplib.HTTPConnection`.
+
+    :param port:
+        Port used for this HTTP Connection (None is equivalent to 80), passed
+        into :class:`httplib.HTTPConnection`.
+
+    :param strict:
+        Causes BadStatusLine to be raised if the status line can't be parsed
+        as a valid HTTP/1.0 or 1.1 status line, passed into
+        :class:`httplib.HTTPConnection`.
+
+    :param timeout:
+        Socket timeout for each individual connection, can be a float. None
+        disables timeout.
+
+    :param maxsize:
+        Number of connections to save that can be reused. More than 1 is useful
+        in multithreaded situations. If ``block`` is set to false, more
+        connections will be created but they will not be saved once they've
+        been used.
+
+    :param block:
+        If set to True, no more than ``maxsize`` connections will be used at
+        a time. When no free connections are available, the call will block
+        until a connection has been released. This is a useful side effect for
+        particular multithreaded situations where one does not want to use more
+        than maxsize connections per host to prevent flooding.
+
+    :param headers:
+        Headers to include with all requests, unless other headers are given
+        explicitly.
+    """
+
+    scheme = 'http'
+
+    def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1,
+                 block=False, headers=None):
+        ConnectionPool.__init__(self, host, port)
+        RequestMethods.__init__(self, headers)
+
+        self.strict = strict
+        self.timeout = timeout
+        self.pool = self.QueueCls(maxsize)
+        self.block = block
+
+        # Fill the queue up so that doing get() on it will block properly
+        for _ in xrange(maxsize):
+            self.pool.put(None)
+
+        # These are mostly for testing and debugging purposes.
+        self.num_connections = 0
+        self.num_requests = 0
+
+    def _new_conn(self):
+        """
+        Return a fresh :class:`httplib.HTTPConnection`.
+        """
+        self.num_connections += 1
+        log.info("Starting new HTTP connection (%d): %s" %
+                 (self.num_connections, self.host))
+        return HTTPConnection(host=self.host,
+                              port=self.port,
+                              strict=self.strict)
+
+    def _get_conn(self, timeout=None):
+        """
+        Get a connection. Will return a pooled connection if one is available.
+
+        If no connections are available and :prop:`.block` is ``False``, then a
+        fresh connection is returned.
+
+        :param timeout:
+            Seconds to wait before giving up and raising
+            :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
+            :prop:`.block` is ``True``.
+        """
+        conn = None
+        try:
+            conn = self.pool.get(block=self.block, timeout=timeout)
+
+        except AttributeError: # self.pool is None
+            raise ClosedPoolError(self, "Pool is closed.")
+
+        except Empty:
+            if self.block:
+                raise EmptyPoolError(self,
+                                     "Pool reached maximum size and no more "
+                                     "connections are allowed.")
+            pass  # Oh well, we'll create a new connection then
+
+        # If this is a persistent connection, check if it got disconnected
+        if conn and is_connection_dropped(conn):
+            log.info("Resetting dropped connection: %s" % self.host)
+            conn.close()
+
+        return conn or self._new_conn()
+
+    def _put_conn(self, conn):
+        """
+        Put a connection back into the pool.
+
+        :param conn:
+            Connection object for the current host and port as returned by
+            :meth:`._new_conn` or :meth:`._get_conn`.
+
+        If the pool is already full, the connection is closed and discarded
+        because we exceeded maxsize. If connections are discarded frequently,
+        then maxsize should be increased.
+
+        If the pool is closed, then the connection will be closed and discarded.
+        """
+        try:
+            self.pool.put(conn, block=False)
+            return # Everything is dandy, done.
+        except AttributeError:
+            # self.pool is None.
+            pass
+        except Full:
+            # This should never happen if self.block == True
+            log.warning("HttpConnectionPool is full, discarding connection: %s"
+                        % self.host)
+
+        # Connection never got put back into the pool, close it.
+        conn.close()
+
+    def _make_request(self, conn, method, url, timeout=_Default,
+                      **httplib_request_kw):
+        """
+        Perform a request on a given httplib connection object taken from our
+        pool.
+        """
+        self.num_requests += 1
+
+        if timeout is _Default:
+            timeout = self.timeout
+
+        conn.timeout = timeout # This only does anything in Py26+
+        conn.request(method, url, **httplib_request_kw)
+
+        # Set timeout
+        sock = getattr(conn, 'sock', False) # AppEngine doesn't have sock attr.
+        if sock:
+            sock.settimeout(timeout)
+
+        try: # Python 2.7+, use buffering of HTTP responses
+            httplib_response = conn.getresponse(buffering=True)
+        except TypeError: # Python 2.6 and older
+            httplib_response = conn.getresponse()
+
+        # AppEngine doesn't have a version attr.
+        http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
+        log.debug("\"%s %s %s\" %s %s" % (method, url, http_version,
+                                          httplib_response.status,
+                                          httplib_response.length))
+        return httplib_response
+
+    def close(self):
+        """
+        Close all pooled connections and disable the pool.
+        """
+        # Disable access to the pool
+        old_pool, self.pool = self.pool, None
+
+        try:
+            while True:
+                conn = old_pool.get(block=False)
+                if conn:
+                    conn.close()
+
+        except Empty:
+            pass # Done.
+
+    def is_same_host(self, url):
+        """
+        Check if the given ``url`` is a member of the same host as this
+        connection pool.
+        """
+        if url.startswith('/'):
+            return True
+
+        # TODO: Add optional support for socket.gethostbyname checking.
+        scheme, host, port = get_host(url)
+
+        if self.port and not port:
+            # Use explicit default port for comparison when none is given.
+            port = port_by_scheme.get(scheme)
+
+        return (scheme, host, port) == (self.scheme, self.host, self.port)
+
+    def urlopen(self, method, url, body=None, headers=None, retries=3,
+                redirect=True, assert_same_host=True, timeout=_Default,
+                pool_timeout=None, release_conn=None, **response_kw):
+        """
+        Get a connection from the pool and perform an HTTP request. This is the
+        lowest level call for making a request, so you'll need to specify all
+        the raw details.
+
+        .. note::
+
+           More commonly, it's appropriate to use a convenience method provided
+           by :class:`.RequestMethods`, such as :meth:`request`.
+
+        .. note::
+
+           `release_conn` will only behave as expected if
+           `preload_content=False` because we want to make
+           `preload_content=False` the default behaviour someday soon without
+           breaking backwards compatibility.
+
+        :param method:
+            HTTP request method (such as GET, POST, PUT, etc.)
+
+        :param body:
+            Data to send in the request body (useful for creating
+            POST requests, see HTTPConnectionPool.post_url for
+            more convenience).
+
+        :param headers:
+            Dictionary of custom headers to send, such as User-Agent,
+            If-None-Match, etc. If None, pool headers are used. If provided,
+            these headers completely replace any pool-specific headers.
+
+        :param retries:
+            Number of retries to allow before raising a MaxRetryError exception.
+
+        :param redirect:
+            If True, automatically handle redirects (status codes 301, 302,
+            303, 307). Each redirect counts as a retry.
+
+        :param assert_same_host:
+            If ``True``, will make sure that the host of the pool requests is
+            consistent else will raise HostChangedError. When False, you can
+            use the pool on an HTTP proxy and request foreign hosts.
+
+        :param timeout:
+            If specified, overrides the default timeout for this one request.
+
+        :param pool_timeout:
+            If set and the pool is set to block=True, then this method will
+            block for ``pool_timeout`` seconds and raise EmptyPoolError if no
+            connection is available within the time period.
+
+        :param release_conn:
+            If False, then the urlopen call will not release the connection
+            back into the pool once a response is received (but will release if
+            you read the entire contents of the response such as when
+            `preload_content=True`). This is useful if you're not preloading
+            the response's content immediately. You will need to call
+            ``r.release_conn()`` on the response ``r`` to return the connection
+            back into the pool. If None, it takes the value of
+            ``response_kw.get('preload_content', True)``.
+
+        :param \**response_kw:
+            Additional parameters are passed to
+            :meth:`urllib3.response.HTTPResponse.from_httplib`
+        """
+        if headers is None:
+            headers = self.headers
+
+        if retries < 0:
+            raise MaxRetryError(self, url)
+
+        if timeout is _Default:
+            timeout = self.timeout
+
+        if release_conn is None:
+            release_conn = response_kw.get('preload_content', True)
+
+        # Check host
+        if assert_same_host and not self.is_same_host(url):
+            host = "%s://%s" % (self.scheme, self.host)
+            if self.port:
+                host = "%s:%d" % (host, self.port)
+
+            raise HostChangedError(self, url, retries - 1)
+
+        conn = None
+
+        try:
+            # Request a connection from the queue
+            conn = self._get_conn(timeout=pool_timeout)
+
+            # Make the request on the httplib connection object
+            httplib_response = self._make_request(conn, method, url,
+                                                  timeout=timeout,
+                                                  body=body, headers=headers)
+
+            # If we're going to release the connection in ``finally:``, then
+            # the request doesn't need to know about the connection. Otherwise
+            # it will also try to release it and we'll have a double-release
+            # mess.
+            response_conn = not release_conn and conn
+
+            # Import httplib's response into our own wrapper object
+            response = HTTPResponse.from_httplib(httplib_response,
+                                                 pool=self,
+                                                 connection=response_conn,
+                                                 **response_kw)
+
+            # else:
+            #     The connection will be put back into the pool when
+            #     ``response.release_conn()`` is called (implicitly by
+            #     ``response.read()``)
+
+        except Empty as e:
+            # Timed out by queue
+            raise TimeoutError(self, url,
+                               "Request timed out. (pool_timeout=%s)" %
+                               pool_timeout)
+
+        except SocketTimeout as e:
+            # Timed out by socket
+            raise TimeoutError(self, url,
+                               "Request timed out. (timeout=%s)" %
+                               timeout)
+
+        except BaseSSLError as e:
+            # SSL certificate error
+            raise SSLError(e)
+
+        except CertificateError as e:
+            # Name mismatch
+            raise SSLError(e)
+
+        except (HTTPException, SocketError) as e:
+            # Connection broken, discard. It will be replaced next _get_conn().
+            conn = None
+            # This is necessary so we can access e below
+            err = e
+
+            if retries == 0:
+                raise MaxRetryError(self, url, e)
+
+        finally:
+            if release_conn:
+                # Put the connection back to be reused. If the connection is
+                # expired then it will be None, which will get replaced with a
+                # fresh connection during _get_conn.
+                self._put_conn(conn)
+
+        if not conn:
+            # Try again
+            log.warn("Retrying (%d attempts remain) after connection "
+                     "broken by '%r': %s" % (retries, err, url))
+            return self.urlopen(method, url, body, headers, retries - 1,
+                                redirect, assert_same_host,
+                                timeout=timeout, pool_timeout=pool_timeout,
+                                release_conn=release_conn, **response_kw)
+
+        # Handle redirect?
+        redirect_location = redirect and response.get_redirect_location()
+        if redirect_location:
+            if response.status == 303:
+                method = 'GET'
+            log.info("Redirecting %s -> %s" % (url, redirect_location))
+            return self.urlopen(method, redirect_location, body, headers,
+                                retries - 1, redirect, assert_same_host,
+                                timeout=timeout, pool_timeout=pool_timeout,
+                                release_conn=release_conn, **response_kw)
+
+        return response
+
+
+class HTTPSConnectionPool(HTTPConnectionPool):
+    """
+    Same as :class:`.HTTPConnectionPool`, but HTTPS.
+
+    When Python is compiled with the :mod:`ssl` module, then
+    :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates,
+    instead of :class:`httplib.HTTPSConnection`.
+
+    :class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``,
+    ``assert_hostname`` and ``host`` in this order to verify connections.
+
+    The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs`` and
+    ``ssl_version`` are only used if :mod:`ssl` is available and are fed into
+    :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket
+    into an SSL socket.
+    """
+
+    scheme = 'https'
+
+    def __init__(self, host, port=None,
+                 strict=False, timeout=None, maxsize=1,
+                 block=False, headers=None,
+                 key_file=None, cert_file=None, cert_reqs=None,
+                 ca_certs=None, ssl_version=None,
+                 assert_hostname=None, assert_fingerprint=None):
+
+        HTTPConnectionPool.__init__(self, host, port,
+                                    strict, timeout, maxsize,
+                                    block, headers)
+        self.key_file = key_file
+        self.cert_file = cert_file
+        self.cert_reqs = cert_reqs
+        self.ca_certs = ca_certs
+        self.ssl_version = ssl_version
+        self.assert_hostname = assert_hostname
+        self.assert_fingerprint = assert_fingerprint
+
+    def _new_conn(self):
+        """
+        Return a fresh :class:`httplib.HTTPSConnection`.
+        """
+        self.num_connections += 1
+        log.info("Starting new HTTPS connection (%d): %s"
+                 % (self.num_connections, self.host))
+
+        if not ssl:  # Platform-specific: Python compiled without +ssl
+            if not HTTPSConnection or HTTPSConnection is object:
+                raise SSLError("Can't connect to HTTPS URL because the SSL "
+                               "module is not available.")
+
+            return HTTPSConnection(host=self.host,
+                                   port=self.port,
+                                   strict=self.strict)
+
+        connection = VerifiedHTTPSConnection(host=self.host,
+                                             port=self.port,
+                                             strict=self.strict)
+        connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
+                            cert_reqs=self.cert_reqs, ca_certs=self.ca_certs,
+                            assert_hostname=self.assert_hostname,
+                            assert_fingerprint=self.assert_fingerprint)
+
+        connection.ssl_version = self.ssl_version
+
+        return connection
+
+
+def connection_from_url(url, **kw):
+    """
+    Given a url, return an :class:`.ConnectionPool` instance of its host.
+
+    This is a shortcut for not having to parse out the scheme, host, and port
+    of the url before creating an :class:`.ConnectionPool` instance.
+
+    :param url:
+        Absolute URL string that must include the scheme. Port is optional.
+
+    :param \**kw:
+        Passes additional parameters to the constructor of the appropriate
+        :class:`.ConnectionPool`. Useful for specifying things like
+        timeout, maxsize, headers, etc.
+
+    Example: ::
+
+        >>> conn = connection_from_url('http://google.com/')
+        >>> r = conn.request('GET', '/')
+    """
+    scheme, host, port = get_host(url)
+    if scheme == 'https':
+        return HTTPSConnectionPool(host, port=port, **kw)
+    else:
+        return HTTPConnectionPool(host, port=port, **kw)