Update 'requests' lib to v2.0.

third_party/requests/utils.py

 # -*- coding: utf-8 -*-
 
 """
 requests.utils
 ~~~~~~~~~~~~~~
 
 This module provides utility functions that are used within Requests
 that are also useful for external consumption.
 
 """
 
 import cgi
 import codecs
 import collections
 import os
 import platform
 import re
 import sys
 from netrc import netrc, NetrcParseError
 
 from . import __version__
 from . import certs
 from .compat import parse_http_list as _parse_list_header
-from .compat import quote, urlparse, bytes, str, OrderedDict, urlunparse
+from .compat import (quote, urlparse, bytes, str, OrderedDict, urlunparse,
+                     is_py2, is_py3, builtin_str, getproxies, proxy_bypass)
 from .cookies import RequestsCookieJar, cookiejar_from_dict
 from .structures import CaseInsensitiveDict
+from .exceptions import MissingSchema, InvalidURL
 
 _hush_pyflakes = (RequestsCookieJar,)
 
 NETRC_FILES = ('.netrc', '_netrc')
 
 DEFAULT_CA_BUNDLE_PATH = certs.where()
 
 
 def dict_to_sequence(d):
     """Returns an internal sequence dictionary update."""
@@ skipping 220 matching lines @@
     return cj
 
 
 def get_encodings_from_content(content):
     """Returns encodings from given content string.
 
     :param content: bytestring to extract encodings from.
     """
 
     charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
+    pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I)
+    xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]')
 
-    return charset_re.findall(content)
+    return (charset_re.findall(content) +
+            pragma_re.findall(content) +
+            xml_re.findall(content))
 
 
 def get_encoding_from_headers(headers):
     """Returns encodings from given HTTP Header Dict.
 
     :param headers: dictionary to extract encoding from.
     """
 
     content_type = headers.get('content-type')
 
@@ skipping 15 matching lines @@
     if r.encoding is None:
         for item in iterator:
             yield item
         return
 
     decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace')
     for chunk in iterator:
         rv = decoder.decode(chunk)
         if rv:
             yield rv
-    rv = decoder.decode('', final=True)
+    rv = decoder.decode(b'', final=True)
     if rv:
         yield rv
 
 
 def iter_slices(string, slice_length):
     """Iterate over slices of a string."""
     pos = 0
     while pos < len(string):
         yield string[pos:pos + slice_length]
         pos += slice_length
@@ skipping 39 matching lines @@
 
 
 def unquote_unreserved(uri):
     """Un-escape any percent-escape sequences in a URI that are unreserved
     characters. This leaves all reserved, illegal and non-ASCII bytes encoded.
     """
     parts = uri.split('%')
     for i in range(1, len(parts)):
         h = parts[i][0:2]
         if len(h) == 2 and h.isalnum():
-            c = chr(int(h, 16))
+            try:
+                c = chr(int(h, 16))
+            except ValueError:
+                raise InvalidURL("Invalid percent-escape sequence: '%s'" % h)
+
             if c in UNRESERVED_SET:
                 parts[i] = c + parts[i][2:]
             else:
                 parts[i] = '%' + parts[i]
         else:
             parts[i] = '%' + parts[i]
     return ''.join(parts)
 
 
 def requote_uri(uri):
     """Re-quote the given URI.
 
     This function passes the given URI through an unquote/quote cycle to
     ensure that it is fully and consistently quoted.
     """
     # Unquote only the unreserved characters
     # Then quote only illegal characters (do not quote reserved, unreserved,
     # or '%')
     return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~")
 
 
 def get_environ_proxies(url):
     """Return a dict of environment proxies."""
 
-    proxy_keys = [
-        'all',
-        'http',
-        'https',
-        'ftp',
-        'socks'
-    ]
-
     get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
 
     # First check whether no_proxy is defined. If it is, check that the URL
     # we're getting isn't in the no_proxy list.
     no_proxy = get_proxy('no_proxy')
+    netloc = urlparse(url).netloc
 
     if no_proxy:
         # We need to check whether we match here. We need to see if we match
         # the end of the netloc, both with and without the port.
-        no_proxy = no_proxy.split(',')
-        netloc = urlparse(url).netloc
+        no_proxy = no_proxy.replace(' ', '').split(',')
 
         for host in no_proxy:
             if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
                 # The URL does match something in no_proxy, so we don't want
                 # to apply the proxies on this URL.
                 return {}
 
+    # If the system proxy settings indicate that this URL should be bypassed,
+    # don't proxy.
+    if proxy_bypass(netloc):
+        return {}
+
     # If we get here, we either didn't have no_proxy set or we're not going
-    # anywhere that no_proxy applies to.
-    proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys]
-    return dict([(key, val) for (key, val) in proxies if val])
+    # anywhere that no_proxy applies to, and the system settings don't require
+    # bypassing the proxy for the current URL.
+    return getproxies()
 
 
 def default_user_agent():
     """Return a string representing the default user agent."""
     _implementation = platform.python_implementation()
 
     if _implementation == 'CPython':
         _implementation_version = platform.python_version()
     elif _implementation == 'PyPy':
         _implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major,
@@ skipping 90 matching lines @@
         # Did not detect 2 valid UTF-16 ascii-range characters
     if nullcount == 3:
         if sample[:3] == _null3:
             return 'utf-32-be'
         if sample[1:] == _null3:
             return 'utf-32-le'
         # Did not detect a valid UTF-32 ascii-range character
     return None
 
 
-def prepend_scheme_if_needed(url, new_scheme):
-    '''Given a URL that may or may not have a scheme, prepend the given scheme.
-    Does not replace a present scheme with the one provided as an argument.'''
-    scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme)
+def except_on_missing_scheme(url):
+    """Given a URL, raise a MissingSchema exception if the scheme is missing.
+    """
+    scheme, netloc, path, params, query, fragment = urlparse(url)
 
-    # urlparse is a finicky beast, and sometimes decides that there isn't a
-    # netloc present. Assume that it's being over-cautious, and switch netloc
-    # and path if urlparse decided there was no netloc.
-    if not netloc:
-        netloc, path = path, netloc
-
-    return urlunparse((scheme, netloc, path, params, query, fragment))
+    if not scheme:
+        raise MissingSchema('Proxy URLs must have explicit schemes.')
 
 
 def get_auth_from_url(url):
     """Given a url with authentication components, extract them into a tuple of
     username,password."""
     if url:
         parsed = urlparse(url)
         return (parsed.username, parsed.password)
     else:
         return ('', '')
+
+
+def to_native_string(string, encoding='ascii'):
+    """
+    Given a string object, regardless of type, returns a representation of that
+    string in the native string type, encoding and decoding where necessary.
+    This assumes ASCII unless told otherwise.
+    """
+    out = None
+
+    if isinstance(string, builtin_str):
+        out = string
+    else:
+        if is_py2:
+            out = string.encode(encoding)
+        else:
+            out = string.decode(encoding)
+
+    return out