Index: third_party/google-endpoints/requests/utils.py |
diff --git a/third_party/google-endpoints/requests/utils.py b/third_party/google-endpoints/requests/utils.py |
new file mode 100644 |
index 0000000000000000000000000000000000000000..47325090ce6285977e6ea381479c8f544d9bc482 |
--- /dev/null |
+++ b/third_party/google-endpoints/requests/utils.py |
@@ -0,0 +1,827 @@ |
+# -*- coding: utf-8 -*- |
+ |
+""" |
+requests.utils |
+~~~~~~~~~~~~~~ |
+ |
+This module provides utility functions that are used within Requests |
+that are also useful for external consumption. |
+""" |
+ |
+import cgi |
+import codecs |
+import collections |
+import io |
+import os |
+import re |
+import socket |
+import struct |
+import warnings |
+ |
+from . import __version__ |
+from . import certs |
+# to_native_string is unused here, but imported here for backwards compatibility |
+from ._internal_utils import to_native_string |
+from .compat import parse_http_list as _parse_list_header |
+from .compat import ( |
+ quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, |
+ proxy_bypass, urlunparse, basestring, integer_types) |
+from .cookies import RequestsCookieJar, cookiejar_from_dict |
+from .structures import CaseInsensitiveDict |
+from .exceptions import ( |
+ InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError) |
+ |
+_hush_pyflakes = (RequestsCookieJar,) |
+ |
+NETRC_FILES = ('.netrc', '_netrc') |
+ |
+DEFAULT_CA_BUNDLE_PATH = certs.where() |
+ |
+ |
+def dict_to_sequence(d): |
+ """Returns an internal sequence dictionary update.""" |
+ |
+ if hasattr(d, 'items'): |
+ d = d.items() |
+ |
+ return d |
+ |
+ |
+def super_len(o): |
+ total_length = None |
+ current_position = 0 |
+ |
+ if hasattr(o, '__len__'): |
+ total_length = len(o) |
+ |
+ elif hasattr(o, 'len'): |
+ total_length = o.len |
+ |
+ elif hasattr(o, 'fileno'): |
+ try: |
+ fileno = o.fileno() |
+ except io.UnsupportedOperation: |
+ pass |
+ else: |
+ total_length = os.fstat(fileno).st_size |
+ |
+ # Having used fstat to determine the file length, we need to |
+ # confirm that this file was opened up in binary mode. |
+ if 'b' not in o.mode: |
+ warnings.warn(( |
+ "Requests has determined the content-length for this " |
+ "request using the binary size of the file: however, the " |
+ "file has been opened in text mode (i.e. without the 'b' " |
+ "flag in the mode). This may lead to an incorrect " |
+ "content-length. In Requests 3.0, support will be removed " |
+ "for files in text mode."), |
+ FileModeWarning |
+ ) |
+ |
+ if hasattr(o, 'tell'): |
+ try: |
+ current_position = o.tell() |
+ except (OSError, IOError): |
+ # This can happen in some weird situations, such as when the file |
+ # is actually a special file descriptor like stdin. In this |
+ # instance, we don't know what the length is, so set it to zero and |
+ # let requests chunk it instead. |
+ if total_length is not None: |
+ current_position = total_length |
+ else: |
+ if hasattr(o, 'seek') and total_length is None: |
+ # StringIO and BytesIO have seek but no useable fileno |
+ |
+ # seek to end of file |
+ o.seek(0, 2) |
+ total_length = o.tell() |
+ |
+ # seek back to current position to support |
+ # partially read file-like objects |
+ o.seek(current_position or 0) |
+ |
+ if total_length is None: |
+ total_length = 0 |
+ |
+ return max(0, total_length - current_position) |
+ |
+ |
+def get_netrc_auth(url, raise_errors=False): |
+ """Returns the Requests tuple auth for a given url from netrc.""" |
+ |
+ try: |
+ from netrc import netrc, NetrcParseError |
+ |
+ netrc_path = None |
+ |
+ for f in NETRC_FILES: |
+ try: |
+ loc = os.path.expanduser('~/{0}'.format(f)) |
+ except KeyError: |
+ # os.path.expanduser can fail when $HOME is undefined and |
+ # getpwuid fails. See http://bugs.python.org/issue20164 & |
+ # https://github.com/kennethreitz/requests/issues/1846 |
+ return |
+ |
+ if os.path.exists(loc): |
+ netrc_path = loc |
+ break |
+ |
+ # Abort early if there isn't one. |
+ if netrc_path is None: |
+ return |
+ |
+ ri = urlparse(url) |
+ |
+ # Strip port numbers from netloc. This weird `if...encode`` dance is |
+ # used for Python 3.2, which doesn't support unicode literals. |
+ splitstr = b':' |
+ if isinstance(url, str): |
+ splitstr = splitstr.decode('ascii') |
+ host = ri.netloc.split(splitstr)[0] |
+ |
+ try: |
+ _netrc = netrc(netrc_path).authenticators(host) |
+ if _netrc: |
+ # Return with login / password |
+ login_i = (0 if _netrc[0] else 1) |
+ return (_netrc[login_i], _netrc[2]) |
+ except (NetrcParseError, IOError): |
+ # If there was a parsing error or a permissions issue reading the file, |
+ # we'll just skip netrc auth unless explicitly asked to raise errors. |
+ if raise_errors: |
+ raise |
+ |
+ # AppEngine hackiness. |
+ except (ImportError, AttributeError): |
+ pass |
+ |
+ |
+def guess_filename(obj): |
+ """Tries to guess the filename of the given object.""" |
+ name = getattr(obj, 'name', None) |
+ if (name and isinstance(name, basestring) and name[0] != '<' and |
+ name[-1] != '>'): |
+ return os.path.basename(name) |
+ |
+ |
+def from_key_val_list(value): |
+ """Take an object and test to see if it can be represented as a |
+ dictionary. Unless it can not be represented as such, return an |
+ OrderedDict, e.g., |
+ |
+ :: |
+ |
+ >>> from_key_val_list([('key', 'val')]) |
+ OrderedDict([('key', 'val')]) |
+ >>> from_key_val_list('string') |
+ ValueError: need more than 1 value to unpack |
+ >>> from_key_val_list({'key': 'val'}) |
+ OrderedDict([('key', 'val')]) |
+ |
+ :rtype: OrderedDict |
+ """ |
+ if value is None: |
+ return None |
+ |
+ if isinstance(value, (str, bytes, bool, int)): |
+ raise ValueError('cannot encode objects that are not 2-tuples') |
+ |
+ return OrderedDict(value) |
+ |
+ |
+def to_key_val_list(value): |
+ """Take an object and test to see if it can be represented as a |
+ dictionary. If it can be, return a list of tuples, e.g., |
+ |
+ :: |
+ |
+ >>> to_key_val_list([('key', 'val')]) |
+ [('key', 'val')] |
+ >>> to_key_val_list({'key': 'val'}) |
+ [('key', 'val')] |
+ >>> to_key_val_list('string') |
+ ValueError: cannot encode objects that are not 2-tuples. |
+ |
+ :rtype: list |
+ """ |
+ if value is None: |
+ return None |
+ |
+ if isinstance(value, (str, bytes, bool, int)): |
+ raise ValueError('cannot encode objects that are not 2-tuples') |
+ |
+ if isinstance(value, collections.Mapping): |
+ value = value.items() |
+ |
+ return list(value) |
+ |
+ |
+# From mitsuhiko/werkzeug (used with permission). |
+def parse_list_header(value): |
+ """Parse lists as described by RFC 2068 Section 2. |
+ |
+ In particular, parse comma-separated lists where the elements of |
+ the list may include quoted-strings. A quoted-string could |
+ contain a comma. A non-quoted string could have quotes in the |
+ middle. Quotes are removed automatically after parsing. |
+ |
+ It basically works like :func:`parse_set_header` just that items |
+ may appear multiple times and case sensitivity is preserved. |
+ |
+ The return value is a standard :class:`list`: |
+ |
+ >>> parse_list_header('token, "quoted value"') |
+ ['token', 'quoted value'] |
+ |
+ To create a header from the :class:`list` again, use the |
+ :func:`dump_header` function. |
+ |
+ :param value: a string with a list header. |
+ :return: :class:`list` |
+ :rtype: list |
+ """ |
+ result = [] |
+ for item in _parse_list_header(value): |
+ if item[:1] == item[-1:] == '"': |
+ item = unquote_header_value(item[1:-1]) |
+ result.append(item) |
+ return result |
+ |
+ |
+# From mitsuhiko/werkzeug (used with permission). |
+def parse_dict_header(value): |
+ """Parse lists of key, value pairs as described by RFC 2068 Section 2 and |
+ convert them into a python dict: |
+ |
+ >>> d = parse_dict_header('foo="is a fish", bar="as well"') |
+ >>> type(d) is dict |
+ True |
+ >>> sorted(d.items()) |
+ [('bar', 'as well'), ('foo', 'is a fish')] |
+ |
+ If there is no value for a key it will be `None`: |
+ |
+ >>> parse_dict_header('key_without_value') |
+ {'key_without_value': None} |
+ |
+ To create a header from the :class:`dict` again, use the |
+ :func:`dump_header` function. |
+ |
+ :param value: a string with a dict header. |
+ :return: :class:`dict` |
+ :rtype: dict |
+ """ |
+ result = {} |
+ for item in _parse_list_header(value): |
+ if '=' not in item: |
+ result[item] = None |
+ continue |
+ name, value = item.split('=', 1) |
+ if value[:1] == value[-1:] == '"': |
+ value = unquote_header_value(value[1:-1]) |
+ result[name] = value |
+ return result |
+ |
+ |
+# From mitsuhiko/werkzeug (used with permission). |
+def unquote_header_value(value, is_filename=False): |
+ r"""Unquotes a header value. (Reversal of :func:`quote_header_value`). |
+ This does not use the real unquoting but what browsers are actually |
+ using for quoting. |
+ |
+ :param value: the header value to unquote. |
+ :rtype: str |
+ """ |
+ if value and value[0] == value[-1] == '"': |
+ # this is not the real unquoting, but fixing this so that the |
+ # RFC is met will result in bugs with internet explorer and |
+ # probably some other browsers as well. IE for example is |
+ # uploading files with "C:\foo\bar.txt" as filename |
+ value = value[1:-1] |
+ |
+ # if this is a filename and the starting characters look like |
+ # a UNC path, then just return the value without quotes. Using the |
+ # replace sequence below on a UNC path has the effect of turning |
+ # the leading double slash into a single slash and then |
+ # _fix_ie_filename() doesn't work correctly. See #458. |
+ if not is_filename or value[:2] != '\\\\': |
+ return value.replace('\\\\', '\\').replace('\\"', '"') |
+ return value |
+ |
+ |
+def dict_from_cookiejar(cj): |
+ """Returns a key/value dictionary from a CookieJar. |
+ |
+ :param cj: CookieJar object to extract cookies from. |
+ :rtype: dict |
+ """ |
+ |
+ cookie_dict = {} |
+ |
+ for cookie in cj: |
+ cookie_dict[cookie.name] = cookie.value |
+ |
+ return cookie_dict |
+ |
+ |
+def add_dict_to_cookiejar(cj, cookie_dict): |
+ """Returns a CookieJar from a key/value dictionary. |
+ |
+ :param cj: CookieJar to insert cookies into. |
+ :param cookie_dict: Dict of key/values to insert into CookieJar. |
+ :rtype: CookieJar |
+ """ |
+ |
+ return cookiejar_from_dict(cookie_dict, cj) |
+ |
+ |
+def get_encodings_from_content(content): |
+ """Returns encodings from given content string. |
+ |
+ :param content: bytestring to extract encodings from. |
+ """ |
+ warnings.warn(( |
+ 'In requests 3.0, get_encodings_from_content will be removed. For ' |
+ 'more information, please see the discussion on issue #2266. (This' |
+ ' warning should only appear once.)'), |
+ DeprecationWarning) |
+ |
+ charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I) |
+ pragma_re = re.compile(r'<meta.*?content=["\']*;?charset=(.+?)["\'>]', flags=re.I) |
+ xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]') |
+ |
+ return (charset_re.findall(content) + |
+ pragma_re.findall(content) + |
+ xml_re.findall(content)) |
+ |
+ |
+def get_encoding_from_headers(headers): |
+ """Returns encodings from given HTTP Header Dict. |
+ |
+ :param headers: dictionary to extract encoding from. |
+ :rtype: str |
+ """ |
+ |
+ content_type = headers.get('content-type') |
+ |
+ if not content_type: |
+ return None |
+ |
+ content_type, params = cgi.parse_header(content_type) |
+ |
+ if 'charset' in params: |
+ return params['charset'].strip("'\"") |
+ |
+ if 'text' in content_type: |
+ return 'ISO-8859-1' |
+ |
+ |
+def stream_decode_response_unicode(iterator, r): |
+ """Stream decodes a iterator.""" |
+ |
+ if r.encoding is None: |
+ for item in iterator: |
+ yield item |
+ return |
+ |
+ decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace') |
+ for chunk in iterator: |
+ rv = decoder.decode(chunk) |
+ if rv: |
+ yield rv |
+ rv = decoder.decode(b'', final=True) |
+ if rv: |
+ yield rv |
+ |
+ |
+def iter_slices(string, slice_length): |
+ """Iterate over slices of a string.""" |
+ pos = 0 |
+ if slice_length is None or slice_length <= 0: |
+ slice_length = len(string) |
+ while pos < len(string): |
+ yield string[pos:pos + slice_length] |
+ pos += slice_length |
+ |
+ |
+def get_unicode_from_response(r): |
+ """Returns the requested content back in unicode. |
+ |
+ :param r: Response object to get unicode content from. |
+ |
+ Tried: |
+ |
+ 1. charset from content-type |
+ 2. fall back and replace all unicode characters |
+ |
+ :rtype: str |
+ """ |
+ warnings.warn(( |
+ 'In requests 3.0, get_unicode_from_response will be removed. For ' |
+ 'more information, please see the discussion on issue #2266. (This' |
+ ' warning should only appear once.)'), |
+ DeprecationWarning) |
+ |
+ tried_encodings = [] |
+ |
+ # Try charset from content-type |
+ encoding = get_encoding_from_headers(r.headers) |
+ |
+ if encoding: |
+ try: |
+ return str(r.content, encoding) |
+ except UnicodeError: |
+ tried_encodings.append(encoding) |
+ |
+ # Fall back: |
+ try: |
+ return str(r.content, encoding, errors='replace') |
+ except TypeError: |
+ return r.content |
+ |
+ |
+# The unreserved URI characters (RFC 3986) |
+UNRESERVED_SET = frozenset( |
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" |
+ + "0123456789-._~") |
+ |
+ |
+def unquote_unreserved(uri): |
+ """Un-escape any percent-escape sequences in a URI that are unreserved |
+ characters. This leaves all reserved, illegal and non-ASCII bytes encoded. |
+ |
+ :rtype: str |
+ """ |
+ parts = uri.split('%') |
+ for i in range(1, len(parts)): |
+ h = parts[i][0:2] |
+ if len(h) == 2 and h.isalnum(): |
+ try: |
+ c = chr(int(h, 16)) |
+ except ValueError: |
+ raise InvalidURL("Invalid percent-escape sequence: '%s'" % h) |
+ |
+ if c in UNRESERVED_SET: |
+ parts[i] = c + parts[i][2:] |
+ else: |
+ parts[i] = '%' + parts[i] |
+ else: |
+ parts[i] = '%' + parts[i] |
+ return ''.join(parts) |
+ |
+ |
+def requote_uri(uri): |
+ """Re-quote the given URI. |
+ |
+ This function passes the given URI through an unquote/quote cycle to |
+ ensure that it is fully and consistently quoted. |
+ |
+ :rtype: str |
+ """ |
+ safe_with_percent = "!#$%&'()*+,/:;=?@[]~" |
+ safe_without_percent = "!#$&'()*+,/:;=?@[]~" |
+ try: |
+ # Unquote only the unreserved characters |
+ # Then quote only illegal characters (do not quote reserved, |
+ # unreserved, or '%') |
+ return quote(unquote_unreserved(uri), safe=safe_with_percent) |
+ except InvalidURL: |
+ # We couldn't unquote the given URI, so let's try quoting it, but |
+ # there may be unquoted '%'s in the URI. We need to make sure they're |
+ # properly quoted so they do not cause issues elsewhere. |
+ return quote(uri, safe=safe_without_percent) |
+ |
+ |
+def address_in_network(ip, net): |
+ """This function allows you to check if on IP belongs to a network subnet |
+ |
+ Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 |
+ returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 |
+ |
+ :rtype: bool |
+ """ |
+ ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0] |
+ netaddr, bits = net.split('/') |
+ netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0] |
+ network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask |
+ return (ipaddr & netmask) == (network & netmask) |
+ |
+ |
+def dotted_netmask(mask): |
+ """Converts mask from /xx format to xxx.xxx.xxx.xxx |
+ |
+ Example: if mask is 24 function returns 255.255.255.0 |
+ |
+ :rtype: str |
+ """ |
+ bits = 0xffffffff ^ (1 << 32 - mask) - 1 |
+ return socket.inet_ntoa(struct.pack('>I', bits)) |
+ |
+ |
+def is_ipv4_address(string_ip): |
+ """ |
+ :rtype: bool |
+ """ |
+ try: |
+ socket.inet_aton(string_ip) |
+ except socket.error: |
+ return False |
+ return True |
+ |
+ |
+def is_valid_cidr(string_network): |
+ """ |
+ Very simple check of the cidr format in no_proxy variable. |
+ |
+ :rtype: bool |
+ """ |
+ if string_network.count('/') == 1: |
+ try: |
+ mask = int(string_network.split('/')[1]) |
+ except ValueError: |
+ return False |
+ |
+ if mask < 1 or mask > 32: |
+ return False |
+ |
+ try: |
+ socket.inet_aton(string_network.split('/')[0]) |
+ except socket.error: |
+ return False |
+ else: |
+ return False |
+ return True |
+ |
+ |
+def should_bypass_proxies(url): |
+ """ |
+ Returns whether we should bypass proxies or not. |
+ |
+ :rtype: bool |
+ """ |
+ get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) |
+ |
+ # First check whether no_proxy is defined. If it is, check that the URL |
+ # we're getting isn't in the no_proxy list. |
+ no_proxy = get_proxy('no_proxy') |
+ netloc = urlparse(url).netloc |
+ |
+ if no_proxy: |
+ # We need to check whether we match here. We need to see if we match |
+ # the end of the netloc, both with and without the port. |
+ no_proxy = ( |
+ host for host in no_proxy.replace(' ', '').split(',') if host |
+ ) |
+ |
+ ip = netloc.split(':')[0] |
+ if is_ipv4_address(ip): |
+ for proxy_ip in no_proxy: |
+ if is_valid_cidr(proxy_ip): |
+ if address_in_network(ip, proxy_ip): |
+ return True |
+ elif ip == proxy_ip: |
+ # If no_proxy ip was defined in plain IP notation instead of cidr notation & |
+ # matches the IP of the index |
+ return True |
+ else: |
+ for host in no_proxy: |
+ if netloc.endswith(host) or netloc.split(':')[0].endswith(host): |
+ # The URL does match something in no_proxy, so we don't want |
+ # to apply the proxies on this URL. |
+ return True |
+ |
+ # If the system proxy settings indicate that this URL should be bypassed, |
+ # don't proxy. |
+ # The proxy_bypass function is incredibly buggy on OS X in early versions |
+ # of Python 2.6, so allow this call to fail. Only catch the specific |
+ # exceptions we've seen, though: this call failing in other ways can reveal |
+ # legitimate problems. |
+ try: |
+ bypass = proxy_bypass(netloc) |
+ except (TypeError, socket.gaierror): |
+ bypass = False |
+ |
+ if bypass: |
+ return True |
+ |
+ return False |
+ |
+ |
+def get_environ_proxies(url): |
+ """ |
+ Return a dict of environment proxies. |
+ |
+ :rtype: dict |
+ """ |
+ if should_bypass_proxies(url): |
+ return {} |
+ else: |
+ return getproxies() |
+ |
+ |
+def select_proxy(url, proxies): |
+ """Select a proxy for the url, if applicable. |
+ |
+ :param url: The url being for the request |
+ :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs |
+ """ |
+ proxies = proxies or {} |
+ urlparts = urlparse(url) |
+ if urlparts.hostname is None: |
+ return proxies.get(urlparts.scheme, proxies.get('all')) |
+ |
+ proxy_keys = [ |
+ urlparts.scheme + '://' + urlparts.hostname, |
+ urlparts.scheme, |
+ 'all://' + urlparts.hostname, |
+ 'all', |
+ ] |
+ proxy = None |
+ for proxy_key in proxy_keys: |
+ if proxy_key in proxies: |
+ proxy = proxies[proxy_key] |
+ break |
+ |
+ return proxy |
+ |
+ |
+def default_user_agent(name="python-requests"): |
+ """ |
+ Return a string representing the default user agent. |
+ |
+ :rtype: str |
+ """ |
+ return '%s/%s' % (name, __version__) |
+ |
+ |
+def default_headers(): |
+ """ |
+ :rtype: requests.structures.CaseInsensitiveDict |
+ """ |
+ return CaseInsensitiveDict({ |
+ 'User-Agent': default_user_agent(), |
+ 'Accept-Encoding': ', '.join(('gzip', 'deflate')), |
+ 'Accept': '*/*', |
+ 'Connection': 'keep-alive', |
+ }) |
+ |
+ |
+def parse_header_links(value): |
+ """Return a dict of parsed link headers proxies. |
+ |
+ i.e. Link: <http:/.../front.jpeg>; rel=front; type="image/jpeg",<http://.../back.jpeg>; rel=back;type="image/jpeg" |
+ |
+ :rtype: list |
+ """ |
+ |
+ links = [] |
+ |
+ replace_chars = ' \'"' |
+ |
+ for val in re.split(', *<', value): |
+ try: |
+ url, params = val.split(';', 1) |
+ except ValueError: |
+ url, params = val, '' |
+ |
+ link = {'url': url.strip('<> \'"')} |
+ |
+ for param in params.split(';'): |
+ try: |
+ key, value = param.split('=') |
+ except ValueError: |
+ break |
+ |
+ link[key.strip(replace_chars)] = value.strip(replace_chars) |
+ |
+ links.append(link) |
+ |
+ return links |
+ |
+ |
+# Null bytes; no need to recreate these on each call to guess_json_utf |
+_null = '\x00'.encode('ascii') # encoding to ASCII for Python 3 |
+_null2 = _null * 2 |
+_null3 = _null * 3 |
+ |
+ |
+def guess_json_utf(data): |
+ """ |
+ :rtype: str |
+ """ |
+ # JSON always starts with two ASCII characters, so detection is as |
+ # easy as counting the nulls and from their location and count |
+ # determine the encoding. Also detect a BOM, if present. |
+ sample = data[:4] |
+ if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE): |
+ return 'utf-32' # BOM included |
+ if sample[:3] == codecs.BOM_UTF8: |
+ return 'utf-8-sig' # BOM included, MS style (discouraged) |
+ if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE): |
+ return 'utf-16' # BOM included |
+ nullcount = sample.count(_null) |
+ if nullcount == 0: |
+ return 'utf-8' |
+ if nullcount == 2: |
+ if sample[::2] == _null2: # 1st and 3rd are null |
+ return 'utf-16-be' |
+ if sample[1::2] == _null2: # 2nd and 4th are null |
+ return 'utf-16-le' |
+ # Did not detect 2 valid UTF-16 ascii-range characters |
+ if nullcount == 3: |
+ if sample[:3] == _null3: |
+ return 'utf-32-be' |
+ if sample[1:] == _null3: |
+ return 'utf-32-le' |
+ # Did not detect a valid UTF-32 ascii-range character |
+ return None |
+ |
+ |
+def prepend_scheme_if_needed(url, new_scheme): |
+ """Given a URL that may or may not have a scheme, prepend the given scheme. |
+ Does not replace a present scheme with the one provided as an argument. |
+ |
+ :rtype: str |
+ """ |
+ scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) |
+ |
+ # urlparse is a finicky beast, and sometimes decides that there isn't a |
+ # netloc present. Assume that it's being over-cautious, and switch netloc |
+ # and path if urlparse decided there was no netloc. |
+ if not netloc: |
+ netloc, path = path, netloc |
+ |
+ return urlunparse((scheme, netloc, path, params, query, fragment)) |
+ |
+ |
+def get_auth_from_url(url): |
+ """Given a url with authentication components, extract them into a tuple of |
+ username,password. |
+ |
+ :rtype: (str,str) |
+ """ |
+ parsed = urlparse(url) |
+ |
+ try: |
+ auth = (unquote(parsed.username), unquote(parsed.password)) |
+ except (AttributeError, TypeError): |
+ auth = ('', '') |
+ |
+ return auth |
+ |
+ |
+# Moved outside of function to avoid recompile every call |
+_CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') |
+_CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') |
+ |
+def check_header_validity(header): |
+ """Verifies that header value is a string which doesn't contain |
+ leading whitespace or return characters. This prevents unintended |
+ header injection. |
+ |
+ :param header: tuple, in the format (name, value). |
+ """ |
+ name, value = header |
+ |
+ if isinstance(value, bytes): |
+ pat = _CLEAN_HEADER_REGEX_BYTE |
+ else: |
+ pat = _CLEAN_HEADER_REGEX_STR |
+ try: |
+ if not pat.match(value): |
+ raise InvalidHeader("Invalid return character or leading space in header: %s" % name) |
+ except TypeError: |
+ raise InvalidHeader("Header value %s must be of type str or bytes, " |
+ "not %s" % (value, type(value))) |
+ |
+ |
+def urldefragauth(url): |
+ """ |
+ Given a url remove the fragment and the authentication part. |
+ |
+ :rtype: str |
+ """ |
+ scheme, netloc, path, params, query, fragment = urlparse(url) |
+ |
+ # see func:`prepend_scheme_if_needed` |
+ if not netloc: |
+ netloc, path = path, netloc |
+ |
+ netloc = netloc.rsplit('@', 1)[-1] |
+ |
+ return urlunparse((scheme, netloc, path, params, query, '')) |
+ |
+def rewind_body(prepared_request): |
+ """Move file pointer back to its recorded starting position |
+ so it can be read again on redirect. |
+ """ |
+ body_seek = getattr(prepared_request.body, 'seek', None) |
+ if body_seek is not None and isinstance(prepared_request._body_position, integer_types): |
+ try: |
+ body_seek(prepared_request._body_position) |
+ except (IOError, OSError): |
+ raise UnrewindableBodyError("An error occured when rewinding request " |
+ "body for redirect.") |
+ else: |
+ raise UnrewindableBodyError("Unable to rewind request body for redirect.") |