Index: tools/telemetry/third_party/gsutil/third_party/httplib2/python3/httplib2/__init__.py |
diff --git a/tools/telemetry/third_party/gsutil/third_party/httplib2/python3/httplib2/__init__.py b/tools/telemetry/third_party/gsutil/third_party/httplib2/python3/httplib2/__init__.py |
deleted file mode 100644 |
index bf6c2e9b4b9cf96327e42a70a3ba7ffdf73a62e3..0000000000000000000000000000000000000000 |
--- a/tools/telemetry/third_party/gsutil/third_party/httplib2/python3/httplib2/__init__.py |
+++ /dev/null |
@@ -1,1349 +0,0 @@ |
- |
-""" |
-httplib2 |
- |
-A caching http interface that supports ETags and gzip |
-to conserve bandwidth. |
- |
-Requires Python 3.0 or later |
- |
-Changelog: |
-2009-05-28, Pilgrim: ported to Python 3 |
-2007-08-18, Rick: Modified so it's able to use a socks proxy if needed. |
- |
-""" |
- |
-__author__ = "Joe Gregorio (joe@bitworking.org)" |
-__copyright__ = "Copyright 2006, Joe Gregorio" |
-__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", |
- "James Antill", |
- "Xavier Verges Farrero", |
- "Jonathan Feinberg", |
- "Blair Zajac", |
- "Sam Ruby", |
- "Louis Nyffenegger", |
- "Mark Pilgrim"] |
-__license__ = "MIT" |
-__version__ = "0.7.7" |
- |
-import re |
-import sys |
-import email |
-import email.utils |
-import email.message |
-import email.feedparser |
-import io |
-import gzip |
-import zlib |
-import http.client |
-import urllib.parse |
-import base64 |
-import os |
-import copy |
-import calendar |
-import time |
-import random |
-import errno |
-from hashlib import sha1 as _sha, md5 as _md5 |
-import hmac |
-from gettext import gettext as _ |
-import socket |
-import ssl |
-_ssl_wrap_socket = ssl.wrap_socket |
- |
-try: |
- import socks |
-except ImportError: |
- socks = None |
- |
-from .iri2uri import iri2uri |
- |
-def has_timeout(timeout): |
- if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'): |
- return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) |
- return (timeout is not None) |
- |
-__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', |
- 'RedirectMissingLocation', 'RedirectLimit', |
- 'FailedToDecompressContent', 'UnimplementedDigestAuthOptionError', |
- 'UnimplementedHmacDigestAuthOptionError', |
- 'debuglevel', 'RETRIES'] |
- |
- |
-# The httplib debug level, set to a non-zero value to get debug output |
-debuglevel = 0 |
- |
-# A request will be tried 'RETRIES' times if it fails at the socket/connection level. |
-RETRIES = 2 |
- |
-# All exceptions raised here derive from HttpLib2Error |
-class HttpLib2Error(Exception): pass |
- |
-# Some exceptions can be caught and optionally |
-# be turned back into responses. |
-class HttpLib2ErrorWithResponse(HttpLib2Error): |
- def __init__(self, desc, response, content): |
- self.response = response |
- self.content = content |
- HttpLib2Error.__init__(self, desc) |
- |
-class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass |
-class RedirectLimit(HttpLib2ErrorWithResponse): pass |
-class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass |
-class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass |
-class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass |
- |
-class MalformedHeader(HttpLib2Error): pass |
-class RelativeURIError(HttpLib2Error): pass |
-class ServerNotFoundError(HttpLib2Error): pass |
-class CertificateValidationUnsupportedInPython31(HttpLib2Error): pass |
- |
-# Open Items: |
-# ----------- |
-# Proxy support |
- |
-# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) |
- |
-# Pluggable cache storage (supports storing the cache in |
-# flat files by default. We need a plug-in architecture |
-# that can support Berkeley DB and Squid) |
- |
-# == Known Issues == |
-# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. |
-# Does not handle Cache-Control: max-stale |
-# Does not use Age: headers when calculating cache freshness. |
- |
- |
-# The number of redirections to follow before giving up. |
-# Note that only GET redirects are automatically followed. |
-# Will also honor 301 requests by saving that info and never |
-# requesting that URI again. |
-DEFAULT_MAX_REDIRECTS = 5 |
- |
-# Which headers are hop-by-hop headers by default |
-HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] |
- |
-# Default CA certificates file bundled with httplib2. |
-CA_CERTS = os.path.join( |
- os.path.dirname(os.path.abspath(__file__ )), "cacerts.txt") |
- |
-def _get_end2end_headers(response): |
- hopbyhop = list(HOP_BY_HOP) |
- hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')]) |
- return [header for header in list(response.keys()) if header not in hopbyhop] |
- |
-URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") |
- |
-def parse_uri(uri): |
- """Parses a URI using the regex given in Appendix B of RFC 3986. |
- |
- (scheme, authority, path, query, fragment) = parse_uri(uri) |
- """ |
- groups = URI.match(uri).groups() |
- return (groups[1], groups[3], groups[4], groups[6], groups[8]) |
- |
-def urlnorm(uri): |
- (scheme, authority, path, query, fragment) = parse_uri(uri) |
- if not scheme or not authority: |
- raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) |
- authority = authority.lower() |
- scheme = scheme.lower() |
- if not path: |
- path = "/" |
- # Could do syntax based normalization of the URI before |
- # computing the digest. See Section 6.2.2 of Std 66. |
- request_uri = query and "?".join([path, query]) or path |
- scheme = scheme.lower() |
- defrag_uri = scheme + "://" + authority + request_uri |
- return scheme, authority, request_uri, defrag_uri |
- |
- |
-# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) |
-re_url_scheme = re.compile(br'^\w+://') |
-re_url_scheme_s = re.compile(r'^\w+://') |
-re_slash = re.compile(br'[?/:|]+') |
- |
-def safename(filename): |
- """Return a filename suitable for the cache. |
- |
- Strips dangerous and common characters to create a filename we |
- can use to store the cache in. |
- """ |
- |
- try: |
- if re_url_scheme_s.match(filename): |
- if isinstance(filename,bytes): |
- filename = filename.decode('utf-8') |
- filename = filename.encode('idna') |
- else: |
- filename = filename.encode('idna') |
- except UnicodeError: |
- pass |
- if isinstance(filename,str): |
- filename=filename.encode('utf-8') |
- filemd5 = _md5(filename).hexdigest().encode('utf-8') |
- filename = re_url_scheme.sub(b"", filename) |
- filename = re_slash.sub(b",", filename) |
- |
- # limit length of filename |
- if len(filename)>200: |
- filename=filename[:200] |
- return b",".join((filename, filemd5)).decode('utf-8') |
- |
-NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') |
-def _normalize_headers(headers): |
- return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.items()]) |
- |
-def _parse_cache_control(headers): |
- retval = {} |
- if 'cache-control' in headers: |
- parts = headers['cache-control'].split(',') |
- parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] |
- parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] |
- retval = dict(parts_with_args + parts_wo_args) |
- return retval |
- |
-# Whether to use a strict mode to parse WWW-Authenticate headers |
-# Might lead to bad results in case of ill-formed header value, |
-# so disabled by default, falling back to relaxed parsing. |
-# Set to true to turn on, usefull for testing servers. |
-USE_WWW_AUTH_STRICT_PARSING = 0 |
- |
-# In regex below: |
-# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP |
-# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space |
-# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: |
-# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x08\x0A-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"? |
-WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?<!\")[^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") |
-WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(?<!\")[^ \t\r\n,]+(?!\"))\"?)(.*)$") |
-UNQUOTE_PAIRS = re.compile(r'\\(.)') |
-def _parse_www_authenticate(headers, headername='www-authenticate'): |
- """Returns a dictionary of dictionaries, one dict |
- per auth_scheme.""" |
- retval = {} |
- if headername in headers: |
- try: |
- authenticate = headers[headername].strip() |
- www_auth = USE_WWW_AUTH_STRICT_PARSING and WWW_AUTH_STRICT or WWW_AUTH_RELAXED |
- while authenticate: |
- # Break off the scheme at the beginning of the line |
- if headername == 'authentication-info': |
- (auth_scheme, the_rest) = ('digest', authenticate) |
- else: |
- (auth_scheme, the_rest) = authenticate.split(" ", 1) |
- # Now loop over all the key value pairs that come after the scheme, |
- # being careful not to roll into the next scheme |
- match = www_auth.search(the_rest) |
- auth_params = {} |
- while match: |
- if match and len(match.groups()) == 3: |
- (key, value, the_rest) = match.groups() |
- auth_params[key.lower()] = UNQUOTE_PAIRS.sub(r'\1', value) # '\\'.join([x.replace('\\', '') for x in value.split('\\\\')]) |
- match = www_auth.search(the_rest) |
- retval[auth_scheme.lower()] = auth_params |
- authenticate = the_rest.strip() |
- except ValueError: |
- raise MalformedHeader("WWW-Authenticate") |
- return retval |
- |
- |
-def _entry_disposition(response_headers, request_headers): |
- """Determine freshness from the Date, Expires and Cache-Control headers. |
- |
- We don't handle the following: |
- |
- 1. Cache-Control: max-stale |
- 2. Age: headers are not used in the calculations. |
- |
- Not that this algorithm is simpler than you might think |
- because we are operating as a private (non-shared) cache. |
- This lets us ignore 's-maxage'. We can also ignore |
- 'proxy-invalidate' since we aren't a proxy. |
- We will never return a stale document as |
- fresh as a design decision, and thus the non-implementation |
- of 'max-stale'. This also lets us safely ignore 'must-revalidate' |
- since we operate as if every server has sent 'must-revalidate'. |
- Since we are private we get to ignore both 'public' and |
- 'private' parameters. We also ignore 'no-transform' since |
- we don't do any transformations. |
- The 'no-store' parameter is handled at a higher level. |
- So the only Cache-Control parameters we look at are: |
- |
- no-cache |
- only-if-cached |
- max-age |
- min-fresh |
- """ |
- |
- retval = "STALE" |
- cc = _parse_cache_control(request_headers) |
- cc_response = _parse_cache_control(response_headers) |
- |
- if 'pragma' in request_headers and request_headers['pragma'].lower().find('no-cache') != -1: |
- retval = "TRANSPARENT" |
- if 'cache-control' not in request_headers: |
- request_headers['cache-control'] = 'no-cache' |
- elif 'no-cache' in cc: |
- retval = "TRANSPARENT" |
- elif 'no-cache' in cc_response: |
- retval = "STALE" |
- elif 'only-if-cached' in cc: |
- retval = "FRESH" |
- elif 'date' in response_headers: |
- date = calendar.timegm(email.utils.parsedate_tz(response_headers['date'])) |
- now = time.time() |
- current_age = max(0, now - date) |
- if 'max-age' in cc_response: |
- try: |
- freshness_lifetime = int(cc_response['max-age']) |
- except ValueError: |
- freshness_lifetime = 0 |
- elif 'expires' in response_headers: |
- expires = email.utils.parsedate_tz(response_headers['expires']) |
- if None == expires: |
- freshness_lifetime = 0 |
- else: |
- freshness_lifetime = max(0, calendar.timegm(expires) - date) |
- else: |
- freshness_lifetime = 0 |
- if 'max-age' in cc: |
- try: |
- freshness_lifetime = int(cc['max-age']) |
- except ValueError: |
- freshness_lifetime = 0 |
- if 'min-fresh' in cc: |
- try: |
- min_fresh = int(cc['min-fresh']) |
- except ValueError: |
- min_fresh = 0 |
- current_age += min_fresh |
- if freshness_lifetime > current_age: |
- retval = "FRESH" |
- return retval |
- |
-def _decompressContent(response, new_content): |
- content = new_content |
- try: |
- encoding = response.get('content-encoding', None) |
- if encoding in ['gzip', 'deflate']: |
- if encoding == 'gzip': |
- content = gzip.GzipFile(fileobj=io.BytesIO(new_content)).read() |
- if encoding == 'deflate': |
- content = zlib.decompress(content) |
- response['content-length'] = str(len(content)) |
- # Record the historical presence of the encoding in a way the won't interfere. |
- response['-content-encoding'] = response['content-encoding'] |
- del response['content-encoding'] |
- except IOError: |
- content = "" |
- raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content) |
- return content |
- |
-def _bind_write_headers(msg): |
- from email.header import Header |
- def _write_headers(self): |
- # Self refers to the Generator object |
- for h, v in msg.items(): |
- print('%s:' % h, end=' ', file=self._fp) |
- if isinstance(v, Header): |
- print(v.encode(maxlinelen=self._maxheaderlen), file=self._fp) |
- else: |
- # Header's got lots of smarts, so use it. |
- header = Header(v, maxlinelen=self._maxheaderlen, charset='utf-8', |
- header_name=h) |
- print(header.encode(), file=self._fp) |
- # A blank line always separates headers from body |
- print(file=self._fp) |
- return _write_headers |
- |
-def _updateCache(request_headers, response_headers, content, cache, cachekey): |
- if cachekey: |
- cc = _parse_cache_control(request_headers) |
- cc_response = _parse_cache_control(response_headers) |
- if 'no-store' in cc or 'no-store' in cc_response: |
- cache.delete(cachekey) |
- else: |
- info = email.message.Message() |
- for key, value in response_headers.items(): |
- if key not in ['status','content-encoding','transfer-encoding']: |
- info[key] = value |
- |
- # Add annotations to the cache to indicate what headers |
- # are variant for this request. |
- vary = response_headers.get('vary', None) |
- if vary: |
- vary_headers = vary.lower().replace(' ', '').split(',') |
- for header in vary_headers: |
- key = '-varied-%s' % header |
- try: |
- info[key] = request_headers[header] |
- except KeyError: |
- pass |
- |
- status = response_headers.status |
- if status == 304: |
- status = 200 |
- |
- status_header = 'status: %d\r\n' % status |
- |
- try: |
- header_str = info.as_string() |
- except UnicodeEncodeError: |
- setattr(info, '_write_headers', _bind_write_headers(info)) |
- header_str = info.as_string() |
- |
- header_str = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", header_str) |
- text = b"".join([status_header.encode('utf-8'), header_str.encode('utf-8'), content]) |
- |
- cache.set(cachekey, text) |
- |
-def _cnonce(): |
- dig = _md5(("%s:%s" % (time.ctime(), ["0123456789"[random.randrange(0, 9)] for i in range(20)])).encode('utf-8')).hexdigest() |
- return dig[:16] |
- |
-def _wsse_username_token(cnonce, iso_now, password): |
- return base64.b64encode(_sha(("%s%s%s" % (cnonce, iso_now, password)).encode('utf-8')).digest()).strip() |
- |
- |
-# For credentials we need two things, first |
-# a pool of credential to try (not necesarily tied to BAsic, Digest, etc.) |
-# Then we also need a list of URIs that have already demanded authentication |
-# That list is tricky since sub-URIs can take the same auth, or the |
-# auth scheme may change as you descend the tree. |
-# So we also need each Auth instance to be able to tell us |
-# how close to the 'top' it is. |
- |
-class Authentication(object): |
- def __init__(self, credentials, host, request_uri, headers, response, content, http): |
- (scheme, authority, path, query, fragment) = parse_uri(request_uri) |
- self.path = path |
- self.host = host |
- self.credentials = credentials |
- self.http = http |
- |
- def depth(self, request_uri): |
- (scheme, authority, path, query, fragment) = parse_uri(request_uri) |
- return request_uri[len(self.path):].count("/") |
- |
- def inscope(self, host, request_uri): |
- # XXX Should we normalize the request_uri? |
- (scheme, authority, path, query, fragment) = parse_uri(request_uri) |
- return (host == self.host) and path.startswith(self.path) |
- |
- def request(self, method, request_uri, headers, content): |
- """Modify the request headers to add the appropriate |
- Authorization header. Over-rise this in sub-classes.""" |
- pass |
- |
- def response(self, response, content): |
- """Gives us a chance to update with new nonces |
- or such returned from the last authorized response. |
- Over-rise this in sub-classes if necessary. |
- |
- Return TRUE is the request is to be retried, for |
- example Digest may return stale=true. |
- """ |
- return False |
- |
- def __eq__(self, auth): |
- return False |
- |
- def __ne__(self, auth): |
- return True |
- |
- def __lt__(self, auth): |
- return True |
- |
- def __gt__(self, auth): |
- return False |
- |
- def __le__(self, auth): |
- return True |
- |
- def __ge__(self, auth): |
- return False |
- |
- def __bool__(self): |
- return True |
- |
- |
-class BasicAuthentication(Authentication): |
- def __init__(self, credentials, host, request_uri, headers, response, content, http): |
- Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) |
- |
- def request(self, method, request_uri, headers, content): |
- """Modify the request headers to add the appropriate |
- Authorization header.""" |
- headers['authorization'] = 'Basic ' + base64.b64encode(("%s:%s" % self.credentials).encode('utf-8')).strip().decode('utf-8') |
- |
- |
-class DigestAuthentication(Authentication): |
- """Only do qop='auth' and MD5, since that |
- is all Apache currently implements""" |
- def __init__(self, credentials, host, request_uri, headers, response, content, http): |
- Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) |
- challenge = _parse_www_authenticate(response, 'www-authenticate') |
- self.challenge = challenge['digest'] |
- qop = self.challenge.get('qop', 'auth') |
- self.challenge['qop'] = ('auth' in [x.strip() for x in qop.split()]) and 'auth' or None |
- if self.challenge['qop'] is None: |
- raise UnimplementedDigestAuthOptionError( _("Unsupported value for qop: %s." % qop)) |
- self.challenge['algorithm'] = self.challenge.get('algorithm', 'MD5').upper() |
- if self.challenge['algorithm'] != 'MD5': |
- raise UnimplementedDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm'])) |
- self.A1 = "".join([self.credentials[0], ":", self.challenge['realm'], ":", self.credentials[1]]) |
- self.challenge['nc'] = 1 |
- |
- def request(self, method, request_uri, headers, content, cnonce = None): |
- """Modify the request headers""" |
- H = lambda x: _md5(x.encode('utf-8')).hexdigest() |
- KD = lambda s, d: H("%s:%s" % (s, d)) |
- A2 = "".join([method, ":", request_uri]) |
- self.challenge['cnonce'] = cnonce or _cnonce() |
- request_digest = '"%s"' % KD(H(self.A1), "%s:%s:%s:%s:%s" % ( |
- self.challenge['nonce'], |
- '%08x' % self.challenge['nc'], |
- self.challenge['cnonce'], |
- self.challenge['qop'], H(A2))) |
- headers['authorization'] = 'Digest username="%s", realm="%s", nonce="%s", uri="%s", algorithm=%s, response=%s, qop=%s, nc=%08x, cnonce="%s"' % ( |
- self.credentials[0], |
- self.challenge['realm'], |
- self.challenge['nonce'], |
- request_uri, |
- self.challenge['algorithm'], |
- request_digest, |
- self.challenge['qop'], |
- self.challenge['nc'], |
- self.challenge['cnonce']) |
- if self.challenge.get('opaque'): |
- headers['authorization'] += ', opaque="%s"' % self.challenge['opaque'] |
- self.challenge['nc'] += 1 |
- |
- def response(self, response, content): |
- if 'authentication-info' not in response: |
- challenge = _parse_www_authenticate(response, 'www-authenticate').get('digest', {}) |
- if 'true' == challenge.get('stale'): |
- self.challenge['nonce'] = challenge['nonce'] |
- self.challenge['nc'] = 1 |
- return True |
- else: |
- updated_challenge = _parse_www_authenticate(response, 'authentication-info').get('digest', {}) |
- |
- if 'nextnonce' in updated_challenge: |
- self.challenge['nonce'] = updated_challenge['nextnonce'] |
- self.challenge['nc'] = 1 |
- return False |
- |
- |
-class HmacDigestAuthentication(Authentication): |
- """Adapted from Robert Sayre's code and DigestAuthentication above.""" |
- __author__ = "Thomas Broyer (t.broyer@ltgt.net)" |
- |
- def __init__(self, credentials, host, request_uri, headers, response, content, http): |
- Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) |
- challenge = _parse_www_authenticate(response, 'www-authenticate') |
- self.challenge = challenge['hmacdigest'] |
- # TODO: self.challenge['domain'] |
- self.challenge['reason'] = self.challenge.get('reason', 'unauthorized') |
- if self.challenge['reason'] not in ['unauthorized', 'integrity']: |
- self.challenge['reason'] = 'unauthorized' |
- self.challenge['salt'] = self.challenge.get('salt', '') |
- if not self.challenge.get('snonce'): |
- raise UnimplementedHmacDigestAuthOptionError( _("The challenge doesn't contain a server nonce, or this one is empty.")) |
- self.challenge['algorithm'] = self.challenge.get('algorithm', 'HMAC-SHA-1') |
- if self.challenge['algorithm'] not in ['HMAC-SHA-1', 'HMAC-MD5']: |
- raise UnimplementedHmacDigestAuthOptionError( _("Unsupported value for algorithm: %s." % self.challenge['algorithm'])) |
- self.challenge['pw-algorithm'] = self.challenge.get('pw-algorithm', 'SHA-1') |
- if self.challenge['pw-algorithm'] not in ['SHA-1', 'MD5']: |
- raise UnimplementedHmacDigestAuthOptionError( _("Unsupported value for pw-algorithm: %s." % self.challenge['pw-algorithm'])) |
- if self.challenge['algorithm'] == 'HMAC-MD5': |
- self.hashmod = _md5 |
- else: |
- self.hashmod = _sha |
- if self.challenge['pw-algorithm'] == 'MD5': |
- self.pwhashmod = _md5 |
- else: |
- self.pwhashmod = _sha |
- self.key = "".join([self.credentials[0], ":", |
- self.pwhashmod.new("".join([self.credentials[1], self.challenge['salt']])).hexdigest().lower(), |
- ":", self.challenge['realm']]) |
- self.key = self.pwhashmod.new(self.key).hexdigest().lower() |
- |
- def request(self, method, request_uri, headers, content): |
- """Modify the request headers""" |
- keys = _get_end2end_headers(headers) |
- keylist = "".join(["%s " % k for k in keys]) |
- headers_val = "".join([headers[k] for k in keys]) |
- created = time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()) |
- cnonce = _cnonce() |
- request_digest = "%s:%s:%s:%s:%s" % (method, request_uri, cnonce, self.challenge['snonce'], headers_val) |
- request_digest = hmac.new(self.key, request_digest, self.hashmod).hexdigest().lower() |
- headers['authorization'] = 'HMACDigest username="%s", realm="%s", snonce="%s", cnonce="%s", uri="%s", created="%s", response="%s", headers="%s"' % ( |
- self.credentials[0], |
- self.challenge['realm'], |
- self.challenge['snonce'], |
- cnonce, |
- request_uri, |
- created, |
- request_digest, |
- keylist) |
- |
- def response(self, response, content): |
- challenge = _parse_www_authenticate(response, 'www-authenticate').get('hmacdigest', {}) |
- if challenge.get('reason') in ['integrity', 'stale']: |
- return True |
- return False |
- |
- |
-class WsseAuthentication(Authentication): |
- """This is thinly tested and should not be relied upon. |
- At this time there isn't any third party server to test against. |
- Blogger and TypePad implemented this algorithm at one point |
- but Blogger has since switched to Basic over HTTPS and |
- TypePad has implemented it wrong, by never issuing a 401 |
- challenge but instead requiring your client to telepathically know that |
- their endpoint is expecting WSSE profile="UsernameToken".""" |
- def __init__(self, credentials, host, request_uri, headers, response, content, http): |
- Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) |
- |
- def request(self, method, request_uri, headers, content): |
- """Modify the request headers to add the appropriate |
- Authorization header.""" |
- headers['authorization'] = 'WSSE profile="UsernameToken"' |
- iso_now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) |
- cnonce = _cnonce() |
- password_digest = _wsse_username_token(cnonce, iso_now, self.credentials[1]) |
- headers['X-WSSE'] = 'UsernameToken Username="%s", PasswordDigest="%s", Nonce="%s", Created="%s"' % ( |
- self.credentials[0], |
- password_digest, |
- cnonce, |
- iso_now) |
- |
-class GoogleLoginAuthentication(Authentication): |
- def __init__(self, credentials, host, request_uri, headers, response, content, http): |
- from urllib.parse import urlencode |
- Authentication.__init__(self, credentials, host, request_uri, headers, response, content, http) |
- challenge = _parse_www_authenticate(response, 'www-authenticate') |
- service = challenge['googlelogin'].get('service', 'xapi') |
- # Bloggger actually returns the service in the challenge |
- # For the rest we guess based on the URI |
- if service == 'xapi' and request_uri.find("calendar") > 0: |
- service = "cl" |
- # No point in guessing Base or Spreadsheet |
- #elif request_uri.find("spreadsheets") > 0: |
- # service = "wise" |
- |
- auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) |
- resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) |
- lines = content.split('\n') |
- d = dict([tuple(line.split("=", 1)) for line in lines if line]) |
- if resp.status == 403: |
- self.Auth = "" |
- else: |
- self.Auth = d['Auth'] |
- |
- def request(self, method, request_uri, headers, content): |
- """Modify the request headers to add the appropriate |
- Authorization header.""" |
- headers['authorization'] = 'GoogleLogin Auth=' + self.Auth |
- |
- |
-AUTH_SCHEME_CLASSES = { |
- "basic": BasicAuthentication, |
- "wsse": WsseAuthentication, |
- "digest": DigestAuthentication, |
- "hmacdigest": HmacDigestAuthentication, |
- "googlelogin": GoogleLoginAuthentication |
-} |
- |
-AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] |
- |
-class FileCache(object): |
- """Uses a local directory as a store for cached files. |
- Not really safe to use if multiple threads or processes are going to |
- be running on the same cache. |
- """ |
- def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior |
- self.cache = cache |
- self.safe = safe |
- if not os.path.exists(cache): |
- os.makedirs(self.cache) |
- |
- def get(self, key): |
- retval = None |
- cacheFullPath = os.path.join(self.cache, self.safe(key)) |
- try: |
- f = open(cacheFullPath, "rb") |
- retval = f.read() |
- f.close() |
- except IOError: |
- pass |
- return retval |
- |
- def set(self, key, value): |
- cacheFullPath = os.path.join(self.cache, self.safe(key)) |
- f = open(cacheFullPath, "wb") |
- f.write(value) |
- f.close() |
- |
- def delete(self, key): |
- cacheFullPath = os.path.join(self.cache, self.safe(key)) |
- if os.path.exists(cacheFullPath): |
- os.remove(cacheFullPath) |
- |
-class Credentials(object): |
- def __init__(self): |
- self.credentials = [] |
- |
- def add(self, name, password, domain=""): |
- self.credentials.append((domain.lower(), name, password)) |
- |
- def clear(self): |
- self.credentials = [] |
- |
- def iter(self, domain): |
- for (cdomain, name, password) in self.credentials: |
- if cdomain == "" or domain == cdomain: |
- yield (name, password) |
- |
-class KeyCerts(Credentials): |
- """Identical to Credentials except that |
- name/password are mapped to key/cert.""" |
- pass |
- |
- |
-class ProxyInfo(object): |
- """Collect information required to use a proxy.""" |
- def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): |
- """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX |
- constants. For example: |
- |
-p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) |
- """ |
- self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass |
- |
- def astuple(self): |
- return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, |
- self.proxy_user, self.proxy_pass) |
- |
- def isgood(self): |
- return socks and (self.proxy_host != None) and (self.proxy_port != None) |
- |
- |
-def proxy_info_from_environment(method='http'): |
- """ |
- Read proxy info from the environment variables. |
- """ |
- if method not in ('http', 'https'): |
- return |
- |
- env_var = method + '_proxy' |
- url = os.environ.get(env_var, os.environ.get(env_var.upper())) |
- if not url: |
- return |
- return proxy_info_from_url(url, method) |
- |
- |
-def proxy_info_from_url(url, method='http'): |
- """ |
- Construct a ProxyInfo from a URL (such as http_proxy env var) |
- """ |
- url = urllib.parse.urlparse(url) |
- username = None |
- password = None |
- port = None |
- if '@' in url[1]: |
- ident, host_port = url[1].split('@', 1) |
- if ':' in ident: |
- username, password = ident.split(':', 1) |
- else: |
- password = ident |
- else: |
- host_port = url[1] |
- if ':' in host_port: |
- host, port = host_port.split(':', 1) |
- else: |
- host = host_port |
- |
- if port: |
- port = int(port) |
- else: |
- port = dict(https=443, http=80)[method] |
- |
- proxy_type = 3 # socks.PROXY_TYPE_HTTP |
- return ProxyInfo( |
- proxy_type = proxy_type, |
- proxy_host = host, |
- proxy_port = port, |
- proxy_user = username or None, |
- proxy_pass = password or None, |
- ) |
- |
- |
-class HTTPConnectionWithTimeout(http.client.HTTPConnection): |
- """HTTPConnection subclass that supports timeouts |
- |
- HTTPConnection subclass that supports timeouts |
- |
- All timeouts are in seconds. If None is passed for timeout then |
- Python's default timeout for sockets will be used. See for example |
- the docs of socket.setdefaulttimeout(): |
- http://docs.python.org/library/socket.html#socket.setdefaulttimeout |
- """ |
- |
- def __init__(self, host, port=None, timeout=None, proxy_info=None): |
- http.client.HTTPConnection.__init__(self, host, port=port, |
- timeout=timeout) |
- self.proxy_info = proxy_info |
- |
- |
-class HTTPSConnectionWithTimeout(http.client.HTTPSConnection): |
- """ |
- This class allows communication via SSL. |
- |
- All timeouts are in seconds. If None is passed for timeout then |
- Python's default timeout for sockets will be used. See for example |
- the docs of socket.setdefaulttimeout(): |
- http://docs.python.org/library/socket.html#socket.setdefaulttimeout |
- """ |
- |
- def __init__(self, host, port=None, key_file=None, cert_file=None, |
- timeout=None, proxy_info=None, |
- ca_certs=None, disable_ssl_certificate_validation=False): |
- self.proxy_info = proxy_info |
- context = None |
- if ca_certs is None: |
- ca_certs = CA_CERTS |
- if (cert_file or ca_certs) and not disable_ssl_certificate_validation: |
- if not hasattr(ssl, 'SSLContext'): |
- raise CertificateValidationUnsupportedInPython31() |
- context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) |
- context.verify_mode = ssl.CERT_REQUIRED |
- if cert_file: |
- context.load_cert_chain(cert_file, key_file) |
- if ca_certs: |
- context.load_verify_locations(ca_certs) |
- http.client.HTTPSConnection.__init__( |
- self, host, port=port, key_file=key_file, |
- cert_file=cert_file, timeout=timeout, context=context, |
- check_hostname=True) |
- |
- |
-SCHEME_TO_CONNECTION = { |
- 'http': HTTPConnectionWithTimeout, |
- 'https': HTTPSConnectionWithTimeout, |
-} |
- |
-class Http(object): |
- """An HTTP client that handles: |
- |
- - all methods |
- - caching |
- - ETags |
- - compression, |
- - HTTPS |
- - Basic |
- - Digest |
- - WSSE |
- |
- and more. |
- """ |
- def __init__(self, cache=None, timeout=None, |
- proxy_info=proxy_info_from_environment, |
- ca_certs=None, disable_ssl_certificate_validation=False): |
- """If 'cache' is a string then it is used as a directory name for |
- a disk cache. Otherwise it must be an object that supports the |
- same interface as FileCache. |
- |
- All timeouts are in seconds. If None is passed for timeout |
- then Python's default timeout for sockets will be used. See |
- for example the docs of socket.setdefaulttimeout(): |
- http://docs.python.org/library/socket.html#socket.setdefaulttimeout |
- |
- `proxy_info` may be: |
- - a callable that takes the http scheme ('http' or 'https') and |
- returns a ProxyInfo instance per request. By default, uses |
- proxy_info_from_environment. |
- - a ProxyInfo instance (static proxy config). |
- - None (proxy disabled). |
- |
- ca_certs is the path of a file containing root CA certificates for SSL |
- server certificate validation. By default, a CA cert file bundled with |
- httplib2 is used. |
- |
- If disable_ssl_certificate_validation is true, SSL cert validation will |
- not be performed. |
-""" |
- self.proxy_info = proxy_info |
- self.ca_certs = ca_certs |
- self.disable_ssl_certificate_validation = \ |
- disable_ssl_certificate_validation |
- # Map domain name to an httplib connection |
- self.connections = {} |
- # The location of the cache, for now a directory |
- # where cached responses are held. |
- if cache and isinstance(cache, str): |
- self.cache = FileCache(cache) |
- else: |
- self.cache = cache |
- |
- # Name/password |
- self.credentials = Credentials() |
- |
- # Key/cert |
- self.certificates = KeyCerts() |
- |
- # authorization objects |
- self.authorizations = [] |
- |
- # If set to False then no redirects are followed, even safe ones. |
- self.follow_redirects = True |
- |
- # Which HTTP methods do we apply optimistic concurrency to, i.e. |
- # which methods get an "if-match:" etag header added to them. |
- self.optimistic_concurrency_methods = ["PUT", "PATCH"] |
- |
- # If 'follow_redirects' is True, and this is set to True then |
- # all redirecs are followed, including unsafe ones. |
- self.follow_all_redirects = False |
- |
- self.ignore_etag = False |
- |
- self.force_exception_to_status_code = False |
- |
- self.timeout = timeout |
- |
- # Keep Authorization: headers on a redirect. |
- self.forward_authorization_headers = False |
- |
- def __getstate__(self): |
- state_dict = copy.copy(self.__dict__) |
- # In case request is augmented by some foreign object such as |
- # credentials which handle auth |
- if 'request' in state_dict: |
- del state_dict['request'] |
- if 'connections' in state_dict: |
- del state_dict['connections'] |
- return state_dict |
- |
- def __setstate__(self, state): |
- self.__dict__.update(state) |
- self.connections = {} |
- |
- def _auth_from_challenge(self, host, request_uri, headers, response, content): |
- """A generator that creates Authorization objects |
- that can be applied to requests. |
- """ |
- challenges = _parse_www_authenticate(response, 'www-authenticate') |
- for cred in self.credentials.iter(host): |
- for scheme in AUTH_SCHEME_ORDER: |
- if scheme in challenges: |
- yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) |
- |
- def add_credentials(self, name, password, domain=""): |
- """Add a name and password that will be used |
- any time a request requires authentication.""" |
- self.credentials.add(name, password, domain) |
- |
- def add_certificate(self, key, cert, domain): |
- """Add a key and cert that will be used |
- any time a request requires authentication.""" |
- self.certificates.add(key, cert, domain) |
- |
- def clear_credentials(self): |
- """Remove all the names and passwords |
- that are used for authentication""" |
- self.credentials.clear() |
- self.authorizations = [] |
- |
- def _conn_request(self, conn, request_uri, method, body, headers): |
- for i in range(RETRIES): |
- try: |
- if conn.sock is None: |
- conn.connect() |
- conn.request(method, request_uri, body, headers) |
- except socket.timeout: |
- conn.close() |
- raise |
- except socket.gaierror: |
- conn.close() |
- raise ServerNotFoundError("Unable to find the server at %s" % conn.host) |
- except socket.error as e: |
- errno_ = (e.args[0].errno if isinstance(e.args[0], socket.error) else e.errno) |
- if errno_ == errno.ECONNREFUSED: # Connection refused |
- raise |
- except http.client.HTTPException: |
- if conn.sock is None: |
- if i < RETRIES-1: |
- conn.close() |
- conn.connect() |
- continue |
- else: |
- conn.close() |
- raise |
- if i < RETRIES-1: |
- conn.close() |
- conn.connect() |
- continue |
- # Just because the server closed the connection doesn't apparently mean |
- # that the server didn't send a response. |
- pass |
- try: |
- response = conn.getresponse() |
- except socket.timeout: |
- raise |
- except (socket.error, http.client.HTTPException): |
- conn.close() |
- if i == 0: |
- conn.close() |
- conn.connect() |
- continue |
- else: |
- raise |
- else: |
- content = b"" |
- if method == "HEAD": |
- conn.close() |
- else: |
- content = response.read() |
- response = Response(response) |
- if method != "HEAD": |
- content = _decompressContent(response, content) |
- |
- break |
- return (response, content) |
- |
- |
- def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): |
- """Do the actual request using the connection object |
- and also follow one level of redirects if necessary""" |
- |
- auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] |
- auth = auths and sorted(auths)[0][1] or None |
- if auth: |
- auth.request(method, request_uri, headers, body) |
- |
- (response, content) = self._conn_request(conn, request_uri, method, body, headers) |
- |
- if auth: |
- if auth.response(response, body): |
- auth.request(method, request_uri, headers, body) |
- (response, content) = self._conn_request(conn, request_uri, method, body, headers ) |
- response._stale_digest = 1 |
- |
- if response.status == 401: |
- for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): |
- authorization.request(method, request_uri, headers, body) |
- (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) |
- if response.status != 401: |
- self.authorizations.append(authorization) |
- authorization.response(response, body) |
- break |
- |
- if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): |
- if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: |
- # Pick out the location header and basically start from the beginning |
- # remembering first to strip the ETag header and decrement our 'depth' |
- if redirections: |
- if 'location' not in response and response.status != 300: |
- raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content) |
- # Fix-up relative redirects (which violate an RFC 2616 MUST) |
- if 'location' in response: |
- location = response['location'] |
- (scheme, authority, path, query, fragment) = parse_uri(location) |
- if authority == None: |
- response['location'] = urllib.parse.urljoin(absolute_uri, location) |
- if response.status == 301 and method in ["GET", "HEAD"]: |
- response['-x-permanent-redirect-url'] = response['location'] |
- if 'content-location' not in response: |
- response['content-location'] = absolute_uri |
- _updateCache(headers, response, content, self.cache, cachekey) |
- if 'if-none-match' in headers: |
- del headers['if-none-match'] |
- if 'if-modified-since' in headers: |
- del headers['if-modified-since'] |
- if 'authorization' in headers and not self.forward_authorization_headers: |
- del headers['authorization'] |
- if 'location' in response: |
- location = response['location'] |
- old_response = copy.deepcopy(response) |
- if 'content-location' not in old_response: |
- old_response['content-location'] = absolute_uri |
- redirect_method = method |
- if response.status in [302, 303]: |
- redirect_method = "GET" |
- body = None |
- (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) |
- response.previous = old_response |
- else: |
- raise RedirectLimit("Redirected more times than redirection_limit allows.", response, content) |
- elif response.status in [200, 203] and method in ["GET", "HEAD"]: |
- # Don't cache 206's since we aren't going to handle byte range requests |
- if 'content-location' not in response: |
- response['content-location'] = absolute_uri |
- _updateCache(headers, response, content, self.cache, cachekey) |
- |
- return (response, content) |
- |
- def _normalize_headers(self, headers): |
- return _normalize_headers(headers) |
- |
-# Need to catch and rebrand some exceptions |
-# Then need to optionally turn all exceptions into status codes |
-# including all socket.* and httplib.* exceptions. |
- |
- |
- def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): |
- """ Performs a single HTTP request. |
-The 'uri' is the URI of the HTTP resource and can begin |
-with either 'http' or 'https'. The value of 'uri' must be an absolute URI. |
- |
-The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. |
-There is no restriction on the methods allowed. |
- |
-The 'body' is the entity body to be sent with the request. It is a string |
-object. |
- |
-Any extra headers that are to be sent with the request should be provided in the |
-'headers' dictionary. |
- |
-The maximum number of redirect to follow before raising an |
-exception is 'redirections. The default is 5. |
- |
-The return value is a tuple of (response, content), the first |
-being and instance of the 'Response' class, the second being |
-a string that contains the response entity body. |
- """ |
- try: |
- if headers is None: |
- headers = {} |
- else: |
- headers = self._normalize_headers(headers) |
- |
- if 'user-agent' not in headers: |
- headers['user-agent'] = "Python-httplib2/%s (gzip)" % __version__ |
- |
- uri = iri2uri(uri) |
- |
- (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) |
- domain_port = authority.split(":")[0:2] |
- if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http': |
- scheme = 'https' |
- authority = domain_port[0] |
- |
- conn_key = scheme+":"+authority |
- if conn_key in self.connections: |
- conn = self.connections[conn_key] |
- else: |
- if not connection_type: |
- connection_type = SCHEME_TO_CONNECTION[scheme] |
- certs = list(self.certificates.iter(authority)) |
- if issubclass(connection_type, HTTPSConnectionWithTimeout): |
- if certs: |
- conn = self.connections[conn_key] = connection_type( |
- authority, key_file=certs[0][0], |
- cert_file=certs[0][1], timeout=self.timeout, |
- proxy_info=self.proxy_info, |
- ca_certs=self.ca_certs, |
- disable_ssl_certificate_validation= |
- self.disable_ssl_certificate_validation) |
- else: |
- conn = self.connections[conn_key] = connection_type( |
- authority, timeout=self.timeout, |
- proxy_info=self.proxy_info, |
- ca_certs=self.ca_certs, |
- disable_ssl_certificate_validation= |
- self.disable_ssl_certificate_validation) |
- else: |
- conn = self.connections[conn_key] = connection_type( |
- authority, timeout=self.timeout, |
- proxy_info=self.proxy_info) |
- conn.set_debuglevel(debuglevel) |
- |
- if 'range' not in headers and 'accept-encoding' not in headers: |
- headers['accept-encoding'] = 'gzip, deflate' |
- |
- info = email.message.Message() |
- cached_value = None |
- if self.cache: |
- cachekey = defrag_uri |
- cached_value = self.cache.get(cachekey) |
- if cached_value: |
- try: |
- info, content = cached_value.split(b'\r\n\r\n', 1) |
- info = email.message_from_bytes(info) |
- for k, v in info.items(): |
- if v.startswith('=?') and v.endswith('?='): |
- info.replace_header(k, |
- str(*email.header.decode_header(v)[0])) |
- except (IndexError, ValueError): |
- self.cache.delete(cachekey) |
- cachekey = None |
- cached_value = None |
- else: |
- cachekey = None |
- |
- if method in self.optimistic_concurrency_methods and self.cache and 'etag' in info and not self.ignore_etag and 'if-match' not in headers: |
- # http://www.w3.org/1999/04/Editing/ |
- headers['if-match'] = info['etag'] |
- |
- if method not in ["GET", "HEAD"] and self.cache and cachekey: |
- # RFC 2616 Section 13.10 |
- self.cache.delete(cachekey) |
- |
- # Check the vary header in the cache to see if this request |
- # matches what varies in the cache. |
- if method in ['GET', 'HEAD'] and 'vary' in info: |
- vary = info['vary'] |
- vary_headers = vary.lower().replace(' ', '').split(',') |
- for header in vary_headers: |
- key = '-varied-%s' % header |
- value = info[key] |
- if headers.get(header, None) != value: |
- cached_value = None |
- break |
- |
- if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: |
- if '-x-permanent-redirect-url' in info: |
- # Should cached permanent redirects be counted in our redirection count? For now, yes. |
- if redirections <= 0: |
- raise RedirectLimit("Redirected more times than redirection_limit allows.", {}, "") |
- (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) |
- response.previous = Response(info) |
- response.previous.fromcache = True |
- else: |
- # Determine our course of action: |
- # Is the cached entry fresh or stale? |
- # Has the client requested a non-cached response? |
- # |
- # There seems to be three possible answers: |
- # 1. [FRESH] Return the cache entry w/o doing a GET |
- # 2. [STALE] Do the GET (but add in cache validators if available) |
- # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request |
- entry_disposition = _entry_disposition(info, headers) |
- |
- if entry_disposition == "FRESH": |
- if not cached_value: |
- info['status'] = '504' |
- content = b"" |
- response = Response(info) |
- if cached_value: |
- response.fromcache = True |
- return (response, content) |
- |
- if entry_disposition == "STALE": |
- if 'etag' in info and not self.ignore_etag and not 'if-none-match' in headers: |
- headers['if-none-match'] = info['etag'] |
- if 'last-modified' in info and not 'last-modified' in headers: |
- headers['if-modified-since'] = info['last-modified'] |
- elif entry_disposition == "TRANSPARENT": |
- pass |
- |
- (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) |
- |
- if response.status == 304 and method == "GET": |
- # Rewrite the cache entry with the new end-to-end headers |
- # Take all headers that are in response |
- # and overwrite their values in info. |
- # unless they are hop-by-hop, or are listed in the connection header. |
- |
- for key in _get_end2end_headers(response): |
- info[key] = response[key] |
- merged_response = Response(info) |
- if hasattr(response, "_stale_digest"): |
- merged_response._stale_digest = response._stale_digest |
- _updateCache(headers, merged_response, content, self.cache, cachekey) |
- response = merged_response |
- response.status = 200 |
- response.fromcache = True |
- |
- elif response.status == 200: |
- content = new_content |
- else: |
- self.cache.delete(cachekey) |
- content = new_content |
- else: |
- cc = _parse_cache_control(headers) |
- if 'only-if-cached'in cc: |
- info['status'] = '504' |
- response = Response(info) |
- content = b"" |
- else: |
- (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) |
- except Exception as e: |
- if self.force_exception_to_status_code: |
- if isinstance(e, HttpLib2ErrorWithResponse): |
- response = e.response |
- content = e.content |
- response.status = 500 |
- response.reason = str(e) |
- elif isinstance(e, socket.timeout): |
- content = b"Request Timeout" |
- response = Response({ |
- "content-type": "text/plain", |
- "status": "408", |
- "content-length": len(content) |
- }) |
- response.reason = "Request Timeout" |
- else: |
- content = str(e).encode('utf-8') |
- response = Response({ |
- "content-type": "text/plain", |
- "status": "400", |
- "content-length": len(content) |
- }) |
- response.reason = "Bad Request" |
- else: |
- raise |
- |
- |
- return (response, content) |
- |
- |
- |
-class Response(dict): |
- """An object more like email.message than httplib.HTTPResponse.""" |
- |
- """Is this response from our local cache""" |
- fromcache = False |
- |
- """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """ |
- version = 11 |
- |
- "Status code returned by server. " |
- status = 200 |
- |
- """Reason phrase returned by server.""" |
- reason = "Ok" |
- |
- previous = None |
- |
- def __init__(self, info): |
- # info is either an email.message or |
- # an httplib.HTTPResponse object. |
- if isinstance(info, http.client.HTTPResponse): |
- for key, value in info.getheaders(): |
- key = key.lower() |
- prev = self.get(key) |
- if prev is not None: |
- value = ', '.join((prev, value)) |
- self[key] = value |
- self.status = info.status |
- self['status'] = str(self.status) |
- self.reason = info.reason |
- self.version = info.version |
- elif isinstance(info, email.message.Message): |
- for key, value in list(info.items()): |
- self[key.lower()] = value |
- self.status = int(self['status']) |
- else: |
- for key, value in info.items(): |
- self[key.lower()] = value |
- self.status = int(self.get('status', self.status)) |
- |
- |
- def __getattr__(self, name): |
- if name == 'dict': |
- return self |
- else: |
- raise AttributeError(name) |