| Index: tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py
|
| diff --git a/tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py b/tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py
|
| deleted file mode 100755
|
| index 94c7e324615f4389ad52c08db78d3d482e59a42a..0000000000000000000000000000000000000000
|
| --- a/tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py
|
| +++ /dev/null
|
| @@ -1,369 +0,0 @@
|
| -#!/usr/bin/env python
|
| -"""HTTP wrapper for apitools.
|
| -
|
| -This library wraps the underlying http library we use, which is
|
| -currently httplib2.
|
| -"""
|
| -
|
| -import collections
|
| -import contextlib
|
| -import logging
|
| -import socket
|
| -import time
|
| -
|
| -import httplib2
|
| -import six
|
| -from six.moves import http_client
|
| -from six.moves.urllib import parse
|
| -
|
| -from apitools.base.py import exceptions
|
| -from apitools.base.py import util
|
| -
|
| -__all__ = [
|
| - 'CheckResponse',
|
| - 'GetHttp',
|
| - 'HandleExceptionsAndRebuildHttpConnections',
|
| - 'MakeRequest',
|
| - 'RebuildHttpConnections',
|
| - 'Request',
|
| - 'Response',
|
| - 'RethrowExceptionHandler',
|
| -]
|
| -
|
| -
|
| -# 308 and 429 don't have names in httplib.
|
| -RESUME_INCOMPLETE = 308
|
| -TOO_MANY_REQUESTS = 429
|
| -_REDIRECT_STATUS_CODES = (
|
| - http_client.MOVED_PERMANENTLY,
|
| - http_client.FOUND,
|
| - http_client.SEE_OTHER,
|
| - http_client.TEMPORARY_REDIRECT,
|
| - RESUME_INCOMPLETE,
|
| -)
|
| -
|
| -# http: An httplib2.Http instance.
|
| -# http_request: A http_wrapper.Request.
|
| -# exc: Exception being raised.
|
| -# num_retries: Number of retries consumed; used for exponential backoff.
|
| -ExceptionRetryArgs = collections.namedtuple(
|
| - 'ExceptionRetryArgs', ['http', 'http_request', 'exc', 'num_retries'])
|
| -
|
| -
|
| -@contextlib.contextmanager
|
| -def _Httplib2Debuglevel(http_request, level, http=None):
|
| - """Temporarily change the value of httplib2.debuglevel, if necessary.
|
| -
|
| - If http_request has a `loggable_body` distinct from `body`, then we
|
| - need to prevent httplib2 from logging the full body. This sets
|
| - httplib2.debuglevel for the duration of the `with` block; however,
|
| - that alone won't change the value of existing HTTP connections. If
|
| - an httplib2.Http object is provided, we'll also change the level on
|
| - any cached connections attached to it.
|
| -
|
| - Args:
|
| - http_request: a Request we're logging.
|
| - level: (int) the debuglevel for logging.
|
| - http: (optional) an httplib2.Http whose connections we should
|
| - set the debuglevel on.
|
| -
|
| - Yields:
|
| - None.
|
| - """
|
| - if http_request.loggable_body is None:
|
| - yield
|
| - return
|
| - old_level = httplib2.debuglevel
|
| - http_levels = {}
|
| - httplib2.debuglevel = level
|
| - if http is not None:
|
| - for connection_key, connection in http.connections.items():
|
| - # httplib2 stores two kinds of values in this dict, connection
|
| - # classes and instances. Since the connection types are all
|
| - # old-style classes, we can't easily distinguish by connection
|
| - # type -- so instead we use the key pattern.
|
| - if ':' not in connection_key:
|
| - continue
|
| - http_levels[connection_key] = connection.debuglevel
|
| - connection.set_debuglevel(level)
|
| - yield
|
| - httplib2.debuglevel = old_level
|
| - if http is not None:
|
| - for connection_key, old_level in http_levels.items():
|
| - if connection_key in http.connections:
|
| - http.connections[connection_key].set_debuglevel(old_level)
|
| -
|
| -
|
| -class Request(object):
|
| -
|
| - """Class encapsulating the data for an HTTP request."""
|
| -
|
| - def __init__(self, url='', http_method='GET', headers=None, body=''):
|
| - self.url = url
|
| - self.http_method = http_method
|
| - self.headers = headers or {}
|
| - self.__body = None
|
| - self.__loggable_body = None
|
| - self.body = body
|
| -
|
| - @property
|
| - def loggable_body(self):
|
| - return self.__loggable_body
|
| -
|
| - @loggable_body.setter
|
| - def loggable_body(self, value):
|
| - if self.body is None:
|
| - raise exceptions.RequestError(
|
| - 'Cannot set loggable body on request with no body')
|
| - self.__loggable_body = value
|
| -
|
| - @property
|
| - def body(self):
|
| - return self.__body
|
| -
|
| - @body.setter
|
| - def body(self, value):
|
| - """Sets the request body; handles logging and length measurement."""
|
| - self.__body = value
|
| - if value is not None:
|
| - # Avoid calling len() which cannot exceed 4GiB in 32-bit python.
|
| - body_length = getattr(
|
| - self.__body, 'length', None) or len(self.__body)
|
| - self.headers['content-length'] = str(body_length)
|
| - else:
|
| - self.headers.pop('content-length', None)
|
| - # This line ensures we don't try to print large requests.
|
| - if not isinstance(value, (type(None), six.string_types)):
|
| - self.loggable_body = '<media body>'
|
| -
|
| -
|
| -# Note: currently the order of fields here is important, since we want
|
| -# to be able to pass in the result from httplib2.request.
|
| -class Response(collections.namedtuple(
|
| - 'HttpResponse', ['info', 'content', 'request_url'])):
|
| -
|
| - """Class encapsulating data for an HTTP response."""
|
| - __slots__ = ()
|
| -
|
| - def __len__(self):
|
| - return self.length
|
| -
|
| - @property
|
| - def length(self):
|
| - """Return the length of this response.
|
| -
|
| - We expose this as an attribute since using len() directly can fail
|
| - for responses larger than sys.maxint.
|
| -
|
| - Returns:
|
| - Response length (as int or long)
|
| - """
|
| - def ProcessContentRange(content_range):
|
| - _, _, range_spec = content_range.partition(' ')
|
| - byte_range, _, _ = range_spec.partition('/')
|
| - start, _, end = byte_range.partition('-')
|
| - return int(end) - int(start) + 1
|
| -
|
| - if '-content-encoding' in self.info and 'content-range' in self.info:
|
| - # httplib2 rewrites content-length in the case of a compressed
|
| - # transfer; we can't trust the content-length header in that
|
| - # case, but we *can* trust content-range, if it's present.
|
| - return ProcessContentRange(self.info['content-range'])
|
| - elif 'content-length' in self.info:
|
| - return int(self.info.get('content-length'))
|
| - elif 'content-range' in self.info:
|
| - return ProcessContentRange(self.info['content-range'])
|
| - return len(self.content)
|
| -
|
| - @property
|
| - def status_code(self):
|
| - return int(self.info['status'])
|
| -
|
| - @property
|
| - def retry_after(self):
|
| - if 'retry-after' in self.info:
|
| - return int(self.info['retry-after'])
|
| -
|
| - @property
|
| - def is_redirect(self):
|
| - return (self.status_code in _REDIRECT_STATUS_CODES and
|
| - 'location' in self.info)
|
| -
|
| -
|
| -def CheckResponse(response):
|
| - if response is None:
|
| - # Caller shouldn't call us if the response is None, but handle anyway.
|
| - raise exceptions.RequestError(
|
| - 'Request to url %s did not return a response.' %
|
| - response.request_url)
|
| - elif (response.status_code >= 500 or
|
| - response.status_code == TOO_MANY_REQUESTS):
|
| - raise exceptions.BadStatusCodeError.FromResponse(response)
|
| - elif response.retry_after:
|
| - raise exceptions.RetryAfterError.FromResponse(response)
|
| -
|
| -
|
| -def RebuildHttpConnections(http):
|
| - """Rebuilds all http connections in the httplib2.Http instance.
|
| -
|
| - httplib2 overloads the map in http.connections to contain two different
|
| - types of values:
|
| - { scheme string: connection class } and
|
| - { scheme + authority string : actual http connection }
|
| - Here we remove all of the entries for actual connections so that on the
|
| - next request httplib2 will rebuild them from the connection types.
|
| -
|
| - Args:
|
| - http: An httplib2.Http instance.
|
| - """
|
| - if getattr(http, 'connections', None):
|
| - for conn_key in list(http.connections.keys()):
|
| - if ':' in conn_key:
|
| - del http.connections[conn_key]
|
| -
|
| -
|
| -def RethrowExceptionHandler(*unused_args):
|
| - raise
|
| -
|
| -
|
| -def HandleExceptionsAndRebuildHttpConnections(retry_args):
|
| - """Exception handler for http failures.
|
| -
|
| - This catches known failures and rebuilds the underlying HTTP connections.
|
| -
|
| - Args:
|
| - retry_args: An ExceptionRetryArgs tuple.
|
| - """
|
| - # If the server indicates how long to wait, use that value. Otherwise,
|
| - # calculate the wait time on our own.
|
| - retry_after = None
|
| -
|
| - # Transport failures
|
| - if isinstance(retry_args.exc, (http_client.BadStatusLine,
|
| - http_client.IncompleteRead,
|
| - http_client.ResponseNotReady)):
|
| - logging.debug('Caught HTTP error %s, retrying: %s',
|
| - type(retry_args.exc).__name__, retry_args.exc)
|
| - elif isinstance(retry_args.exc, socket.error):
|
| - logging.debug('Caught socket error, retrying: %s', retry_args.exc)
|
| - elif isinstance(retry_args.exc, socket.gaierror):
|
| - logging.debug(
|
| - 'Caught socket address error, retrying: %s', retry_args.exc)
|
| - elif isinstance(retry_args.exc, socket.timeout):
|
| - logging.debug(
|
| - 'Caught socket timeout error, retrying: %s', retry_args.exc)
|
| - elif isinstance(retry_args.exc, httplib2.ServerNotFoundError):
|
| - logging.debug(
|
| - 'Caught server not found error, retrying: %s', retry_args.exc)
|
| - elif isinstance(retry_args.exc, ValueError):
|
| - # oauth2client tries to JSON-decode the response, which can result
|
| - # in a ValueError if the response was invalid. Until that is fixed in
|
| - # oauth2client, need to handle it here.
|
| - logging.debug('Response content was invalid (%s), retrying',
|
| - retry_args.exc)
|
| - elif isinstance(retry_args.exc, exceptions.RequestError):
|
| - logging.debug('Request returned no response, retrying')
|
| - # API-level failures
|
| - elif isinstance(retry_args.exc, exceptions.BadStatusCodeError):
|
| - logging.debug('Response returned status %s, retrying',
|
| - retry_args.exc.status_code)
|
| - elif isinstance(retry_args.exc, exceptions.RetryAfterError):
|
| - logging.debug('Response returned a retry-after header, retrying')
|
| - retry_after = retry_args.exc.retry_after
|
| - else:
|
| - raise
|
| - RebuildHttpConnections(retry_args.http)
|
| - logging.debug('Retrying request to url %s after exception %s',
|
| - retry_args.http_request.url, retry_args.exc)
|
| - time.sleep(
|
| - retry_after or util.CalculateWaitForRetry(retry_args.num_retries))
|
| -
|
| -
|
| -def MakeRequest(http, http_request, retries=7, redirections=5,
|
| - retry_func=HandleExceptionsAndRebuildHttpConnections,
|
| - check_response_func=CheckResponse):
|
| - """Send http_request via the given http, performing error/retry handling.
|
| -
|
| - Args:
|
| - http: An httplib2.Http instance, or a http multiplexer that delegates to
|
| - an underlying http, for example, HTTPMultiplexer.
|
| - http_request: A Request to send.
|
| - retries: (int, default 5) Number of retries to attempt on 5XX replies.
|
| - redirections: (int, default 5) Number of redirects to follow.
|
| - retry_func: Function to handle retries on exceptions. Arguments are
|
| - (Httplib2.Http, Request, Exception, int num_retries).
|
| - check_response_func: Function to validate the HTTP response.
|
| - Arguments are (Response, response content, url).
|
| -
|
| - Raises:
|
| - InvalidDataFromServerError: if there is no response after retries.
|
| -
|
| - Returns:
|
| - A Response object.
|
| -
|
| - """
|
| - retry = 0
|
| - while True:
|
| - try:
|
| - return _MakeRequestNoRetry(
|
| - http, http_request, redirections=redirections,
|
| - check_response_func=check_response_func)
|
| - # retry_func will consume the exception types it handles and raise.
|
| - # pylint: disable=broad-except
|
| - except Exception as e:
|
| - retry += 1
|
| - if retry >= retries:
|
| - raise
|
| - else:
|
| - retry_func(ExceptionRetryArgs(http, http_request, e, retry))
|
| -
|
| -
|
| -def _MakeRequestNoRetry(http, http_request, redirections=5,
|
| - check_response_func=CheckResponse):
|
| - """Send http_request via the given http.
|
| -
|
| - This wrapper exists to handle translation between the plain httplib2
|
| - request/response types and the Request and Response types above.
|
| -
|
| - Args:
|
| - http: An httplib2.Http instance, or a http multiplexer that delegates to
|
| - an underlying http, for example, HTTPMultiplexer.
|
| - http_request: A Request to send.
|
| - redirections: (int, default 5) Number of redirects to follow.
|
| - check_response_func: Function to validate the HTTP response.
|
| - Arguments are (Response, response content, url).
|
| -
|
| - Returns:
|
| - A Response object.
|
| -
|
| - Raises:
|
| - RequestError if no response could be parsed.
|
| -
|
| - """
|
| - connection_type = None
|
| - # Handle overrides for connection types. This is used if the caller
|
| - # wants control over the underlying connection for managing callbacks
|
| - # or hash digestion.
|
| - if getattr(http, 'connections', None):
|
| - url_scheme = parse.urlsplit(http_request.url).scheme
|
| - if url_scheme and url_scheme in http.connections:
|
| - connection_type = http.connections[url_scheme]
|
| -
|
| - # Custom printing only at debuglevel 4
|
| - new_debuglevel = 4 if httplib2.debuglevel == 4 else 0
|
| - with _Httplib2Debuglevel(http_request, new_debuglevel, http=http):
|
| - info, content = http.request(
|
| - str(http_request.url), method=str(http_request.http_method),
|
| - body=http_request.body, headers=http_request.headers,
|
| - redirections=redirections, connection_type=connection_type)
|
| -
|
| - if info is None:
|
| - raise exceptions.RequestError()
|
| -
|
| - response = Response(info, content, http_request.url)
|
| - check_response_func(response)
|
| - return response
|
| -
|
| -
|
| -def GetHttp(**kwds):
|
| - return httplib2.Http(**kwds)
|
|
|