Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(82)

Unified Diff: tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py

Issue 1260493004: Revert "Add gsutil 4.13 to telemetry/third_party" (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py
diff --git a/tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py b/tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py
deleted file mode 100644
index 94c7e324615f4389ad52c08db78d3d482e59a42a..0000000000000000000000000000000000000000
--- a/tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/http_wrapper.py
+++ /dev/null
@@ -1,369 +0,0 @@
-#!/usr/bin/env python
-"""HTTP wrapper for apitools.
-
-This library wraps the underlying http library we use, which is
-currently httplib2.
-"""
-
-import collections
-import contextlib
-import logging
-import socket
-import time
-
-import httplib2
-import six
-from six.moves import http_client
-from six.moves.urllib import parse
-
-from apitools.base.py import exceptions
-from apitools.base.py import util
-
-__all__ = [
- 'CheckResponse',
- 'GetHttp',
- 'HandleExceptionsAndRebuildHttpConnections',
- 'MakeRequest',
- 'RebuildHttpConnections',
- 'Request',
- 'Response',
- 'RethrowExceptionHandler',
-]
-
-
-# 308 and 429 don't have names in httplib.
-RESUME_INCOMPLETE = 308
-TOO_MANY_REQUESTS = 429
-_REDIRECT_STATUS_CODES = (
- http_client.MOVED_PERMANENTLY,
- http_client.FOUND,
- http_client.SEE_OTHER,
- http_client.TEMPORARY_REDIRECT,
- RESUME_INCOMPLETE,
-)
-
-# http: An httplib2.Http instance.
-# http_request: A http_wrapper.Request.
-# exc: Exception being raised.
-# num_retries: Number of retries consumed; used for exponential backoff.
-ExceptionRetryArgs = collections.namedtuple(
- 'ExceptionRetryArgs', ['http', 'http_request', 'exc', 'num_retries'])
-
-
-@contextlib.contextmanager
-def _Httplib2Debuglevel(http_request, level, http=None):
- """Temporarily change the value of httplib2.debuglevel, if necessary.
-
- If http_request has a `loggable_body` distinct from `body`, then we
- need to prevent httplib2 from logging the full body. This sets
- httplib2.debuglevel for the duration of the `with` block; however,
- that alone won't change the value of existing HTTP connections. If
- an httplib2.Http object is provided, we'll also change the level on
- any cached connections attached to it.
-
- Args:
- http_request: a Request we're logging.
- level: (int) the debuglevel for logging.
- http: (optional) an httplib2.Http whose connections we should
- set the debuglevel on.
-
- Yields:
- None.
- """
- if http_request.loggable_body is None:
- yield
- return
- old_level = httplib2.debuglevel
- http_levels = {}
- httplib2.debuglevel = level
- if http is not None:
- for connection_key, connection in http.connections.items():
- # httplib2 stores two kinds of values in this dict, connection
- # classes and instances. Since the connection types are all
- # old-style classes, we can't easily distinguish by connection
- # type -- so instead we use the key pattern.
- if ':' not in connection_key:
- continue
- http_levels[connection_key] = connection.debuglevel
- connection.set_debuglevel(level)
- yield
- httplib2.debuglevel = old_level
- if http is not None:
- for connection_key, old_level in http_levels.items():
- if connection_key in http.connections:
- http.connections[connection_key].set_debuglevel(old_level)
-
-
-class Request(object):
-
- """Class encapsulating the data for an HTTP request."""
-
- def __init__(self, url='', http_method='GET', headers=None, body=''):
- self.url = url
- self.http_method = http_method
- self.headers = headers or {}
- self.__body = None
- self.__loggable_body = None
- self.body = body
-
- @property
- def loggable_body(self):
- return self.__loggable_body
-
- @loggable_body.setter
- def loggable_body(self, value):
- if self.body is None:
- raise exceptions.RequestError(
- 'Cannot set loggable body on request with no body')
- self.__loggable_body = value
-
- @property
- def body(self):
- return self.__body
-
- @body.setter
- def body(self, value):
- """Sets the request body; handles logging and length measurement."""
- self.__body = value
- if value is not None:
- # Avoid calling len() which cannot exceed 4GiB in 32-bit python.
- body_length = getattr(
- self.__body, 'length', None) or len(self.__body)
- self.headers['content-length'] = str(body_length)
- else:
- self.headers.pop('content-length', None)
- # This line ensures we don't try to print large requests.
- if not isinstance(value, (type(None), six.string_types)):
- self.loggable_body = '<media body>'
-
-
-# Note: currently the order of fields here is important, since we want
-# to be able to pass in the result from httplib2.request.
-class Response(collections.namedtuple(
- 'HttpResponse', ['info', 'content', 'request_url'])):
-
- """Class encapsulating data for an HTTP response."""
- __slots__ = ()
-
- def __len__(self):
- return self.length
-
- @property
- def length(self):
- """Return the length of this response.
-
- We expose this as an attribute since using len() directly can fail
- for responses larger than sys.maxint.
-
- Returns:
- Response length (as int or long)
- """
- def ProcessContentRange(content_range):
- _, _, range_spec = content_range.partition(' ')
- byte_range, _, _ = range_spec.partition('/')
- start, _, end = byte_range.partition('-')
- return int(end) - int(start) + 1
-
- if '-content-encoding' in self.info and 'content-range' in self.info:
- # httplib2 rewrites content-length in the case of a compressed
- # transfer; we can't trust the content-length header in that
- # case, but we *can* trust content-range, if it's present.
- return ProcessContentRange(self.info['content-range'])
- elif 'content-length' in self.info:
- return int(self.info.get('content-length'))
- elif 'content-range' in self.info:
- return ProcessContentRange(self.info['content-range'])
- return len(self.content)
-
- @property
- def status_code(self):
- return int(self.info['status'])
-
- @property
- def retry_after(self):
- if 'retry-after' in self.info:
- return int(self.info['retry-after'])
-
- @property
- def is_redirect(self):
- return (self.status_code in _REDIRECT_STATUS_CODES and
- 'location' in self.info)
-
-
-def CheckResponse(response):
- if response is None:
- # Caller shouldn't call us if the response is None, but handle anyway.
- raise exceptions.RequestError(
- 'Request to url %s did not return a response.' %
- response.request_url)
- elif (response.status_code >= 500 or
- response.status_code == TOO_MANY_REQUESTS):
- raise exceptions.BadStatusCodeError.FromResponse(response)
- elif response.retry_after:
- raise exceptions.RetryAfterError.FromResponse(response)
-
-
-def RebuildHttpConnections(http):
- """Rebuilds all http connections in the httplib2.Http instance.
-
- httplib2 overloads the map in http.connections to contain two different
- types of values:
- { scheme string: connection class } and
- { scheme + authority string : actual http connection }
- Here we remove all of the entries for actual connections so that on the
- next request httplib2 will rebuild them from the connection types.
-
- Args:
- http: An httplib2.Http instance.
- """
- if getattr(http, 'connections', None):
- for conn_key in list(http.connections.keys()):
- if ':' in conn_key:
- del http.connections[conn_key]
-
-
-def RethrowExceptionHandler(*unused_args):
- raise
-
-
-def HandleExceptionsAndRebuildHttpConnections(retry_args):
- """Exception handler for http failures.
-
- This catches known failures and rebuilds the underlying HTTP connections.
-
- Args:
- retry_args: An ExceptionRetryArgs tuple.
- """
- # If the server indicates how long to wait, use that value. Otherwise,
- # calculate the wait time on our own.
- retry_after = None
-
- # Transport failures
- if isinstance(retry_args.exc, (http_client.BadStatusLine,
- http_client.IncompleteRead,
- http_client.ResponseNotReady)):
- logging.debug('Caught HTTP error %s, retrying: %s',
- type(retry_args.exc).__name__, retry_args.exc)
- elif isinstance(retry_args.exc, socket.error):
- logging.debug('Caught socket error, retrying: %s', retry_args.exc)
- elif isinstance(retry_args.exc, socket.gaierror):
- logging.debug(
- 'Caught socket address error, retrying: %s', retry_args.exc)
- elif isinstance(retry_args.exc, socket.timeout):
- logging.debug(
- 'Caught socket timeout error, retrying: %s', retry_args.exc)
- elif isinstance(retry_args.exc, httplib2.ServerNotFoundError):
- logging.debug(
- 'Caught server not found error, retrying: %s', retry_args.exc)
- elif isinstance(retry_args.exc, ValueError):
- # oauth2client tries to JSON-decode the response, which can result
- # in a ValueError if the response was invalid. Until that is fixed in
- # oauth2client, need to handle it here.
- logging.debug('Response content was invalid (%s), retrying',
- retry_args.exc)
- elif isinstance(retry_args.exc, exceptions.RequestError):
- logging.debug('Request returned no response, retrying')
- # API-level failures
- elif isinstance(retry_args.exc, exceptions.BadStatusCodeError):
- logging.debug('Response returned status %s, retrying',
- retry_args.exc.status_code)
- elif isinstance(retry_args.exc, exceptions.RetryAfterError):
- logging.debug('Response returned a retry-after header, retrying')
- retry_after = retry_args.exc.retry_after
- else:
- raise
- RebuildHttpConnections(retry_args.http)
- logging.debug('Retrying request to url %s after exception %s',
- retry_args.http_request.url, retry_args.exc)
- time.sleep(
- retry_after or util.CalculateWaitForRetry(retry_args.num_retries))
-
-
-def MakeRequest(http, http_request, retries=7, redirections=5,
- retry_func=HandleExceptionsAndRebuildHttpConnections,
- check_response_func=CheckResponse):
- """Send http_request via the given http, performing error/retry handling.
-
- Args:
- http: An httplib2.Http instance, or a http multiplexer that delegates to
- an underlying http, for example, HTTPMultiplexer.
- http_request: A Request to send.
- retries: (int, default 5) Number of retries to attempt on 5XX replies.
- redirections: (int, default 5) Number of redirects to follow.
- retry_func: Function to handle retries on exceptions. Arguments are
- (Httplib2.Http, Request, Exception, int num_retries).
- check_response_func: Function to validate the HTTP response.
- Arguments are (Response, response content, url).
-
- Raises:
- InvalidDataFromServerError: if there is no response after retries.
-
- Returns:
- A Response object.
-
- """
- retry = 0
- while True:
- try:
- return _MakeRequestNoRetry(
- http, http_request, redirections=redirections,
- check_response_func=check_response_func)
- # retry_func will consume the exception types it handles and raise.
- # pylint: disable=broad-except
- except Exception as e:
- retry += 1
- if retry >= retries:
- raise
- else:
- retry_func(ExceptionRetryArgs(http, http_request, e, retry))
-
-
-def _MakeRequestNoRetry(http, http_request, redirections=5,
- check_response_func=CheckResponse):
- """Send http_request via the given http.
-
- This wrapper exists to handle translation between the plain httplib2
- request/response types and the Request and Response types above.
-
- Args:
- http: An httplib2.Http instance, or a http multiplexer that delegates to
- an underlying http, for example, HTTPMultiplexer.
- http_request: A Request to send.
- redirections: (int, default 5) Number of redirects to follow.
- check_response_func: Function to validate the HTTP response.
- Arguments are (Response, response content, url).
-
- Returns:
- A Response object.
-
- Raises:
- RequestError if no response could be parsed.
-
- """
- connection_type = None
- # Handle overrides for connection types. This is used if the caller
- # wants control over the underlying connection for managing callbacks
- # or hash digestion.
- if getattr(http, 'connections', None):
- url_scheme = parse.urlsplit(http_request.url).scheme
- if url_scheme and url_scheme in http.connections:
- connection_type = http.connections[url_scheme]
-
- # Custom printing only at debuglevel 4
- new_debuglevel = 4 if httplib2.debuglevel == 4 else 0
- with _Httplib2Debuglevel(http_request, new_debuglevel, http=http):
- info, content = http.request(
- str(http_request.url), method=str(http_request.http_method),
- body=http_request.body, headers=http_request.headers,
- redirections=redirections, connection_type=connection_type)
-
- if info is None:
- raise exceptions.RequestError()
-
- response = Response(info, content, http_request.url)
- check_response_func(response)
- return response
-
-
-def GetHttp(**kwds):
- return httplib2.Http(**kwds)

Powered by Google App Engine
This is Rietveld 408576698