Add gsutil/third_party to telemetry/third_party/gsutilz/third_party.

tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py

Index: tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py
diff --git a/tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py b/tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py
new file mode 100755
index 0000000000000000000000000000000000000000..94c7e324615f4389ad52c08db78d3d482e59a42a
--- /dev/null
+++ b/tools/telemetry/third_party/gsutilz/third_party/apitools/apitools/base/py/http_wrapper.py
@@ -0,0 +1,369 @@
+#!/usr/bin/env python
+"""HTTP wrapper for apitools.
+
+This library wraps the underlying http library we use, which is
+currently httplib2.
+"""
+
+import collections
+import contextlib
+import logging
+import socket
+import time
+
+import httplib2
+import six
+from six.moves import http_client
+from six.moves.urllib import parse
+
+from apitools.base.py import exceptions
+from apitools.base.py import util
+
+__all__ = [
+    'CheckResponse',
+    'GetHttp',
+    'HandleExceptionsAndRebuildHttpConnections',
+    'MakeRequest',
+    'RebuildHttpConnections',
+    'Request',
+    'Response',
+    'RethrowExceptionHandler',
+]
+
+
+# 308 and 429 don't have names in httplib.
+RESUME_INCOMPLETE = 308
+TOO_MANY_REQUESTS = 429
+_REDIRECT_STATUS_CODES = (
+    http_client.MOVED_PERMANENTLY,
+    http_client.FOUND,
+    http_client.SEE_OTHER,
+    http_client.TEMPORARY_REDIRECT,
+    RESUME_INCOMPLETE,
+)
+
+# http: An httplib2.Http instance.
+# http_request: A http_wrapper.Request.
+# exc: Exception being raised.
+# num_retries: Number of retries consumed; used for exponential backoff.
+ExceptionRetryArgs = collections.namedtuple(
+    'ExceptionRetryArgs', ['http', 'http_request', 'exc', 'num_retries'])
+
+
+@contextlib.contextmanager
+def _Httplib2Debuglevel(http_request, level, http=None):
+    """Temporarily change the value of httplib2.debuglevel, if necessary.
+
+    If http_request has a `loggable_body` distinct from `body`, then we
+    need to prevent httplib2 from logging the full body. This sets
+    httplib2.debuglevel for the duration of the `with` block; however,
+    that alone won't change the value of existing HTTP connections. If
+    an httplib2.Http object is provided, we'll also change the level on
+    any cached connections attached to it.
+
+    Args:
+      http_request: a Request we're logging.
+      level: (int) the debuglevel for logging.
+      http: (optional) an httplib2.Http whose connections we should
+        set the debuglevel on.
+
+    Yields:
+      None.
+    """
+    if http_request.loggable_body is None:
+        yield
+        return
+    old_level = httplib2.debuglevel
+    http_levels = {}
+    httplib2.debuglevel = level
+    if http is not None:
+        for connection_key, connection in http.connections.items():
+            # httplib2 stores two kinds of values in this dict, connection
+            # classes and instances. Since the connection types are all
+            # old-style classes, we can't easily distinguish by connection
+            # type -- so instead we use the key pattern.
+            if ':' not in connection_key:
+                continue
+            http_levels[connection_key] = connection.debuglevel
+            connection.set_debuglevel(level)
+    yield
+    httplib2.debuglevel = old_level
+    if http is not None:
+        for connection_key, old_level in http_levels.items():
+            if connection_key in http.connections:
+                http.connections[connection_key].set_debuglevel(old_level)
+
+
+class Request(object):
+
+    """Class encapsulating the data for an HTTP request."""
+
+    def __init__(self, url='', http_method='GET', headers=None, body=''):
+        self.url = url
+        self.http_method = http_method
+        self.headers = headers or {}
+        self.__body = None
+        self.__loggable_body = None
+        self.body = body
+
+    @property
+    def loggable_body(self):
+        return self.__loggable_body
+
+    @loggable_body.setter
+    def loggable_body(self, value):
+        if self.body is None:
+            raise exceptions.RequestError(
+                'Cannot set loggable body on request with no body')
+        self.__loggable_body = value
+
+    @property
+    def body(self):
+        return self.__body
+
+    @body.setter
+    def body(self, value):
+        """Sets the request body; handles logging and length measurement."""
+        self.__body = value
+        if value is not None:
+            # Avoid calling len() which cannot exceed 4GiB in 32-bit python.
+            body_length = getattr(
+                self.__body, 'length', None) or len(self.__body)
+            self.headers['content-length'] = str(body_length)
+        else:
+            self.headers.pop('content-length', None)
+        # This line ensures we don't try to print large requests.
+        if not isinstance(value, (type(None), six.string_types)):
+            self.loggable_body = '<media body>'
+
+
+# Note: currently the order of fields here is important, since we want
+# to be able to pass in the result from httplib2.request.
+class Response(collections.namedtuple(
+        'HttpResponse', ['info', 'content', 'request_url'])):
+
+    """Class encapsulating data for an HTTP response."""
+    __slots__ = ()
+
+    def __len__(self):
+        return self.length
+
+    @property
+    def length(self):
+        """Return the length of this response.
+
+        We expose this as an attribute since using len() directly can fail
+        for responses larger than sys.maxint.
+
+        Returns:
+          Response length (as int or long)
+        """
+        def ProcessContentRange(content_range):
+            _, _, range_spec = content_range.partition(' ')
+            byte_range, _, _ = range_spec.partition('/')
+            start, _, end = byte_range.partition('-')
+            return int(end) - int(start) + 1
+
+        if '-content-encoding' in self.info and 'content-range' in self.info:
+            # httplib2 rewrites content-length in the case of a compressed
+            # transfer; we can't trust the content-length header in that
+            # case, but we *can* trust content-range, if it's present.
+            return ProcessContentRange(self.info['content-range'])
+        elif 'content-length' in self.info:
+            return int(self.info.get('content-length'))
+        elif 'content-range' in self.info:
+            return ProcessContentRange(self.info['content-range'])
+        return len(self.content)
+
+    @property
+    def status_code(self):
+        return int(self.info['status'])
+
+    @property
+    def retry_after(self):
+        if 'retry-after' in self.info:
+            return int(self.info['retry-after'])
+
+    @property
+    def is_redirect(self):
+        return (self.status_code in _REDIRECT_STATUS_CODES and
+                'location' in self.info)
+
+
+def CheckResponse(response):
+    if response is None:
+        # Caller shouldn't call us if the response is None, but handle anyway.
+        raise exceptions.RequestError(
+            'Request to url %s did not return a response.' %
+            response.request_url)
+    elif (response.status_code >= 500 or
+          response.status_code == TOO_MANY_REQUESTS):
+        raise exceptions.BadStatusCodeError.FromResponse(response)
+    elif response.retry_after:
+        raise exceptions.RetryAfterError.FromResponse(response)
+
+
+def RebuildHttpConnections(http):
+    """Rebuilds all http connections in the httplib2.Http instance.
+
+    httplib2 overloads the map in http.connections to contain two different
+    types of values:
+    { scheme string:  connection class } and
+    { scheme + authority string : actual http connection }
+    Here we remove all of the entries for actual connections so that on the
+    next request httplib2 will rebuild them from the connection types.
+
+    Args:
+      http: An httplib2.Http instance.
+    """
+    if getattr(http, 'connections', None):
+        for conn_key in list(http.connections.keys()):
+            if ':' in conn_key:
+                del http.connections[conn_key]
+
+
+def RethrowExceptionHandler(*unused_args):
+    raise
+
+
+def HandleExceptionsAndRebuildHttpConnections(retry_args):
+    """Exception handler for http failures.
+
+    This catches known failures and rebuilds the underlying HTTP connections.
+
+    Args:
+      retry_args: An ExceptionRetryArgs tuple.
+    """
+    # If the server indicates how long to wait, use that value.  Otherwise,
+    # calculate the wait time on our own.
+    retry_after = None
+
+    # Transport failures
+    if isinstance(retry_args.exc, (http_client.BadStatusLine,
+                                   http_client.IncompleteRead,
+                                   http_client.ResponseNotReady)):
+        logging.debug('Caught HTTP error %s, retrying: %s',
+                      type(retry_args.exc).__name__, retry_args.exc)
+    elif isinstance(retry_args.exc, socket.error):
+        logging.debug('Caught socket error, retrying: %s', retry_args.exc)
+    elif isinstance(retry_args.exc, socket.gaierror):
+        logging.debug(
+            'Caught socket address error, retrying: %s', retry_args.exc)
+    elif isinstance(retry_args.exc, socket.timeout):
+        logging.debug(
+            'Caught socket timeout error, retrying: %s', retry_args.exc)
+    elif isinstance(retry_args.exc, httplib2.ServerNotFoundError):
+        logging.debug(
+            'Caught server not found error, retrying: %s', retry_args.exc)
+    elif isinstance(retry_args.exc, ValueError):
+        # oauth2client tries to JSON-decode the response, which can result
+        # in a ValueError if the response was invalid. Until that is fixed in
+        # oauth2client, need to handle it here.
+        logging.debug('Response content was invalid (%s), retrying',
+                      retry_args.exc)
+    elif isinstance(retry_args.exc, exceptions.RequestError):
+        logging.debug('Request returned no response, retrying')
+    # API-level failures
+    elif isinstance(retry_args.exc, exceptions.BadStatusCodeError):
+        logging.debug('Response returned status %s, retrying',
+                      retry_args.exc.status_code)
+    elif isinstance(retry_args.exc, exceptions.RetryAfterError):
+        logging.debug('Response returned a retry-after header, retrying')
+        retry_after = retry_args.exc.retry_after
+    else:
+        raise
+    RebuildHttpConnections(retry_args.http)
+    logging.debug('Retrying request to url %s after exception %s',
+                  retry_args.http_request.url, retry_args.exc)
+    time.sleep(
+        retry_after or util.CalculateWaitForRetry(retry_args.num_retries))
+
+
+def MakeRequest(http, http_request, retries=7, redirections=5,
+                retry_func=HandleExceptionsAndRebuildHttpConnections,
+                check_response_func=CheckResponse):
+    """Send http_request via the given http, performing error/retry handling.
+
+    Args:
+      http: An httplib2.Http instance, or a http multiplexer that delegates to
+          an underlying http, for example, HTTPMultiplexer.
+      http_request: A Request to send.
+      retries: (int, default 5) Number of retries to attempt on 5XX replies.
+      redirections: (int, default 5) Number of redirects to follow.
+      retry_func: Function to handle retries on exceptions. Arguments are
+          (Httplib2.Http, Request, Exception, int num_retries).
+      check_response_func: Function to validate the HTTP response.
+          Arguments are (Response, response content, url).
+
+    Raises:
+      InvalidDataFromServerError: if there is no response after retries.
+
+    Returns:
+      A Response object.
+
+    """
+    retry = 0
+    while True:
+        try:
+            return _MakeRequestNoRetry(
+                http, http_request, redirections=redirections,
+                check_response_func=check_response_func)
+        # retry_func will consume the exception types it handles and raise.
+        # pylint: disable=broad-except
+        except Exception as e:
+            retry += 1
+            if retry >= retries:
+                raise
+            else:
+                retry_func(ExceptionRetryArgs(http, http_request, e, retry))
+
+
+def _MakeRequestNoRetry(http, http_request, redirections=5,
+                        check_response_func=CheckResponse):
+    """Send http_request via the given http.
+
+    This wrapper exists to handle translation between the plain httplib2
+    request/response types and the Request and Response types above.
+
+    Args:
+      http: An httplib2.Http instance, or a http multiplexer that delegates to
+          an underlying http, for example, HTTPMultiplexer.
+      http_request: A Request to send.
+      redirections: (int, default 5) Number of redirects to follow.
+      check_response_func: Function to validate the HTTP response.
+          Arguments are (Response, response content, url).
+
+    Returns:
+      A Response object.
+
+    Raises:
+      RequestError if no response could be parsed.
+
+    """
+    connection_type = None
+    # Handle overrides for connection types.  This is used if the caller
+    # wants control over the underlying connection for managing callbacks
+    # or hash digestion.
+    if getattr(http, 'connections', None):
+        url_scheme = parse.urlsplit(http_request.url).scheme
+        if url_scheme and url_scheme in http.connections:
+            connection_type = http.connections[url_scheme]
+
+    # Custom printing only at debuglevel 4
+    new_debuglevel = 4 if httplib2.debuglevel == 4 else 0
+    with _Httplib2Debuglevel(http_request, new_debuglevel, http=http):
+        info, content = http.request(
+            str(http_request.url), method=str(http_request.http_method),
+            body=http_request.body, headers=http_request.headers,
+            redirections=redirections, connection_type=connection_type)
+
+    if info is None:
+        raise exceptions.RequestError()
+
+    response = Response(info, content, http_request.url)
+    check_response_func(response)
+    return response
+
+
+def GetHttp(**kwds):
+    return httplib2.Http(**kwds)