Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(468)

Unified Diff: tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/batch.py

Issue 1260493004: Revert "Add gsutil 4.13 to telemetry/third_party" (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/batch.py
diff --git a/tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/batch.py b/tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/batch.py
deleted file mode 100644
index 4910d4ac60ae418c49ce8b472b62383ace969349..0000000000000000000000000000000000000000
--- a/tools/telemetry/third_party/gsutil/third_party/apitools/apitools/base/py/batch.py
+++ /dev/null
@@ -1,467 +0,0 @@
-#!/usr/bin/env python
-"""Library for handling batch HTTP requests for apitools."""
-
-import collections
-import email.generator as generator
-import email.mime.multipart as mime_multipart
-import email.mime.nonmultipart as mime_nonmultipart
-import email.parser as email_parser
-import itertools
-import time
-import uuid
-
-import six
-from six.moves import http_client
-from six.moves import urllib_parse
-
-from apitools.base.py import exceptions
-from apitools.base.py import http_wrapper
-
-__all__ = [
- 'BatchApiRequest',
-]
-
-
-class RequestResponseAndHandler(collections.namedtuple(
- 'RequestResponseAndHandler', ['request', 'response', 'handler'])):
-
- """Container for data related to completing an HTTP request.
-
- This contains an HTTP request, its response, and a callback for handling
- the response from the server.
-
- Attributes:
- request: An http_wrapper.Request object representing the HTTP request.
- response: The http_wrapper.Response object returned from the server.
- handler: A callback function accepting two arguments, response
- and exception. Response is an http_wrapper.Response object, and
- exception is an apiclient.errors.HttpError object if an error
- occurred, or otherwise None.
- """
-
-
-class BatchApiRequest(object):
-
- class ApiCall(object):
-
- """Holds request and response information for each request.
-
- ApiCalls are ultimately exposed to the client once the HTTP
- batch request has been completed.
-
- Attributes:
- http_request: A client-supplied http_wrapper.Request to be
- submitted to the server.
- response: A http_wrapper.Response object given by the server as a
- response to the user request, or None if an error occurred.
- exception: An apiclient.errors.HttpError object if an error
- occurred, or None.
-
- """
-
- def __init__(self, request, retryable_codes, service, method_config):
- """Initialize an individual API request.
-
- Args:
- request: An http_wrapper.Request object.
- retryable_codes: A list of integer HTTP codes that can
- be retried.
- service: A service inheriting from base_api.BaseApiService.
- method_config: Method config for the desired API request.
-
- """
- self.__retryable_codes = list(
- set(retryable_codes + [http_client.UNAUTHORIZED]))
- self.__http_response = None
- self.__service = service
- self.__method_config = method_config
-
- self.http_request = request
- # TODO(user): Add some validation to these fields.
- self.__response = None
- self.__exception = None
-
- @property
- def is_error(self):
- return self.exception is not None
-
- @property
- def response(self):
- return self.__response
-
- @property
- def exception(self):
- return self.__exception
-
- @property
- def authorization_failed(self):
- return (self.__http_response and (
- self.__http_response.status_code == http_client.UNAUTHORIZED))
-
- @property
- def terminal_state(self):
- if self.__http_response is None:
- return False
- response_code = self.__http_response.status_code
- return response_code not in self.__retryable_codes
-
- def HandleResponse(self, http_response, exception):
- """Handles an incoming http response to the request in http_request.
-
- This is intended to be used as a callback function for
- BatchHttpRequest.Add.
-
- Args:
- http_response: Deserialized http_wrapper.Response object.
- exception: apiclient.errors.HttpError object if an error
- occurred.
-
- """
- self.__http_response = http_response
- self.__exception = exception
- if self.terminal_state and not self.__exception:
- self.__response = self.__service.ProcessHttpResponse(
- self.__method_config, self.__http_response)
-
- def __init__(self, batch_url=None, retryable_codes=None):
- """Initialize a batch API request object.
-
- Args:
- batch_url: Base URL for batch API calls.
- retryable_codes: A list of integer HTTP codes that can be retried.
- """
- self.api_requests = []
- self.retryable_codes = retryable_codes or []
- self.batch_url = batch_url or 'https://www.googleapis.com/batch'
-
- def Add(self, service, method, request, global_params=None):
- """Add a request to the batch.
-
- Args:
- service: A class inheriting base_api.BaseApiService.
- method: A string indicated desired method from the service. See
- the example in the class docstring.
- request: An input message appropriate for the specified
- service.method.
- global_params: Optional additional parameters to pass into
- method.PrepareHttpRequest.
-
- Returns:
- None
-
- """
- # Retrieve the configs for the desired method and service.
- method_config = service.GetMethodConfig(method)
- upload_config = service.GetUploadConfig(method)
-
- # Prepare the HTTP Request.
- http_request = service.PrepareHttpRequest(
- method_config, request, global_params=global_params,
- upload_config=upload_config)
-
- # Create the request and add it to our master list.
- api_request = self.ApiCall(
- http_request, self.retryable_codes, service, method_config)
- self.api_requests.append(api_request)
-
- def Execute(self, http, sleep_between_polls=5, max_retries=5):
- """Execute all of the requests in the batch.
-
- Args:
- http: httplib2.Http object for use in the request.
- sleep_between_polls: Integer number of seconds to sleep between
- polls.
- max_retries: Max retries. Any requests that have not succeeded by
- this number of retries simply report the last response or
- exception, whatever it happened to be.
-
- Returns:
- List of ApiCalls.
- """
- requests = [request for request in self.api_requests
- if not request.terminal_state]
-
- for attempt in range(max_retries):
- if attempt:
- time.sleep(sleep_between_polls)
-
- # Create a batch_http_request object and populate it with
- # incomplete requests.
- batch_http_request = BatchHttpRequest(batch_url=self.batch_url)
- for request in requests:
- batch_http_request.Add(
- request.http_request, request.HandleResponse)
- batch_http_request.Execute(http)
-
- # Collect retryable requests.
- requests = [request for request in self.api_requests if not
- request.terminal_state]
-
- if hasattr(http.request, 'credentials'):
- if any(request.authorization_failed for request in requests):
- http.request.credentials.refresh(http)
-
- if not requests:
- break
-
- return self.api_requests
-
-
-class BatchHttpRequest(object):
-
- """Batches multiple http_wrapper.Request objects into a single request."""
-
- def __init__(self, batch_url, callback=None):
- """Constructor for a BatchHttpRequest.
-
- Args:
- batch_url: URL to send batch requests to.
- callback: A callback to be called for each response, of the
- form callback(response, exception). The first parameter is
- the deserialized Response object. The second is an
- apiclient.errors.HttpError exception object if an HTTP error
- occurred while processing the request, or None if no error
- occurred.
- """
- # Endpoint to which these requests are sent.
- self.__batch_url = batch_url
-
- # Global callback to be called for each individual response in the
- # batch.
- self.__callback = callback
-
- # List of requests, responses and handlers.
- self.__request_response_handlers = {}
-
- # The last auto generated id.
- self.__last_auto_id = itertools.count()
-
- # Unique ID on which to base the Content-ID headers.
- self.__base_id = uuid.uuid4()
-
- def _ConvertIdToHeader(self, request_id):
- """Convert an id to a Content-ID header value.
-
- Args:
- request_id: String identifier for a individual request.
-
- Returns:
- A Content-ID header with the id_ encoded into it. A UUID is
- prepended to the value because Content-ID headers are
- supposed to be universally unique.
-
- """
- return '<%s+%s>' % (self.__base_id, urllib_parse.quote(request_id))
-
- @staticmethod
- def _ConvertHeaderToId(header):
- """Convert a Content-ID header value to an id.
-
- Presumes the Content-ID header conforms to the format that
- _ConvertIdToHeader() returns.
-
- Args:
- header: A string indicating the Content-ID header value.
-
- Returns:
- The extracted id value.
-
- Raises:
- BatchError if the header is not in the expected format.
- """
- if not (header.startswith('<') or header.endswith('>')):
- raise exceptions.BatchError(
- 'Invalid value for Content-ID: %s' % header)
- if '+' not in header:
- raise exceptions.BatchError(
- 'Invalid value for Content-ID: %s' % header)
- _, request_id = header[1:-1].rsplit('+', 1)
-
- return urllib_parse.unquote(request_id)
-
- def _SerializeRequest(self, request):
- """Convert a http_wrapper.Request object into a string.
-
- Args:
- request: A http_wrapper.Request to serialize.
-
- Returns:
- The request as a string in application/http format.
- """
- # Construct status line
- parsed = urllib_parse.urlsplit(request.url)
- request_line = urllib_parse.urlunsplit(
- (None, None, parsed.path, parsed.query, None))
- status_line = u' '.join((
- request.http_method,
- request_line.decode('utf-8'),
- u'HTTP/1.1\n'
- ))
- major, minor = request.headers.get(
- 'content-type', 'application/json').split('/')
- msg = mime_nonmultipart.MIMENonMultipart(major, minor)
-
- # MIMENonMultipart adds its own Content-Type header.
- # Keep all of the other headers in `request.headers`.
- for key, value in request.headers.items():
- if key == 'content-type':
- continue
- msg[key] = value
-
- msg['Host'] = parsed.netloc
- msg.set_unixfrom(None)
-
- if request.body is not None:
- msg.set_payload(request.body)
-
- # Serialize the mime message.
- str_io = six.StringIO()
- # maxheaderlen=0 means don't line wrap headers.
- gen = generator.Generator(str_io, maxheaderlen=0)
- gen.flatten(msg, unixfrom=False)
- body = str_io.getvalue()
-
- # Strip off the \n\n that the MIME lib tacks onto the end of the
- # payload.
- if request.body is None:
- body = body[:-2]
-
- return status_line + body
-
- def _DeserializeResponse(self, payload):
- """Convert string into Response and content.
-
- Args:
- payload: Header and body string to be deserialized.
-
- Returns:
- A Response object
- """
- # Strip off the status line.
- status_line, payload = payload.split('\n', 1)
- _, status, _ = status_line.split(' ', 2)
-
- # Parse the rest of the response.
- parser = email_parser.Parser()
- msg = parser.parsestr(payload)
-
- # Get the headers.
- info = dict(msg)
- info['status'] = status
-
- # Create Response from the parsed headers.
- content = msg.get_payload()
-
- return http_wrapper.Response(info, content, self.__batch_url)
-
- def _NewId(self):
- """Create a new id.
-
- Auto incrementing number that avoids conflicts with ids already used.
-
- Returns:
- A new unique id string.
- """
- return str(next(self.__last_auto_id))
-
- def Add(self, request, callback=None):
- """Add a new request.
-
- Args:
- request: A http_wrapper.Request to add to the batch.
- callback: A callback to be called for this response, of the
- form callback(response, exception). The first parameter is the
- deserialized response object. The second is an
- apiclient.errors.HttpError exception object if an HTTP error
- occurred while processing the request, or None if no errors
- occurred.
-
- Returns:
- None
- """
- handler = RequestResponseAndHandler(request, None, callback)
- self.__request_response_handlers[self._NewId()] = handler
-
- def _Execute(self, http):
- """Serialize batch request, send to server, process response.
-
- Args:
- http: A httplib2.Http object to be used to make the request with.
-
- Raises:
- httplib2.HttpLib2Error if a transport error has occured.
- apiclient.errors.BatchError if the response is the wrong format.
- """
- message = mime_multipart.MIMEMultipart('mixed')
- # Message should not write out its own headers.
- setattr(message, '_write_headers', lambda self: None)
-
- # Add all the individual requests.
- for key in self.__request_response_handlers:
- msg = mime_nonmultipart.MIMENonMultipart('application', 'http')
- msg['Content-Transfer-Encoding'] = 'binary'
- msg['Content-ID'] = self._ConvertIdToHeader(key)
-
- body = self._SerializeRequest(
- self.__request_response_handlers[key].request)
- msg.set_payload(body)
- message.attach(msg)
-
- request = http_wrapper.Request(self.__batch_url, 'POST')
- request.body = message.as_string()
- request.headers['content-type'] = (
- 'multipart/mixed; boundary="%s"') % message.get_boundary()
-
- response = http_wrapper.MakeRequest(http, request)
-
- if response.status_code >= 300:
- raise exceptions.HttpError.FromResponse(response)
-
- # Prepend with a content-type header so Parser can handle it.
- header = 'content-type: %s\r\n\r\n' % response.info['content-type']
-
- parser = email_parser.Parser()
- mime_response = parser.parsestr(header + response.content)
-
- if not mime_response.is_multipart():
- raise exceptions.BatchError(
- 'Response not in multipart/mixed format.')
-
- for part in mime_response.get_payload():
- request_id = self._ConvertHeaderToId(part['Content-ID'])
- response = self._DeserializeResponse(part.get_payload())
-
- # Disable protected access because namedtuple._replace(...)
- # is not actually meant to be protected.
- self.__request_response_handlers[request_id] = (
- self.__request_response_handlers[request_id]._replace(
- response=response))
-
- def Execute(self, http):
- """Execute all the requests as a single batched HTTP request.
-
- Args:
- http: A httplib2.Http object to be used with the request.
-
- Returns:
- None
-
- Raises:
- BatchError if the response is the wrong format.
- """
-
- self._Execute(http)
-
- for key in self.__request_response_handlers:
- response = self.__request_response_handlers[key].response
- callback = self.__request_response_handlers[key].handler
-
- exception = None
-
- if response.status_code >= 300:
- exception = exceptions.HttpError.FromResponse(response)
-
- if callback is not None:
- callback(response, exception)
- if self.__callback is not None:
- self.__callback(response, exception)

Powered by Google App Engine
This is Rietveld 408576698