[catapult] - Copy Telemetry's gsutilz over to third_party.

third_party/gsutil/third_party/protorpc/protorpc/protourlencode.py

Index: third_party/gsutil/third_party/protorpc/protorpc/protourlencode.py
diff --git a/third_party/gsutil/third_party/protorpc/protorpc/protourlencode.py b/third_party/gsutil/third_party/protorpc/protorpc/protourlencode.py
new file mode 100755
index 0000000000000000000000000000000000000000..9f6059e090178f6770a9c84f2a2b7af0a1171585
--- /dev/null
+++ b/third_party/gsutil/third_party/protorpc/protorpc/protourlencode.py
@@ -0,0 +1,563 @@
+#!/usr/bin/env python
+#
+# Copyright 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""URL encoding support for messages types.
+
+Protocol support for URL encoded form parameters.
+
+Nested Fields:
+  Nested fields are repesented by dot separated names.  For example, consider
+  the following messages:
+
+    class WebPage(Message):
+
+      title = StringField(1)
+      tags = StringField(2, repeated=True)
+
+    class WebSite(Message):
+
+      name = StringField(1)
+      home = MessageField(WebPage, 2)
+      pages = MessageField(WebPage, 3, repeated=True)
+
+  And consider the object:
+
+    page = WebPage()
+    page.title = 'Welcome to NewSite 2010'
+
+    site = WebSite()
+    site.name = 'NewSite 2010'
+    site.home = page
+
+  The URL encoded representation of this constellation of objects is.
+
+    name=NewSite+2010&home.title=Welcome+to+NewSite+2010
+
+  An object that exists but does not have any state can be represented with
+  a reference to its name alone with no value assigned to it.  For example:
+
+    page = WebSite()
+    page.name = 'My Empty Site'
+    page.home = WebPage()
+
+  is represented as:
+
+    name=My+Empty+Site&home=
+
+  This represents a site with an empty uninitialized home page.
+
+Repeated Fields:
+  Repeated fields are represented by the name of and the index of each value
+  separated by a dash.  For example, consider the following message:
+
+    home = Page()
+    home.title = 'Nome'
+
+    news = Page()
+    news.title = 'News'
+    news.tags = ['news', 'articles']
+
+    instance = WebSite()
+    instance.name = 'Super fun site'
+    instance.pages = [home, news, preferences]
+
+  An instance of this message can be represented as:
+
+    name=Super+fun+site&page-0.title=Home&pages-1.title=News&...
+    pages-1.tags-0=new&pages-1.tags-1=articles
+
+Helper classes:
+
+  URLEncodedRequestBuilder: Used for encapsulating the logic used for building
+    a request message from a URL encoded RPC.
+"""
+import six
+
+__author__ = 'rafek@google.com (Rafe Kaplan)'
+
+import cgi
+import re
+import urllib
+
+from . import message_types
+from . import messages
+from . import util
+
+__all__ = ['CONTENT_TYPE',
+           'URLEncodedRequestBuilder',
+           'encode_message',
+           'decode_message',
+           ]
+
+CONTENT_TYPE = 'application/x-www-form-urlencoded'
+
+_FIELD_NAME_REGEX = re.compile(r'^([a-zA-Z_][a-zA-Z_0-9]*)(?:-([0-9]+))?$')
+
+
+class URLEncodedRequestBuilder(object):
+  """Helper that encapsulates the logic used for building URL encoded messages.
+
+  This helper is used to map query parameters from a URL encoded RPC to a
+  message instance.
+  """
+
+  @util.positional(2)
+  def __init__(self, message, prefix=''):
+    """Constructor.
+
+    Args:
+      message: Message instance to build from parameters.
+      prefix: Prefix expected at the start of valid parameters.
+    """
+    self.__parameter_prefix = prefix
+
+    # The empty tuple indicates the root message, which has no path.
+    # __messages is a full cache that makes it very easy to look up message
+    # instances by their paths.  See make_path for details about what a path
+    # is.
+    self.__messages = {(): message}
+
+    # This is a cache that stores paths which have been checked for
+    # correctness.  Correctness means that an index is present for repeated
+    # fields on the path and absent for non-repeated fields.  The cache is
+    # also used to check that indexes are added in the right order so that
+    # dicontiguous ranges of indexes are ignored.
+    self.__checked_indexes = set([()])
+
+  def make_path(self, parameter_name):
+    """Parse a parameter name and build a full path to a message value.
+
+    The path of a method is a tuple of 2-tuples describing the names and
+    indexes within repeated fields from the root message (the message being
+    constructed by the builder) to an arbitrarily nested message within it.
+
+    Each 2-tuple node of a path (name, index) is:
+      name: The name of the field that refers to the message instance.
+      index: The index within a repeated field that refers to the message
+        instance, None if not a repeated field.
+
+    For example, consider:
+
+      class VeryInner(messages.Message):
+        ...
+
+      class Inner(messages.Message):
+
+        very_inner = messages.MessageField(VeryInner, 1, repeated=True)
+
+      class Outer(messages.Message):
+
+        inner = messages.MessageField(Inner, 1)
+
+    If this builder is building an instance of Outer, that instance is
+    referred to in the URL encoded parameters without a path.  Therefore
+    its path is ().
+
+    The child 'inner' is referred to by its path (('inner', None)).
+
+    The first child of repeated field 'very_inner' on the Inner instance
+    is referred to by (('inner', None), ('very_inner', 0)).
+
+    Examples:
+      # Correct reference to model where nation is a Message, district is
+      # repeated Message and county is any not repeated field type.
+      >>> make_path('nation.district-2.county')
+      (('nation', None), ('district', 2), ('county', None))
+
+      # Field is not part of model.
+      >>> make_path('nation.made_up_field')
+      None
+
+      # nation field is not repeated and index provided.
+      >>> make_path('nation-1')
+      None
+
+      # district field is repeated and no index provided.
+      >>> make_path('nation.district')
+      None
+
+    Args:
+      parameter_name: Name of query parameter as passed in from the request.
+        in order to make a path, this parameter_name must point to a valid
+        field within the message structure.  Nodes of the path that refer to
+        repeated fields must be indexed with a number, non repeated nodes must
+        not have an index.
+
+    Returns:
+      Parsed version of the parameter_name as a tuple of tuples:
+        attribute: Name of attribute associated with path.
+        index: Postitive integer index when it is a repeated field, else None.
+      Will return None if the parameter_name does not have the right prefix,
+      does not point to a field within the message structure, does not have
+      an index if it is a repeated field or has an index but is not a repeated
+      field.
+    """
+    if parameter_name.startswith(self.__parameter_prefix):
+      parameter_name = parameter_name[len(self.__parameter_prefix):]
+    else:
+      return None
+
+    path = []
+    name = []
+    message_type = type(self.__messages[()])  # Get root message.
+
+    for item in parameter_name.split('.'):
+      # This will catch sub_message.real_message_field.not_real_field
+      if not message_type:
+        return None
+
+      item_match = _FIELD_NAME_REGEX.match(item)
+      if not item_match:
+        return None
+      attribute = item_match.group(1)
+      index = item_match.group(2)
+      if index:
+        index = int(index)
+
+      try:
+        field = message_type.field_by_name(attribute)
+      except KeyError:
+        return None
+
+      if field.repeated != (index is not None):
+        return None
+
+      if isinstance(field, messages.MessageField):
+        message_type = field.message_type
+      else:
+        message_type = None
+
+      # Path is valid so far.  Append node and continue.
+      path.append((attribute, index))
+
+    return tuple(path)
+
+  def __check_index(self, parent_path, name, index):
+    """Check correct index use and value relative to a given path.
+
+    Check that for a given path the index is present for repeated fields
+    and that it is in range for the existing list that it will be inserted
+    in to or appended to.
+
+    Args:
+      parent_path: Path to check against name and index.
+      name: Name of field to check for existance.
+      index: Index to check.  If field is repeated, should be a number within
+        range of the length of the field, or point to the next item for
+        appending.
+    """
+    # Don't worry about non-repeated fields.
+    # It's also ok if index is 0 because that means next insert will append.
+    if not index:
+      return True
+
+    parent = self.__messages.get(parent_path, None)
+    value_list = getattr(parent, name, None)
+    # If the list does not exist then the index should be 0.  Since it is
+    # not, path is not valid.
+    if not value_list:
+      return False
+
+    # The index must either point to an element of the list or to the tail.
+    return len(value_list) >= index
+
+  def __check_indexes(self, path):
+    """Check that all indexes are valid and in the right order.
+
+    This method must iterate over the path and check that all references
+    to indexes point to an existing message or to the end of the list, meaning
+    the next value should be appended to the repeated field.
+
+    Args:
+      path: Path to check indexes for.  Tuple of 2-tuples (name, index).  See
+        make_path for more information.
+
+    Returns:
+      True if all the indexes of the path are within range, else False.
+    """
+    if path in self.__checked_indexes:
+      return True
+
+    # Start with the root message.
+    parent_path = ()
+
+    for name, index in path:
+      next_path = parent_path + ((name, index),)
+      # First look in the checked indexes cache.
+      if next_path not in self.__checked_indexes:
+        if not self.__check_index(parent_path, name, index):
+          return False
+        self.__checked_indexes.add(next_path)
+
+      parent_path = next_path
+
+    return True
+
+  def __get_or_create_path(self, path):
+    """Get a message from the messages cache or create it and add it.
+
+    This method will also create any parent messages based on the path.
+
+    When a new instance of a given message is created, it is stored in
+    __message by its path.
+
+    Args:
+      path: Path of message to get.  Path must be valid, in other words
+        __check_index(path) returns true.  Tuple of 2-tuples (name, index).
+        See make_path for more information.
+
+    Returns:
+      Message instance if the field being pointed to by the path is a
+      message, else will return None for non-message fields.
+    """
+    message = self.__messages.get(path, None)
+    if message:
+      return message
+
+    parent_path = ()
+    parent = self.__messages[()]  # Get the root object
+
+    for name, index in path:
+      field = parent.field_by_name(name)
+      next_path = parent_path + ((name, index),)
+      next_message = self.__messages.get(next_path, None)
+      if next_message is None:
+        next_message = field.message_type()
+        self.__messages[next_path] = next_message
+        if not field.repeated:
+          setattr(parent, field.name, next_message)
+        else:
+          list_value = getattr(parent, field.name, None)
+          if list_value is None:
+            setattr(parent, field.name, [next_message])
+          else:
+            list_value.append(next_message)
+
+      parent_path = next_path
+      parent = next_message
+
+    return parent
+
+  def add_parameter(self, parameter, values):
+    """Add a single parameter.
+
+    Adds a single parameter and its value to the request message.
+
+    Args:
+      parameter: Query string parameter to map to request.
+      values: List of values to assign to request message.
+
+    Returns:
+      True if parameter was valid and added to the message, else False.
+
+    Raises:
+      DecodeError if the parameter refers to a valid field, and the values
+      parameter does not have one and only one value.  Non-valid query
+      parameters may have multiple values and should not cause an error.
+    """
+    path = self.make_path(parameter)
+
+    if not path:
+      return False
+
+    # Must check that all indexes of all items in the path are correct before
+    # instantiating any of them.  For example, consider:
+    #
+    #   class Repeated(object):
+    #     ...
+    #
+    #   class Inner(object):
+    #
+    #     repeated = messages.MessageField(Repeated, 1, repeated=True)
+    #
+    #   class Outer(object):
+    #
+    #     inner = messages.MessageField(Inner, 1)
+    #
+    #   instance = Outer()
+    #   builder = URLEncodedRequestBuilder(instance)
+    #   builder.add_parameter('inner.repeated')
+    #
+    #   assert not hasattr(instance, 'inner')
+    #
+    # The check is done relative to the instance of Outer pass in to the
+    # constructor of the builder.  This instance is not referred to at all
+    # because all names are assumed to be relative to it.
+    #
+    # The 'repeated' part of the path is not correct because it is missing an
+    # index.  Because it is missing an index, it should not create an instance
+    # of Repeated.  In this case add_parameter will return False and have no
+    # side effects.
+    #
+    # A correct path that would cause a new Inner instance to be inserted at
+    # instance.inner and a new Repeated instance to be appended to the
+    # instance.inner.repeated list would be 'inner.repeated-0'.
+    if not self.__check_indexes(path):
+      return False
+
+    # Ok to build objects.
+    parent_path = path[:-1]
+    parent = self.__get_or_create_path(parent_path)
+    name, index = path[-1]
+    field = parent.field_by_name(name)
+
+    if len(values) != 1:
+      raise messages.DecodeError(
+          'Found repeated values for field %s.' % field.name)
+
+    value = values[0]
+
+    if isinstance(field, messages.IntegerField):
+      converted_value = int(value)
+    elif isinstance(field, message_types.DateTimeField):
+      try:
+        converted_value = util.decode_datetime(value)
+      except ValueError as e:
+        raise messages.DecodeError(e)
+    elif isinstance(field, messages.MessageField):
+      # Just make sure it's instantiated.  Assignment to field or
+      # appending to list is done in __get_or_create_path.
+      self.__get_or_create_path(path)
+      return True
+    elif isinstance(field, messages.StringField):
+      converted_value = value.decode('utf-8')
+    elif isinstance(field, messages.BooleanField):
+      converted_value = value.lower() == 'true' and True or False
+    else:
+      try:
+        converted_value = field.type(value)
+      except TypeError:
+        raise messages.DecodeError('Invalid enum value "%s"' % value)
+
+    if field.repeated:
+      value_list = getattr(parent, field.name, None)
+      if value_list is None:
+        setattr(parent, field.name, [converted_value])
+      else:
+        if index == len(value_list):
+          value_list.append(converted_value)
+        else:
+          # Index should never be above len(value_list) because it was
+          # verified during the index check above.
+          value_list[index] = converted_value
+    else:
+      setattr(parent, field.name, converted_value)
+
+    return True
+
+
+@util.positional(1)
+def encode_message(message, prefix=''):
+  """Encode Message instance to url-encoded string.
+
+  Args:
+    message: Message instance to encode in to url-encoded string.
+    prefix: Prefix to append to field names of contained values.
+
+  Returns:
+    String encoding of Message in URL encoded format.
+
+  Raises:
+    messages.ValidationError if message is not initialized.
+  """
+  message.check_initialized()
+
+  parameters = []
+  def build_message(parent, prefix):
+    """Recursively build parameter list for URL response.
+
+    Args:
+      parent: Message to build parameters for.
+      prefix: Prefix to append to field names of contained values.
+
+    Returns:
+      True if some value of parent was added to the parameters list,
+      else False, meaning the object contained no values.
+    """
+    has_any_values = False
+    for field in sorted(parent.all_fields(), key=lambda f: f.number):
+      next_value = parent.get_assigned_value(field.name)
+      if next_value is None:
+        continue
+
+      # Found a value.  Ultimate return value should be True.
+      has_any_values = True
+
+      # Normalize all values in to a list.
+      if not field.repeated:
+        next_value = [next_value]
+
+      for index, item in enumerate(next_value):
+        # Create a name with an index if it is a repeated field.
+        if field.repeated:
+          field_name = '%s%s-%s' % (prefix, field.name, index)
+        else:
+          field_name = prefix + field.name
+
+        if isinstance(field, message_types.DateTimeField):
+          # DateTimeField stores its data as a RFC 3339 compliant string.
+          parameters.append((field_name, item.isoformat()))
+        elif isinstance(field, messages.MessageField):
+          # Message fields must be recursed in to in order to construct
+          # their component parameter values.
+          if not build_message(item, field_name + '.'):
+            # The nested message is empty.  Append an empty value to
+            # represent it.
+            parameters.append((field_name, ''))
+        elif isinstance(field, messages.BooleanField):
+          parameters.append((field_name, item and 'true' or 'false'))
+        else:
+          if isinstance(item, six.text_type):
+            item = item.encode('utf-8')
+          parameters.append((field_name, str(item)))
+
+    return has_any_values
+
+  build_message(message, prefix)
+
+  # Also add any unrecognized values from the decoded string.
+  for key in message.all_unrecognized_fields():
+    values, _ = message.get_unrecognized_field_info(key)
+    if not isinstance(values, (list, tuple)):
+      values = (values,)
+    for value in values:
+      parameters.append((key, value))
+
+  return urllib.urlencode(parameters)
+
+
+def decode_message(message_type, encoded_message, **kwargs):
+  """Decode urlencoded content to message.
+
+  Args:
+    message_type: Message instance to merge URL encoded content into.
+    encoded_message: URL encoded message.
+    prefix: Prefix to append to field names of contained values.
+
+  Returns:
+    Decoded instance of message_type.
+  """
+  message = message_type()
+  builder = URLEncodedRequestBuilder(message, **kwargs)
+  arguments = cgi.parse_qs(encoded_message, keep_blank_values=True)
+  for argument, values in sorted(six.iteritems(arguments)):
+    added = builder.add_parameter(argument, values)
+    # Save off any unknown values, so they're still accessible.
+    if not added:
+      message.set_unrecognized_field(argument, values, messages.Variant.STRING)
+  message.check_initialized()
+  return message