Revert https://codereview.chromium.org/1291903002 (protobuf roll).

third_party/protobuf/python/google/protobuf/text_format.py

Index: third_party/protobuf/python/google/protobuf/text_format.py
diff --git a/third_party/protobuf/python/google/protobuf/text_format.py b/third_party/protobuf/python/google/protobuf/text_format.py
index 8cbd68222e2edb34f7f984a8666cf6df75d2eb7f..0714c39d44d0aeb9477446b8b62d65b14b14107f 100755
--- a/third_party/protobuf/python/google/protobuf/text_format.py
+++ b/third_party/protobuf/python/google/protobuf/text_format.py
@@ -1,6 +1,6 @@
 # Protocol Buffers - Google's data interchange format
 # Copyright 2008 Google Inc.  All rights reserved.
-# https://developers.google.com/protocol-buffers/
+# http://code.google.com/p/protobuf/
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -28,10 +28,6 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#PY25 compatible for GAE.
-#
-# Copyright 2007 Google Inc. All Rights Reserved.
-
 """Contains routines for printing protocol messages in text format."""
 
 __author__ = 'kenton@google.com (Kenton Varda)'
@@ -39,12 +35,12 @@ __author__ = 'kenton@google.com (Kenton Varda)'
 import cStringIO
 import re
 
+from collections import deque
 from google.protobuf.internal import type_checkers
 from google.protobuf import descriptor
-from google.protobuf import text_encoding
 
-__all__ = ['MessageToString', 'PrintMessage', 'PrintField',
-           'PrintFieldValue', 'Merge']
+__all__ = [ 'MessageToString', 'PrintMessage', 'PrintField',
+            'PrintFieldValue', 'Merge' ]
 
 
 _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
@@ -53,96 +49,36 @@ _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
                      type_checkers.Int64ValueChecker())
 _FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE)
 _FLOAT_NAN = re.compile('nanf?', re.IGNORECASE)
-_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
-                          descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
-
 
-class Error(Exception):
-  """Top-level module error for text_format."""
 
-
-class ParseError(Error):
+class ParseError(Exception):
   """Thrown in case of ASCII parsing error."""
 
 
-def MessageToString(message, as_utf8=False, as_one_line=False,
-                    pointy_brackets=False, use_index_order=False,
-                    float_format=None):
-  """Convert protobuf message to text format.
-
-  Floating point values can be formatted compactly with 15 digits of
-  precision (which is the most that IEEE 754 "double" can guarantee)
-  using float_format='.15g'.
-
-  Args:
-    message: The protocol buffers message.
-    as_utf8: Produce text output in UTF8 format.
-    as_one_line: Don't introduce newlines between fields.
-    pointy_brackets: If True, use angle brackets instead of curly braces for
-      nesting.
-    use_index_order: If True, print fields of a proto message using the order
-      defined in source code instead of the field number. By default, use the
-      field number order.
-    float_format: If set, use this to specify floating point number formatting
-      (per the "Format Specification Mini-Language"); otherwise, str() is used.
-
-  Returns:
-    A string of the text formatted protocol buffer message.
-  """
+def MessageToString(message, as_utf8=False, as_one_line=False):
   out = cStringIO.StringIO()
-  PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line,
-               pointy_brackets=pointy_brackets,
-               use_index_order=use_index_order,
-               float_format=float_format)
+  PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line)
   result = out.getvalue()
   out.close()
   if as_one_line:
     return result.rstrip()
   return result
 
-def _IsMapEntry(field):
-  return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and
-          field.message_type.has_options and
-          field.message_type.GetOptions().map_entry)
-
-def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False,
-                 pointy_brackets=False, use_index_order=False,
-                 float_format=None):
-  fields = message.ListFields()
-  if use_index_order:
-    fields.sort(key=lambda x: x[0].index)
-  for field, value in fields:
-    if _IsMapEntry(field):
-      for key in value:
-        # This is slow for maps with submessage entires because it copies the
-        # entire tree.  Unfortunately this would take significant refactoring
-        # of this file to work around.
-        #
-        # TODO(haberman): refactor and optimize if this becomes an issue.
-        entry_submsg = field.message_type._concrete_class(
-            key=key, value=value[key])
-        PrintField(field, entry_submsg, out, indent, as_utf8, as_one_line,
-                   pointy_brackets=pointy_brackets,
-                   use_index_order=use_index_order, float_format=float_format)
-    elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+
+def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False):
+  for field, value in message.ListFields():
+    if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
       for element in value:
-        PrintField(field, element, out, indent, as_utf8, as_one_line,
-                   pointy_brackets=pointy_brackets,
-                   use_index_order=use_index_order,
-                   float_format=float_format)
+        PrintField(field, element, out, indent, as_utf8, as_one_line)
     else:
-      PrintField(field, value, out, indent, as_utf8, as_one_line,
-                 pointy_brackets=pointy_brackets,
-                 use_index_order=use_index_order,
-                 float_format=float_format)
+      PrintField(field, value, out, indent, as_utf8, as_one_line)
 
 
-def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False,
-               pointy_brackets=False, use_index_order=False, float_format=None):
+def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False):
   """Print a single field name/value pair.  For repeated fields, the value
   should be a single element."""
 
-  out.write(' ' * indent)
+  out.write(' ' * indent);
   if field.is_extension:
     out.write('[')
     if (field.containing_type.GetOptions().message_set_wire_format and
@@ -164,45 +100,27 @@ def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False,
     # don't include it.
     out.write(': ')
 
-  PrintFieldValue(field, value, out, indent, as_utf8, as_one_line,
-                  pointy_brackets=pointy_brackets,
-                  use_index_order=use_index_order,
-                  float_format=float_format)
+  PrintFieldValue(field, value, out, indent, as_utf8, as_one_line)
   if as_one_line:
     out.write(' ')
   else:
     out.write('\n')
 
 
-def PrintFieldValue(field, value, out, indent=0, as_utf8=False,
-                    as_one_line=False, pointy_brackets=False,
-                    use_index_order=False,
-                    float_format=None):
+def PrintFieldValue(field, value, out, indent=0,
+                    as_utf8=False, as_one_line=False):
   """Print a single field value (not including name).  For repeated fields,
   the value should be a single element."""
 
-  if pointy_brackets:
-    openb = '<'
-    closeb = '>'
-  else:
-    openb = '{'
-    closeb = '}'
-
   if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
     if as_one_line:
-      out.write(' %s ' % openb)
-      PrintMessage(value, out, indent, as_utf8, as_one_line,
-                   pointy_brackets=pointy_brackets,
-                   use_index_order=use_index_order,
-                   float_format=float_format)
-      out.write(closeb)
+      out.write(' { ')
+      PrintMessage(value, out, indent, as_utf8, as_one_line)
+      out.write('}')
     else:
-      out.write(' %s\n' % openb)
-      PrintMessage(value, out, indent + 2, as_utf8, as_one_line,
-                   pointy_brackets=pointy_brackets,
-                   use_index_order=use_index_order,
-                   float_format=float_format)
-      out.write(' ' * indent + closeb)
+      out.write(' {\n')
+      PrintMessage(value, out, indent + 2, as_utf8, as_one_line)
+      out.write(' ' * indent + '}')
   elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
     enum_value = field.enum_type.values_by_number.get(value, None)
     if enum_value is not None:
@@ -211,135 +129,46 @@ def PrintFieldValue(field, value, out, indent=0, as_utf8=False,
       out.write(str(value))
   elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
     out.write('\"')
-    if isinstance(value, unicode):
-      out_value = value.encode('utf-8')
-    else:
-      out_value = value
-    if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
-      # We need to escape non-UTF8 chars in TYPE_BYTES field.
-      out_as_utf8 = False
+    if type(value) is unicode:
+      out.write(_CEscape(value.encode('utf-8'), as_utf8))
     else:
-      out_as_utf8 = as_utf8
-    out.write(text_encoding.CEscape(out_value, out_as_utf8))
+      out.write(_CEscape(value, as_utf8))
     out.write('\"')
   elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
     if value:
-      out.write('true')
+      out.write("true")
     else:
-      out.write('false')
-  elif field.cpp_type in _FLOAT_TYPES and float_format is not None:
-    out.write('{1:{0}}'.format(float_format, value))
+      out.write("false")
   else:
     out.write(str(value))
 
 
-def Parse(text, message):
-  """Parses an ASCII representation of a protocol message into a message.
-
-  Args:
-    text: Message ASCII representation.
-    message: A protocol buffer message to merge into.
-
-  Returns:
-    The same message passed as argument.
-
-  Raises:
-    ParseError: On ASCII parsing problems.
-  """
-  if not isinstance(text, str): text = text.decode('utf-8')
-  return ParseLines(text.split('\n'), message)
-
-
 def Merge(text, message):
-  """Parses an ASCII representation of a protocol message into a message.
-
-  Like Parse(), but allows repeated values for a non-repeated field, and uses
-  the last one.
+  """Merges an ASCII representation of a protocol message into a message.
 
   Args:
     text: Message ASCII representation.
     message: A protocol buffer message to merge into.
 
-  Returns:
-    The same message passed as argument.
-
-  Raises:
-    ParseError: On ASCII parsing problems.
-  """
-  return MergeLines(text.split('\n'), message)
-
-
-def ParseLines(lines, message):
-  """Parses an ASCII representation of a protocol message into a message.
-
-  Args:
-    lines: An iterable of lines of a message's ASCII representation.
-    message: A protocol buffer message to merge into.
-
-  Returns:
-    The same message passed as argument.
-
-  Raises:
-    ParseError: On ASCII parsing problems.
-  """
-  _ParseOrMerge(lines, message, False)
-  return message
-
-
-def MergeLines(lines, message):
-  """Parses an ASCII representation of a protocol message into a message.
-
-  Args:
-    lines: An iterable of lines of a message's ASCII representation.
-    message: A protocol buffer message to merge into.
-
-  Returns:
-    The same message passed as argument.
-
-  Raises:
-    ParseError: On ASCII parsing problems.
-  """
-  _ParseOrMerge(lines, message, True)
-  return message
-
-
-def _ParseOrMerge(lines, message, allow_multiple_scalars):
-  """Converts an ASCII representation of a protocol message into a message.
-
-  Args:
-    lines: Lines of a message's ASCII representation.
-    message: A protocol buffer message to merge into.
-    allow_multiple_scalars: Determines if repeated values for a non-repeated
-      field are permitted, e.g., the string "foo: 1 foo: 2" for a
-      required/optional field named "foo".
-
   Raises:
     ParseError: On ASCII parsing problems.
   """
-  tokenizer = _Tokenizer(lines)
+  tokenizer = _Tokenizer(text)
   while not tokenizer.AtEnd():
-    _MergeField(tokenizer, message, allow_multiple_scalars)
+    _MergeField(tokenizer, message)
 
 
-def _MergeField(tokenizer, message, allow_multiple_scalars):
+def _MergeField(tokenizer, message):
   """Merges a single protocol message field into a message.
 
   Args:
     tokenizer: A tokenizer to parse the field name and values.
     message: A protocol message to record the data.
-    allow_multiple_scalars: Determines if repeated values for a non-repeated
-      field are permitted, e.g., the string "foo: 1 foo: 2" for a
-      required/optional field named "foo".
 
   Raises:
     ParseError: In case of ASCII parsing problems.
   """
   message_descriptor = message.DESCRIPTOR
-  if (hasattr(message_descriptor, 'syntax') and
-      message_descriptor.syntax == 'proto3'):
-    # Proto3 doesn't represent presence so we can't test if multiple
-    # scalars have occurred.  We have to allow them.
-    allow_multiple_scalars = True
   if tokenizer.TryConsume('['):
     name = [tokenizer.ConsumeIdentifier()]
     while tokenizer.TryConsume('.'):
@@ -350,9 +179,7 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
       raise tokenizer.ParseErrorPreviousToken(
           'Message type "%s" does not have extensions.' %
           message_descriptor.full_name)
-    # pylint: disable=protected-access
     field = message.Extensions._FindExtensionByName(name)
-    # pylint: enable=protected-access
     if not field:
       raise tokenizer.ParseErrorPreviousToken(
           'Extension "%s" not registered.' % name)
@@ -383,7 +210,6 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
               message_descriptor.full_name, name))
 
   if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
-    is_map_entry = _IsMapEntry(field)
     tokenizer.TryConsume(':')
 
     if tokenizer.TryConsume('<'):
@@ -395,8 +221,6 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
     if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
       if field.is_extension:
         sub_message = message.Extensions[field].add()
-      elif is_map_entry:
-        sub_message = field.message_type._concrete_class()
       else:
         sub_message = getattr(message, field.name).add()
     else:
@@ -409,34 +233,18 @@ def _MergeField(tokenizer, message, allow_multiple_scalars):
     while not tokenizer.TryConsume(end_token):
       if tokenizer.AtEnd():
         raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token))
-      _MergeField(tokenizer, sub_message, allow_multiple_scalars)
-
-    if is_map_entry:
-      value_cpptype = field.message_type.fields_by_name['value'].cpp_type
-      if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
-        value = getattr(message, field.name)[sub_message.key]
-        value.MergeFrom(sub_message.value)
-      else:
-        getattr(message, field.name)[sub_message.key] = sub_message.value
+      _MergeField(tokenizer, sub_message)
   else:
-    _MergeScalarField(tokenizer, message, field, allow_multiple_scalars)
+    _MergeScalarField(tokenizer, message, field)
 
-  # For historical reasons, fields may optionally be separated by commas or
-  # semicolons.
-  if not tokenizer.TryConsume(','):
-    tokenizer.TryConsume(';')
 
-
-def _MergeScalarField(tokenizer, message, field, allow_multiple_scalars):
+def _MergeScalarField(tokenizer, message, field):
   """Merges a single protocol message scalar field into a message.
 
   Args:
     tokenizer: A tokenizer to parse the field value.
     message: A protocol message to record the data.
     field: The descriptor of the field to be merged.
-    allow_multiple_scalars: Determines if repeated values for a non-repeated
-      field are permitted, e.g., the string "foo: 1 foo: 2" for a
-      required/optional field named "foo".
 
   Raises:
     ParseError: In case of ASCII parsing problems.
@@ -480,19 +288,9 @@ def _MergeScalarField(tokenizer, message, field, allow_multiple_scalars):
       getattr(message, field.name).append(value)
   else:
     if field.is_extension:
-      if not allow_multiple_scalars and message.HasExtension(field):
-        raise tokenizer.ParseErrorPreviousToken(
-            'Message type "%s" should not have multiple "%s" extensions.' %
-            (message.DESCRIPTOR.full_name, field.full_name))
-      else:
-        message.Extensions[field] = value
+      message.Extensions[field] = value
     else:
-      if not allow_multiple_scalars and message.HasField(field.name):
-        raise tokenizer.ParseErrorPreviousToken(
-            'Message type "%s" should not have multiple "%s" fields.' %
-            (message.DESCRIPTOR.full_name, field.name))
-      else:
-        setattr(message, field.name, value)
+      setattr(message, field.name, value)
 
 
 class _Tokenizer(object):
@@ -510,19 +308,20 @@ class _Tokenizer(object):
       '[0-9+-][0-9a-zA-Z_.+-]*|'            # a number
       '\"([^\"\n\\\\]|\\\\.)*(\"|\\\\?$)|'  # a double-quoted string
       '\'([^\'\n\\\\]|\\\\.)*(\'|\\\\?$)')  # a single-quoted string
-  _IDENTIFIER = re.compile(r'\w+')
+  _IDENTIFIER = re.compile('\w+')
+
+  def __init__(self, text_message):
+    self._text_message = text_message
 
-  def __init__(self, lines):
     self._position = 0
     self._line = -1
     self._column = 0
     self._token_start = None
     self.token = ''
-    self._lines = iter(lines)
+    self._lines = deque(text_message.split('\n'))
     self._current_line = ''
     self._previous_line = 0
     self._previous_column = 0
-    self._more_lines = True
     self._SkipWhitespace()
     self.NextToken()
 
@@ -532,19 +331,16 @@ class _Tokenizer(object):
     Returns:
       True iff the end was reached.
     """
-    return not self.token
+    return self.token == ''
 
   def _PopLine(self):
     while len(self._current_line) <= self._column:
-      try:
-        self._current_line = self._lines.next()
-      except StopIteration:
+      if not self._lines:
         self._current_line = ''
-        self._more_lines = False
         return
-      else:
-        self._line += 1
-        self._column = 0
+      self._line += 1
+      self._column = 0
+      self._current_line = self._lines.popleft()
 
   def _SkipWhitespace(self):
     while True:
@@ -701,9 +497,9 @@ class _Tokenizer(object):
     Raises:
       ParseError: If a string value couldn't be consumed.
     """
-    the_bytes = self.ConsumeByteString()
+    bytes = self.ConsumeByteString()
     try:
-      return unicode(the_bytes, 'utf-8')
+      return unicode(bytes, 'utf-8')
     except UnicodeDecodeError, e:
       raise self._StringParseError(e)
 
@@ -716,11 +512,10 @@ class _Tokenizer(object):
     Raises:
       ParseError: If a byte array value couldn't be consumed.
     """
-    the_list = [self._ConsumeSingleByteString()]
-    while self.token and self.token[0] in ('\'', '"'):
-      the_list.append(self._ConsumeSingleByteString())
-    return ''.encode('latin1').join(the_list)  ##PY25
-##!PY25    return b''.join(the_list)
+    list = [self._ConsumeSingleByteString()]
+    while len(self.token) > 0 and self.token[0] in ('\'', '"'):
+      list.append(self._ConsumeSingleByteString())
+    return "".join(list)
 
   def _ConsumeSingleByteString(self):
     """Consume one token of a string literal.
@@ -728,19 +523,16 @@ class _Tokenizer(object):
     String literals (whether bytes or text) can come in multiple adjacent
     tokens which are automatically concatenated, like in C or Python.  This
     method only consumes one token.
-
-    Raises:
-      ParseError: When the wrong format data is found.
     """
     text = self.token
     if len(text) < 1 or text[0] not in ('\'', '"'):
-      raise self._ParseError('Expected string but found: %r' % (text,))
+      raise self._ParseError('Expected string.')
 
     if len(text) < 2 or text[-1] != text[0]:
-      raise self._ParseError('String missing ending quote: %r' % (text,))
+      raise self._ParseError('String missing ending quote.')
 
     try:
-      result = text_encoding.CUnescape(text[1:-1])
+      result = _CUnescape(text[1:-1])
     except ValueError, e:
       raise self._ParseError(str(e))
     self.NextToken()
@@ -782,7 +574,7 @@ class _Tokenizer(object):
     self._column += len(self.token)
     self._SkipWhitespace()
 
-    if not self._more_lines:
+    if not self._lines and len(self._current_line) <= self._column:
       self.token = ''
       return
 
@@ -794,6 +586,40 @@ class _Tokenizer(object):
       self.token = self._current_line[self._column]
 
 
+# text.encode('string_escape') does not seem to satisfy our needs as it
+# encodes unprintable characters using two-digit hex escapes whereas our
+# C++ unescaping function allows hex escapes to be any length.  So,
+# "\0011".encode('string_escape') ends up being "\\x011", which will be
+# decoded in C++ as a single-character string with char code 0x11.
+def _CEscape(text, as_utf8):
+  def escape(c):
+    o = ord(c)
+    if o == 10: return r"\n"   # optional escape
+    if o == 13: return r"\r"   # optional escape
+    if o ==  9: return r"\t"   # optional escape
+    if o == 39: return r"\'"   # optional escape
+
+    if o == 34: return r'\"'   # necessary escape
+    if o == 92: return r"\\"   # necessary escape
+
+    # necessary escapes
+    if not as_utf8 and (o >= 127 or o < 32): return "\\%03o" % o
+    return c
+  return "".join([escape(c) for c in text])
+
+
+_CUNESCAPE_HEX = re.compile('\\\\x([0-9a-fA-F]{2}|[0-9a-fA-F])')
+
+
+def _CUnescape(text):
+  def ReplaceHex(m):
+    return chr(int(m.group(0)[2:], 16))
+  # This is required because the 'string_escape' encoding doesn't
+  # allow single-digit hex escapes (like '\xf').
+  result = _CUNESCAPE_HEX.sub(ReplaceHex, text)
+  return result.decode('string_escape')
+
+
 def ParseInteger(text, is_signed=False, is_long=False):
   """Parses an integer.
 
@@ -810,13 +636,7 @@ def ParseInteger(text, is_signed=False, is_long=False):
   """
   # Do the actual parsing. Exception handling is propagated to caller.
   try:
-    # We force 32-bit values to int and 64-bit values to long to make
-    # alternate implementations where the distinction is more significant
-    # (e.g. the C++ implementation) simpler.
-    if is_long:
-      result = long(text, 0)
-    else:
-      result = int(text, 0)
+    result = int(text, 0)
   except ValueError:
     raise ValueError('Couldn\'t parse integer: %s' % text)