third_party/protobuf: Update to HEAD (83d681ee2c)

third_party/protobuf/python/google/protobuf/text_format.py

Index: third_party/protobuf/python/google/protobuf/text_format.py
diff --git a/third_party/protobuf/python/google/protobuf/text_format.py b/third_party/protobuf/python/google/protobuf/text_format.py
index 6f1e3c8b725ea2f90b27ad62714de4d4df24a94c..90f6ce42a483e938825ee24a05bcd74c91463b5a 100755
--- a/third_party/protobuf/python/google/protobuf/text_format.py
+++ b/third_party/protobuf/python/google/protobuf/text_format.py
@@ -48,15 +48,15 @@ import re
 import six
 
 if six.PY3:
-  long = int
+  long = int  # pylint: disable=redefined-builtin,invalid-name
 
+# pylint: disable=g-import-not-at-top
 from google.protobuf.internal import type_checkers
 from google.protobuf import descriptor
 from google.protobuf import text_encoding
 
-__all__ = ['MessageToString', 'PrintMessage', 'PrintField',
-           'PrintFieldValue', 'Merge']
-
+__all__ = ['MessageToString', 'PrintMessage', 'PrintField', 'PrintFieldValue',
+           'Merge']
 
 _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
                      type_checkers.Int32ValueChecker(),
@@ -67,6 +67,7 @@ _FLOAT_NAN = re.compile('nanf?', re.IGNORECASE)
 _FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
                           descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
 _QUOTES = frozenset(("'", '"'))
+_ANY_FULL_TYPE_NAME = 'google.protobuf.Any'
 
 
 class Error(Exception):
@@ -74,10 +75,30 @@ class Error(Exception):
 
 
 class ParseError(Error):
-  """Thrown in case of text parsing error."""
+  """Thrown in case of text parsing or tokenizing error."""
+
+  def __init__(self, message=None, line=None, column=None):
+    if message is not None and line is not None:
+      loc = str(line)
+      if column is not None:
+        loc += ':{0}'.format(column)
+      message = '{0} : {1}'.format(loc, message)
+    if message is not None:
+      super(ParseError, self).__init__(message)
+    else:
+      super(ParseError, self).__init__()
+    self._line = line
+    self._column = column
+
+  def GetLine(self):
+    return self._line
+
+  def GetColumn(self):
+    return self._column
 
 
 class TextWriter(object):
+
   def __init__(self, as_utf8):
     if six.PY2:
       self._writer = io.BytesIO()
@@ -97,9 +118,15 @@ class TextWriter(object):
     return self._writer.getvalue()
 
 
-def MessageToString(message, as_utf8=False, as_one_line=False,
-                    pointy_brackets=False, use_index_order=False,
-                    float_format=None, use_field_number=False):
+def MessageToString(message,
+                    as_utf8=False,
+                    as_one_line=False,
+                    pointy_brackets=False,
+                    use_index_order=False,
+                    float_format=None,
+                    use_field_number=False,
+                    descriptor_pool=None,
+                    indent=0):
   """Convert protobuf message to text format.
 
   Floating point values can be formatted compactly with 15 digits of
@@ -119,14 +146,16 @@ def MessageToString(message, as_utf8=False, as_one_line=False,
     float_format: If set, use this to specify floating point number formatting
       (per the "Format Specification Mini-Language"); otherwise, str() is used.
     use_field_number: If True, print field numbers instead of names.
+    descriptor_pool: A DescriptorPool used to resolve Any types.
+    indent: The indent level, in terms of spaces, for pretty print.
 
   Returns:
     A string of the text formatted protocol buffer message.
   """
   out = TextWriter(as_utf8)
-  printer = _Printer(out, 0, as_utf8, as_one_line,
-                     pointy_brackets, use_index_order, float_format,
-                     use_field_number)
+  printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets,
+                     use_index_order, float_format, use_field_number,
+                     descriptor_pool)
   printer.PrintMessage(message)
   result = out.getvalue()
   out.close()
@@ -141,39 +170,87 @@ def _IsMapEntry(field):
           field.message_type.GetOptions().map_entry)
 
 
-def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False,
-                 pointy_brackets=False, use_index_order=False,
-                 float_format=None, use_field_number=False):
-  printer = _Printer(out, indent, as_utf8, as_one_line,
-                     pointy_brackets, use_index_order, float_format,
-                     use_field_number)
+def PrintMessage(message,
+                 out,
+                 indent=0,
+                 as_utf8=False,
+                 as_one_line=False,
+                 pointy_brackets=False,
+                 use_index_order=False,
+                 float_format=None,
+                 use_field_number=False,
+                 descriptor_pool=None):
+  printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets,
+                     use_index_order, float_format, use_field_number,
+                     descriptor_pool)
   printer.PrintMessage(message)
 
 
-def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False,
-               pointy_brackets=False, use_index_order=False, float_format=None):
+def PrintField(field,
+               value,
+               out,
+               indent=0,
+               as_utf8=False,
+               as_one_line=False,
+               pointy_brackets=False,
+               use_index_order=False,
+               float_format=None):
   """Print a single field name/value pair."""
-  printer = _Printer(out, indent, as_utf8, as_one_line,
-                     pointy_brackets, use_index_order, float_format)
+  printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets,
+                     use_index_order, float_format)
   printer.PrintField(field, value)
 
 
-def PrintFieldValue(field, value, out, indent=0, as_utf8=False,
-                    as_one_line=False, pointy_brackets=False,
+def PrintFieldValue(field,
+                    value,
+                    out,
+                    indent=0,
+                    as_utf8=False,
+                    as_one_line=False,
+                    pointy_brackets=False,
                     use_index_order=False,
                     float_format=None):
   """Print a single field value (not including name)."""
-  printer = _Printer(out, indent, as_utf8, as_one_line,
-                     pointy_brackets, use_index_order, float_format)
+  printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets,
+                     use_index_order, float_format)
   printer.PrintFieldValue(field, value)
 
 
+def _BuildMessageFromTypeName(type_name, descriptor_pool):
+  """Returns a protobuf message instance.
+
+  Args:
+    type_name: Fully-qualified protobuf  message type name string.
+    descriptor_pool: DescriptorPool instance.
+
+  Returns:
+    A Message instance of type matching type_name, or None if the a Descriptor
+    wasn't found matching type_name.
+  """
+  # pylint: disable=g-import-not-at-top
+  from google.protobuf import symbol_database
+  database = symbol_database.Default()
+  try:
+    message_descriptor = descriptor_pool.FindMessageTypeByName(type_name)
+  except KeyError:
+    return None
+  message_type = database.GetPrototype(message_descriptor)
+  return message_type()
+
+
 class _Printer(object):
   """Text format printer for protocol message."""
 
-  def __init__(self, out, indent=0, as_utf8=False, as_one_line=False,
-               pointy_brackets=False, use_index_order=False, float_format=None,
-               use_field_number=False):
+  def __init__(self,
+               out,
+               indent=0,
+               as_utf8=False,
+               as_one_line=False,
+               pointy_brackets=False,
+               use_index_order=False,
+               float_format=None,
+               use_field_number=False,
+               descriptor_pool=None):
     """Initialize the Printer.
 
     Floating point values can be formatted compactly with 15 digits of
@@ -195,6 +272,7 @@ class _Printer(object):
         (per the "Format Specification Mini-Language"); otherwise, str() is
         used.
       use_field_number: If True, print field numbers instead of names.
+      descriptor_pool: A DescriptorPool used to resolve Any types.
     """
     self.out = out
     self.indent = indent
@@ -204,6 +282,20 @@ class _Printer(object):
     self.use_index_order = use_index_order
     self.float_format = float_format
     self.use_field_number = use_field_number
+    self.descriptor_pool = descriptor_pool
+
+  def _TryPrintAsAnyMessage(self, message):
+    """Serializes if message is a google.protobuf.Any field."""
+    packed_message = _BuildMessageFromTypeName(message.TypeName(),
+                                               self.descriptor_pool)
+    if packed_message:
+      packed_message.MergeFromString(message.value)
+      self.out.write('%s[%s]' % (self.indent * ' ', message.type_url))
+      self._PrintMessageFieldValue(packed_message)
+      self.out.write(' ' if self.as_one_line else '\n')
+      return True
+    else:
+      return False
 
   def PrintMessage(self, message):
     """Convert protobuf message to text format.
@@ -211,6 +303,9 @@ class _Printer(object):
     Args:
       message: The protocol buffers message.
     """
+    if (message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME and
+        self.descriptor_pool and self._TryPrintAsAnyMessage(message)):
+      return
     fields = message.ListFields()
     if self.use_index_order:
       fields.sort(key=lambda x: x[0].index)
@@ -222,8 +317,7 @@ class _Printer(object):
           # of this file to work around.
           #
           # TODO(haberman): refactor and optimize if this becomes an issue.
-          entry_submsg = field.message_type._concrete_class(
-              key=key, value=value[key])
+          entry_submsg = value.GetEntryClass()(key=key, value=value[key])
           self.PrintField(field, entry_submsg)
       elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
         for element in value:
@@ -264,6 +358,25 @@ class _Printer(object):
     else:
       out.write('\n')
 
+  def _PrintMessageFieldValue(self, value):
+    if self.pointy_brackets:
+      openb = '<'
+      closeb = '>'
+    else:
+      openb = '{'
+      closeb = '}'
+
+    if self.as_one_line:
+      self.out.write(' %s ' % openb)
+      self.PrintMessage(value)
+      self.out.write(closeb)
+    else:
+      self.out.write(' %s\n' % openb)
+      self.indent += 2
+      self.PrintMessage(value)
+      self.indent -= 2
+      self.out.write(' ' * self.indent + closeb)
+
   def PrintFieldValue(self, field, value):
     """Print a single field value (not including name).
 
@@ -274,24 +387,8 @@ class _Printer(object):
       value: The value of the field.
     """
     out = self.out
-    if self.pointy_brackets:
-      openb = '<'
-      closeb = '>'
-    else:
-      openb = '{'
-      closeb = '}'
-
     if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
-      if self.as_one_line:
-        out.write(' %s ' % openb)
-        self.PrintMessage(value)
-        out.write(closeb)
-      else:
-        out.write(' %s\n' % openb)
-        self.indent += 2
-        self.PrintMessage(value)
-        self.indent -= 2
-        out.write(' ' * self.indent + closeb)
+      self._PrintMessageFieldValue(value)
     elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
       enum_value = field.enum_type.values_by_number.get(value, None)
       if enum_value is not None:
@@ -322,9 +419,11 @@ class _Printer(object):
       out.write(str(value))
 
 
-def Parse(text, message,
-          allow_unknown_extension=False, allow_field_number=False):
-  """Parses an text representation of a protocol message into a message.
+def Parse(text,
+          message,
+          allow_unknown_extension=False,
+          allow_field_number=False):
+  """Parses a text representation of a protocol message into a message.
 
   Args:
     text: Message text representation.
@@ -341,13 +440,16 @@ def Parse(text, message,
   """
   if not isinstance(text, str):
     text = text.decode('utf-8')
-  return ParseLines(text.split('\n'), message, allow_unknown_extension,
-                    allow_field_number)
+  return ParseLines(
+      text.split('\n'), message, allow_unknown_extension, allow_field_number)
 
 
-def Merge(text, message, allow_unknown_extension=False,
-          allow_field_number=False):
-  """Parses an text representation of a protocol message into a message.
+def Merge(text,
+          message,
+          allow_unknown_extension=False,
+          allow_field_number=False,
+          descriptor_pool=None):
+  """Parses a text representation of a protocol message into a message.
 
   Like Parse(), but allows repeated values for a non-repeated field, and uses
   the last one.
@@ -358,6 +460,7 @@ def Merge(text, message, allow_unknown_extension=False,
     allow_unknown_extension: if True, skip over missing extensions and keep
       parsing
     allow_field_number: if True, both field number and field name are allowed.
+    descriptor_pool: A DescriptorPool used to resolve Any types.
 
   Returns:
     The same message passed as argument.
@@ -365,13 +468,19 @@ def Merge(text, message, allow_unknown_extension=False,
   Raises:
     ParseError: On text parsing problems.
   """
-  return MergeLines(text.split('\n'), message, allow_unknown_extension,
-                    allow_field_number)
+  return MergeLines(
+      text.split('\n'),
+      message,
+      allow_unknown_extension,
+      allow_field_number,
+      descriptor_pool=descriptor_pool)
 
 
-def ParseLines(lines, message, allow_unknown_extension=False,
+def ParseLines(lines,
+               message,
+               allow_unknown_extension=False,
                allow_field_number=False):
-  """Parses an text representation of a protocol message into a message.
+  """Parses a text representation of a protocol message into a message.
 
   Args:
     lines: An iterable of lines of a message's text representation.
@@ -379,6 +488,7 @@ def ParseLines(lines, message, allow_unknown_extension=False,
     allow_unknown_extension: if True, skip over missing extensions and keep
       parsing
     allow_field_number: if True, both field number and field name are allowed.
+    descriptor_pool: A DescriptorPool used to resolve Any types.
 
   Returns:
     The same message passed as argument.
@@ -390,9 +500,12 @@ def ParseLines(lines, message, allow_unknown_extension=False,
   return parser.ParseLines(lines, message)
 
 
-def MergeLines(lines, message, allow_unknown_extension=False,
-               allow_field_number=False):
-  """Parses an text representation of a protocol message into a message.
+def MergeLines(lines,
+               message,
+               allow_unknown_extension=False,
+               allow_field_number=False,
+               descriptor_pool=None):
+  """Parses a text representation of a protocol message into a message.
 
   Args:
     lines: An iterable of lines of a message's text representation.
@@ -407,41 +520,47 @@ def MergeLines(lines, message, allow_unknown_extension=False,
   Raises:
     ParseError: On text parsing problems.
   """
-  parser = _Parser(allow_unknown_extension, allow_field_number)
+  parser = _Parser(allow_unknown_extension,
+                   allow_field_number,
+                   descriptor_pool=descriptor_pool)
   return parser.MergeLines(lines, message)
 
 
 class _Parser(object):
   """Text format parser for protocol message."""
 
-  def __init__(self, allow_unknown_extension=False, allow_field_number=False):
+  def __init__(self,
+               allow_unknown_extension=False,
+               allow_field_number=False,
+               descriptor_pool=None):
     self.allow_unknown_extension = allow_unknown_extension
     self.allow_field_number = allow_field_number
+    self.descriptor_pool = descriptor_pool
 
   def ParseFromString(self, text, message):
-    """Parses an text representation of a protocol message into a message."""
+    """Parses a text representation of a protocol message into a message."""
     if not isinstance(text, str):
       text = text.decode('utf-8')
     return self.ParseLines(text.split('\n'), message)
 
   def ParseLines(self, lines, message):
-    """Parses an text representation of a protocol message into a message."""
+    """Parses a text representation of a protocol message into a message."""
     self._allow_multiple_scalars = False
     self._ParseOrMerge(lines, message)
     return message
 
   def MergeFromString(self, text, message):
-    """Merges an text representation of a protocol message into a message."""
+    """Merges a text representation of a protocol message into a message."""
     return self._MergeLines(text.split('\n'), message)
 
   def MergeLines(self, lines, message):
-    """Merges an text representation of a protocol message into a message."""
+    """Merges a text representation of a protocol message into a message."""
     self._allow_multiple_scalars = True
     self._ParseOrMerge(lines, message)
     return message
 
   def _ParseOrMerge(self, lines, message):
-    """Converts an text representation of a protocol message into a message.
+    """Converts a text representation of a protocol message into a message.
 
     Args:
       lines: Lines of a message's text representation.
@@ -450,7 +569,7 @@ class _Parser(object):
     Raises:
       ParseError: On text parsing problems.
     """
-    tokenizer = _Tokenizer(lines)
+    tokenizer = Tokenizer(lines)
     while not tokenizer.AtEnd():
       self._MergeField(tokenizer, message)
 
@@ -491,13 +610,13 @@ class _Parser(object):
               'Extension "%s" not registered.' % name)
       elif message_descriptor != field.containing_type:
         raise tokenizer.ParseErrorPreviousToken(
-            'Extension "%s" does not extend message type "%s".' % (
-                name, message_descriptor.full_name))
+            'Extension "%s" does not extend message type "%s".' %
+            (name, message_descriptor.full_name))
 
       tokenizer.Consume(']')
 
     else:
-      name = tokenizer.ConsumeIdentifier()
+      name = tokenizer.ConsumeIdentifierOrNumber()
       if self.allow_field_number and name.isdigit():
         number = ParseInteger(name, True, True)
         field = message_descriptor.fields_by_number.get(number, None)
@@ -520,8 +639,8 @@ class _Parser(object):
 
       if not field:
         raise tokenizer.ParseErrorPreviousToken(
-            'Message type "%s" has no field named "%s".' % (
-                message_descriptor.full_name, name))
+            'Message type "%s" has no field named "%s".' %
+            (message_descriptor.full_name, name))
 
     if field:
       if not self._allow_multiple_scalars and field.containing_oneof:
@@ -532,9 +651,9 @@ class _Parser(object):
         if which_oneof is not None and which_oneof != field.name:
           raise tokenizer.ParseErrorPreviousToken(
               'Field "%s" is specified along with field "%s", another member '
-              'of oneof "%s" for message type "%s".' % (
-                  field.name, which_oneof, field.containing_oneof.name,
-                  message_descriptor.full_name))
+              'of oneof "%s" for message type "%s".' %
+              (field.name, which_oneof, field.containing_oneof.name,
+               message_descriptor.full_name))
 
       if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
         tokenizer.TryConsume(':')
@@ -543,12 +662,13 @@ class _Parser(object):
         tokenizer.Consume(':')
         merger = self._MergeScalarField
 
-      if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED
-          and tokenizer.TryConsume('[')):
+      if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and
+          tokenizer.TryConsume('[')):
         # Short repeated format, e.g. "foo: [1, 2, 3]"
         while True:
           merger(tokenizer, message, field)
-          if tokenizer.TryConsume(']'): break
+          if tokenizer.TryConsume(']'):
+            break
           tokenizer.Consume(',')
 
       else:
@@ -563,6 +683,21 @@ class _Parser(object):
     if not tokenizer.TryConsume(','):
       tokenizer.TryConsume(';')
 
+  def _ConsumeAnyTypeUrl(self, tokenizer):
+    """Consumes a google.protobuf.Any type URL and returns the type name."""
+    # Consume "type.googleapis.com/".
+    tokenizer.ConsumeIdentifier()
+    tokenizer.Consume('.')
+    tokenizer.ConsumeIdentifier()
+    tokenizer.Consume('.')
+    tokenizer.ConsumeIdentifier()
+    tokenizer.Consume('/')
+    # Consume the fully-qualified type name.
+    name = [tokenizer.ConsumeIdentifier()]
+    while tokenizer.TryConsume('.'):
+      name.append(tokenizer.ConsumeIdentifier())
+    return '.'.join(name)
+
   def _MergeMessageField(self, tokenizer, message, field):
     """Merges a single scalar field into a message.
 
@@ -582,12 +717,38 @@ class _Parser(object):
       tokenizer.Consume('{')
       end_token = '}'
 
-    if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+    if (field.message_type.full_name == _ANY_FULL_TYPE_NAME and
+        tokenizer.TryConsume('[')):
+      packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
+      tokenizer.Consume(']')
+      tokenizer.TryConsume(':')
+      if tokenizer.TryConsume('<'):
+        expanded_any_end_token = '>'
+      else:
+        tokenizer.Consume('{')
+        expanded_any_end_token = '}'
+      if not self.descriptor_pool:
+        raise ParseError('Descriptor pool required to parse expanded Any field')
+      expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name,
+                                                           self.descriptor_pool)
+      if not expanded_any_sub_message:
+        raise ParseError('Type %s not found in descriptor pool' %
+                         packed_type_name)
+      while not tokenizer.TryConsume(expanded_any_end_token):
+        if tokenizer.AtEnd():
+          raise tokenizer.ParseErrorPreviousToken('Expected "%s".' %
+                                                  (expanded_any_end_token,))
+        self._MergeField(tokenizer, expanded_any_sub_message)
+      if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
+        any_message = getattr(message, field.name).add()
+      else:
+        any_message = getattr(message, field.name)
+      any_message.Pack(expanded_any_sub_message)
+    elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
       if field.is_extension:
         sub_message = message.Extensions[field].add()
       elif is_map_entry:
-        # pylint: disable=protected-access
-        sub_message = field.message_type._concrete_class()
+        sub_message = getattr(message, field.name).GetEntryClass()()
       else:
         sub_message = getattr(message, field.name).add()
     else:
@@ -628,17 +789,17 @@ class _Parser(object):
     if field.type in (descriptor.FieldDescriptor.TYPE_INT32,
                       descriptor.FieldDescriptor.TYPE_SINT32,
                       descriptor.FieldDescriptor.TYPE_SFIXED32):
-      value = tokenizer.ConsumeInt32()
+      value = _ConsumeInt32(tokenizer)
     elif field.type in (descriptor.FieldDescriptor.TYPE_INT64,
                         descriptor.FieldDescriptor.TYPE_SINT64,
                         descriptor.FieldDescriptor.TYPE_SFIXED64):
-      value = tokenizer.ConsumeInt64()
+      value = _ConsumeInt64(tokenizer)
     elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32,
                         descriptor.FieldDescriptor.TYPE_FIXED32):
-      value = tokenizer.ConsumeUint32()
+      value = _ConsumeUint32(tokenizer)
     elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64,
                         descriptor.FieldDescriptor.TYPE_FIXED64):
-      value = tokenizer.ConsumeUint64()
+      value = _ConsumeUint64(tokenizer)
     elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT,
                         descriptor.FieldDescriptor.TYPE_DOUBLE):
       value = tokenizer.ConsumeFloat()
@@ -753,13 +914,12 @@ def _SkipFieldValue(tokenizer):
     return
 
   if (not tokenizer.TryConsumeIdentifier() and
-      not tokenizer.TryConsumeInt64() and
-      not tokenizer.TryConsumeUint64() and
+      not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and
       not tokenizer.TryConsumeFloat()):
     raise ParseError('Invalid field value: ' + tokenizer.token)
 
 
-class _Tokenizer(object):
+class Tokenizer(object):
   """Protocol buffer text representation tokenizer.
 
   This class handles the lower level string parsing by splitting it into
@@ -768,17 +928,20 @@ class _Tokenizer(object):
   It was directly ported from the Java protocol buffer API.
   """
 
-  _WHITESPACE = re.compile('(\\s|(#.*$))+', re.MULTILINE)
+  _WHITESPACE = re.compile(r'\s+')
+  _COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE)
+  _WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE)
   _TOKEN = re.compile('|'.join([
-      r'[a-zA-Z_][0-9a-zA-Z_+-]*',             # an identifier
+      r'[a-zA-Z_][0-9a-zA-Z_+-]*',  # an identifier
       r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*',  # a number
-  ] + [                                        # quoted str for each quote mark
+  ] + [  # quoted str for each quote mark
       r'{qt}([^{qt}\n\\]|\\.)*({qt}|\\?$)'.format(qt=mark) for mark in _QUOTES
   ]))
 
-  _IDENTIFIER = re.compile(r'\w+')
+  _IDENTIFIER = re.compile(r'[^\d\W]\w*')
+  _IDENTIFIER_OR_NUMBER = re.compile(r'\w+')
 
-  def __init__(self, lines):
+  def __init__(self, lines, skip_comments=True):
     self._position = 0
     self._line = -1
     self._column = 0
@@ -789,6 +952,9 @@ class _Tokenizer(object):
     self._previous_line = 0
     self._previous_column = 0
     self._more_lines = True
+    self._skip_comments = skip_comments
+    self._whitespace_pattern = (skip_comments and self._WHITESPACE_OR_COMMENT
+                                or self._WHITESPACE)
     self._SkipWhitespace()
     self.NextToken()
 
@@ -818,7 +984,7 @@ class _Tokenizer(object):
   def _SkipWhitespace(self):
     while True:
       self._PopLine()
-      match = self._WHITESPACE.match(self._current_line, self._column)
+      match = self._whitespace_pattern.match(self._current_line, self._column)
       if not match:
         break
       length = len(match.group(0))
@@ -848,7 +1014,14 @@ class _Tokenizer(object):
       ParseError: If the text couldn't be consumed.
     """
     if not self.TryConsume(token):
-      raise self._ParseError('Expected "%s".' % token)
+      raise self.ParseError('Expected "%s".' % token)
+
+  def ConsumeComment(self):
+    result = self.token
+    if not self._COMMENT.match(result):
+      raise self.ParseError('Expected comment.')
+    self.NextToken()
+    return result
 
   def TryConsumeIdentifier(self):
     try:
@@ -868,85 +1041,55 @@ class _Tokenizer(object):
     """
     result = self.token
     if not self._IDENTIFIER.match(result):
-      raise self._ParseError('Expected identifier.')
+      raise self.ParseError('Expected identifier.')
     self.NextToken()
     return result
 
-  def ConsumeInt32(self):
-    """Consumes a signed 32bit integer number.
-
-    Returns:
-      The integer parsed.
-
-    Raises:
-      ParseError: If a signed 32bit integer couldn't be consumed.
-    """
+  def TryConsumeIdentifierOrNumber(self):
     try:
-      result = ParseInteger(self.token, is_signed=True, is_long=False)
-    except ValueError as e:
-      raise self._ParseError(str(e))
-    self.NextToken()
-    return result
-
-  def ConsumeUint32(self):
-    """Consumes an unsigned 32bit integer number.
-
-    Returns:
-      The integer parsed.
-
-    Raises:
-      ParseError: If an unsigned 32bit integer couldn't be consumed.
-    """
-    try:
-      result = ParseInteger(self.token, is_signed=False, is_long=False)
-    except ValueError as e:
-      raise self._ParseError(str(e))
-    self.NextToken()
-    return result
-
-  def TryConsumeInt64(self):
-    try:
-      self.ConsumeInt64()
+      self.ConsumeIdentifierOrNumber()
       return True
     except ParseError:
       return False
 
-  def ConsumeInt64(self):
-    """Consumes a signed 64bit integer number.
+  def ConsumeIdentifierOrNumber(self):
+    """Consumes protocol message field identifier.
 
     Returns:
-      The integer parsed.
+      Identifier string.
 
     Raises:
-      ParseError: If a signed 64bit integer couldn't be consumed.
+      ParseError: If an identifier couldn't be consumed.
     """
-    try:
-      result = ParseInteger(self.token, is_signed=True, is_long=True)
-    except ValueError as e:
-      raise self._ParseError(str(e))
+    result = self.token
+    if not self._IDENTIFIER_OR_NUMBER.match(result):
+      raise self.ParseError('Expected identifier or number.')
     self.NextToken()
     return result
 
-  def TryConsumeUint64(self):
+  def TryConsumeInteger(self):
     try:
-      self.ConsumeUint64()
+      # Note: is_long only affects value type, not whether an error is raised.
+      self.ConsumeInteger()
       return True
     except ParseError:
       return False
 
-  def ConsumeUint64(self):
-    """Consumes an unsigned 64bit integer number.
+  def ConsumeInteger(self, is_long=False):
+    """Consumes an integer number.
 
+    Args:
+      is_long: True if the value should be returned as a long integer.
     Returns:
       The integer parsed.
 
     Raises:
-      ParseError: If an unsigned 64bit integer couldn't be consumed.
+      ParseError: If an integer couldn't be consumed.
     """
     try:
-      result = ParseInteger(self.token, is_signed=False, is_long=True)
+      result = _ParseAbstractInteger(self.token, is_long=is_long)
     except ValueError as e:
-      raise self._ParseError(str(e))
+      raise self.ParseError(str(e))
     self.NextToken()
     return result
 
@@ -969,7 +1112,7 @@ class _Tokenizer(object):
     try:
       result = ParseFloat(self.token)
     except ValueError as e:
-      raise self._ParseError(str(e))
+      raise self.ParseError(str(e))
     self.NextToken()
     return result
 
@@ -985,7 +1128,7 @@ class _Tokenizer(object):
     try:
       result = ParseBool(self.token)
     except ValueError as e:
-      raise self._ParseError(str(e))
+      raise self.ParseError(str(e))
     self.NextToken()
     return result
 
@@ -1039,15 +1182,15 @@ class _Tokenizer(object):
     """
     text = self.token
     if len(text) < 1 or text[0] not in _QUOTES:
-      raise self._ParseError('Expected string but found: %r' % (text,))
+      raise self.ParseError('Expected string but found: %r' % (text,))
 
     if len(text) < 2 or text[-1] != text[0]:
-      raise self._ParseError('String missing ending quote: %r' % (text,))
+      raise self.ParseError('String missing ending quote: %r' % (text,))
 
     try:
       result = text_encoding.CUnescape(text[1:-1])
     except ValueError as e:
-      raise self._ParseError(str(e))
+      raise self.ParseError(str(e))
     self.NextToken()
     return result
 
@@ -1055,7 +1198,7 @@ class _Tokenizer(object):
     try:
       result = ParseEnum(field, self.token)
     except ValueError as e:
-      raise self._ParseError(str(e))
+      raise self.ParseError(str(e))
     self.NextToken()
     return result
 
@@ -1068,16 +1211,15 @@ class _Tokenizer(object):
     Returns:
       A ParseError instance.
     """
-    return ParseError('%d:%d : %s' % (
-        self._previous_line + 1, self._previous_column + 1, message))
+    return ParseError(message, self._previous_line + 1,
+                      self._previous_column + 1)
 
-  def _ParseError(self, message):
+  def ParseError(self, message):
     """Creates and *returns* a ParseError for the current token."""
-    return ParseError('%d:%d : %s' % (
-        self._line + 1, self._column + 1, message))
+    return ParseError(message, self._line + 1, self._column + 1)
 
   def _StringParseError(self, e):
-    return self._ParseError('Couldn\'t parse string: ' + str(e))
+    return self.ParseError('Couldn\'t parse string: ' + str(e))
 
   def NextToken(self):
     """Reads the next meaningful token."""
@@ -1092,12 +1234,124 @@ class _Tokenizer(object):
       return
 
     match = self._TOKEN.match(self._current_line, self._column)
+    if not match and not self._skip_comments:
+      match = self._COMMENT.match(self._current_line, self._column)
     if match:
       token = match.group(0)
       self.token = token
     else:
       self.token = self._current_line[self._column]
 
+# Aliased so it can still be accessed by current visibility violators.
+# TODO(dbarnett): Migrate violators to textformat_tokenizer.
+_Tokenizer = Tokenizer  # pylint: disable=invalid-name
+
+
+def _ConsumeInt32(tokenizer):
+  """Consumes a signed 32bit integer number from tokenizer.
+
+  Args:
+    tokenizer: A tokenizer used to parse the number.
+
+  Returns:
+    The integer parsed.
+
+  Raises:
+    ParseError: If a signed 32bit integer couldn't be consumed.
+  """
+  return _ConsumeInteger(tokenizer, is_signed=True, is_long=False)
+
+
+def _ConsumeUint32(tokenizer):
+  """Consumes an unsigned 32bit integer number from tokenizer.
+
+  Args:
+    tokenizer: A tokenizer used to parse the number.
+
+  Returns:
+    The integer parsed.
+
+  Raises:
+    ParseError: If an unsigned 32bit integer couldn't be consumed.
+  """
+  return _ConsumeInteger(tokenizer, is_signed=False, is_long=False)
+
+
+def _TryConsumeInt64(tokenizer):
+  try:
+    _ConsumeInt64(tokenizer)
+    return True
+  except ParseError:
+    return False
+
+
+def _ConsumeInt64(tokenizer):
+  """Consumes a signed 32bit integer number from tokenizer.
+
+  Args:
+    tokenizer: A tokenizer used to parse the number.
+
+  Returns:
+    The integer parsed.
+
+  Raises:
+    ParseError: If a signed 32bit integer couldn't be consumed.
+  """
+  return _ConsumeInteger(tokenizer, is_signed=True, is_long=True)
+
+
+def _TryConsumeUint64(tokenizer):
+  try:
+    _ConsumeUint64(tokenizer)
+    return True
+  except ParseError:
+    return False
+
+
+def _ConsumeUint64(tokenizer):
+  """Consumes an unsigned 64bit integer number from tokenizer.
+
+  Args:
+    tokenizer: A tokenizer used to parse the number.
+
+  Returns:
+    The integer parsed.
+
+  Raises:
+    ParseError: If an unsigned 64bit integer couldn't be consumed.
+  """
+  return _ConsumeInteger(tokenizer, is_signed=False, is_long=True)
+
+
+def _TryConsumeInteger(tokenizer, is_signed=False, is_long=False):
+  try:
+    _ConsumeInteger(tokenizer, is_signed=is_signed, is_long=is_long)
+    return True
+  except ParseError:
+    return False
+
+
+def _ConsumeInteger(tokenizer, is_signed=False, is_long=False):
+  """Consumes an integer number from tokenizer.
+
+  Args:
+    tokenizer: A tokenizer used to parse the number.
+    is_signed: True if a signed integer must be parsed.
+    is_long: True if a long integer must be parsed.
+
+  Returns:
+    The integer parsed.
+
+  Raises:
+    ParseError: If an integer with given characteristics couldn't be consumed.
+  """
+  try:
+    result = ParseInteger(tokenizer.token, is_signed=is_signed, is_long=is_long)
+  except ValueError as e:
+    raise tokenizer.ParseError(str(e))
+  tokenizer.NextToken()
+  return result
+
 
 def ParseInteger(text, is_signed=False, is_long=False):
   """Parses an integer.
@@ -1114,22 +1368,39 @@ def ParseInteger(text, is_signed=False, is_long=False):
     ValueError: Thrown Iff the text is not a valid integer.
   """
   # Do the actual parsing. Exception handling is propagated to caller.
+  result = _ParseAbstractInteger(text, is_long=is_long)
+
+  # Check if the integer is sane. Exceptions handled by callers.
+  checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)]
+  checker.CheckValue(result)
+  return result
+
+
+def _ParseAbstractInteger(text, is_long=False):
+  """Parses an integer without checking size/signedness.
+
+  Args:
+    text: The text to parse.
+    is_long: True if the value should be returned as a long integer.
+
+  Returns:
+    The integer value.
+
+  Raises:
+    ValueError: Thrown Iff the text is not a valid integer.
+  """
+  # Do the actual parsing. Exception handling is propagated to caller.
   try:
     # We force 32-bit values to int and 64-bit values to long to make
     # alternate implementations where the distinction is more significant
     # (e.g. the C++ implementation) simpler.
     if is_long:
-      result = long(text, 0)
+      return long(text, 0)
     else:
-      result = int(text, 0)
+      return int(text, 0)
   except ValueError:
     raise ValueError('Couldn\'t parse integer: %s' % text)
 
-  # Check if the integer is sane. Exceptions handled by callers.
-  checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)]
-  checker.CheckValue(result)
-  return result
-
 
 def ParseFloat(text):
   """Parse a floating point number.
@@ -1175,9 +1446,9 @@ def ParseBool(text):
   Raises:
     ValueError: If text is not a valid boolean.
   """
-  if text in ('true', 't', '1'):
+  if text in ('true', 't', '1', 'True'):
     return True
-  elif text in ('false', 'f', '0'):
+  elif text in ('false', 'f', '0', 'False'):
     return False
   else:
     raise ValueError('Expected "true" or "false".')
@@ -1206,14 +1477,12 @@ def ParseEnum(field, value):
     # Identifier.
     enum_value = enum_descriptor.values_by_name.get(value, None)
     if enum_value is None:
-      raise ValueError(
-          'Enum type "%s" has no value named %s.' % (
-              enum_descriptor.full_name, value))
+      raise ValueError('Enum type "%s" has no value named %s.' %
+                       (enum_descriptor.full_name, value))
   else:
     # Numeric value.
     enum_value = enum_descriptor.values_by_number.get(number, None)
     if enum_value is None:
-      raise ValueError(
-          'Enum type "%s" has no value with number %d.' % (
-              enum_descriptor.full_name, number))
+      raise ValueError('Enum type "%s" has no value with number %d.' %
+                       (enum_descriptor.full_name, number))
   return enum_value.number