Mojo: Mojom: Add basic lexer tests, and remove hex float constants.

mojo/public/bindings/pylib/parse/mojo_lexer.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import re
 import sys
 import os.path
 
 # Try to load the ply module, if not, then assume it is in the third_party
 # directory.
@@ skipping 60 matching lines @@
   ##
   ## All the tokens recognized by the lexer
   ##
   tokens = keywords + (
     # Identifiers
     'NAME',
 
     # Constants
     'ORDINAL',
     'INT_CONST_DEC', 'INT_CONST_OCT', 'INT_CONST_HEX',
-    'FLOAT_CONST', 'HEX_FLOAT_CONST',
+    'FLOAT_CONST',
     'CHAR_CONST',
 
     # String literals
     'STRING_LITERAL',
 
     # Operators
     'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD',
     'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT',
 
     # Assignment
@@ skipping 55 matching lines @@
   # string literals (K&R2: A.2.6)
   string_char = r"""([^"\\\n]|"""+escape_sequence+')'
   string_literal = '"'+string_char+'*"'
   bad_string_literal = '"'+string_char+'*'+bad_escape+string_char+'*"'
 
   # floating constants (K&R2: A.2.5.3)
   exponent_part = r"""([eE][-+]?[0-9]+)"""
   fractional_constant = r"""([0-9]*\.[0-9]+)|([0-9]+\.)"""
   floating_constant = \
       '(((('+fractional_constant+')'+ \
-      exponent_part+'?)|([0-9]+'+exponent_part+'))[FfLl]?)'
-  binary_exponent_part = r'''([pP][+-]?[0-9]+)'''
-  hex_fractional_constant = \
-      '((('+hex_digits+r""")?\."""+hex_digits+')|('+hex_digits+r"""\.))"""
-  hex_floating_constant = \
-      '('+hex_prefix+'('+hex_digits+'|'+hex_fractional_constant+')'+ \
-      binary_exponent_part+'[FfLl]?)'
+      exponent_part+'?)|([0-9]+'+exponent_part+')))'
 
   # Ordinals
   ordinal = r'@[0-9]+'
   missing_ordinal_value = r'@'
   # Don't allow ordinal values in octal (even invalid octal, like 09) or
   # hexadecimal.
   octal_or_hex_ordinal_disallowed = r'@((0[0-9]+)|('+hex_prefix+hex_digits+'))'
 
   ##
   ## Rules for the normal state
@@ skipping 41 matching lines @@
 
   # The following floating and integer constants are defined as
   # functions to impose a strict order (otherwise, decimal
   # is placed before the others because its regex is longer,
   # and this is bad)
   #
   @TOKEN(floating_constant)
   def t_FLOAT_CONST(self, t):
     return t
 
-  @TOKEN(hex_floating_constant)
-  def t_HEX_FLOAT_CONST(self, t):
-    return t
-
   @TOKEN(hex_constant)
   def t_INT_CONST_HEX(self, t):
     return t
 
   @TOKEN(bad_octal_constant)
   def t_BAD_CONST_OCT(self, t):
     msg = "Invalid octal constant"
     self._error(msg, t)
 
   @TOKEN(octal_constant)
@@ skipping 49 matching lines @@
     return t
 
   # Ignore C and C++ style comments
   def t_COMMENT(self, t):
     r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)'
     pass
 
   def t_error(self, t):
     msg = 'Illegal character %s' % repr(t.value[0])
     self._error(msg, t)