Experimental parser: store literals as ints

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
 #       with the distribution.
 #     * Neither the name of Google Inc. nor the names of its
 #       contributors may be used to endorse or promote products derived
 #       from this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+from types import IntType
 from itertools import chain
 from action import Term
 from string import printable
 
 class KeyEncoding(object):
 
   __encodings = {}
 
   @staticmethod
   def get(name):
@@ skipping 99 matching lines @@
 
   @staticmethod
   def epsilon():
     '''Returns a TransitionKey for the epsilon (empty) transition.'''
     return TransitionKey.__cached_key(None, 'epsilon',
       lambda : Term("EPSILON_KEY"))
 
   @staticmethod
   def omega():
     '''Always matches.'''
-    return TransitionKey.__cached_key(None, 'omega',
-      lambda : Term("OMEGA_KEY"))
+    return TransitionKey.__cached_key(None, 'omega', lambda : Term("OMEGA_KEY"))
 
   @staticmethod
   def any(encoding):
     '''Returns a TransitionKey which matches any encoded character.'''
     return TransitionKey.__cached_key(encoding, 'any',
         lambda : encoding.all_components_iter())
 
   @staticmethod
-  def single_char(encoding, char):  # TODO(dcarney): char should be int
+  def single_char(encoding, char):
     '''Returns a TransitionKey for a single-character transition.'''
-    return TransitionKey(encoding, Term("NUMERIC_RANGE_KEY", ord(char), ord(char)))
+    return TransitionKey.range(encoding, char, char)
 
   @staticmethod
   def range(encoding, a, b):
     '''Returns a TransitionKey for a single-character transition.'''
+    assert type(a) == IntType and type(b) == IntType
     return TransitionKey(encoding, Term("NUMERIC_RANGE_KEY", a, b))
 
   @staticmethod
   def unique(term):  # TODO(dcarney): rename
     '''Returns a unique TransitionKey for the given name (and creates it if it
     doesn't exist yet). The TransitionKey won't have any real character range,
     but a newly-created "mock" character range which is separate from all other
     character ranges.'''
     return TransitionKey(None, Term("TERM_KEY", term))
 
   @staticmethod
   def __process_term(encoding, term, components, key_map):
     key = term.name()
     args = term.args()
     if key == 'RANGE':
-      components.append(Term('NUMERIC_RANGE_KEY', ord(args[0]), ord(args[1])))
+      components.append(Term('NUMERIC_RANGE_KEY', args[0], args[1]))
     elif key == 'LITERAL':
-      for char in args[0]:  # TODO(dcarney): don't use strings for literals
-        components.append(Term('NUMERIC_RANGE_KEY', ord(char), ord(char)))
+      components += map(lambda x : Term('NUMERIC_RANGE_KEY', x, x), args)
     elif key == 'CAT':
       for x in args:
         TransitionKey.__process_term(encoding, x, components, key_map)
     elif key == 'CHARACTER_CLASS':
       class_name = args[0]
       if encoding.named_range(class_name):
         c = encoding.named_range(class_name)
         if class_name in key_map:
           assert key_map[class_name] == TransitionKey(encoding, c)
         components.append(c)
@@ skipping 450 matching lines @@
       {
         'whitespace':
           [(9, 9), (11, 12), (32, 32), ('non_primary_whitespace',)],
         'letter':
           [(65, 90), (97, 122), ('non_primary_letter',)],
         'line_terminator':
           [(10, 10), (13, 13), ('non_primary_line_terminator',)],
         'identifier_part_not_letter':
           [(48, 57), (95, 95), ('non_primary_identifier_part_not_letter',)],
       })