Experimental parser: store literals as ints

tools/lexer_generator/regex_parser.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 220 matching lines @@
       p[0] = ("REPEAT", p[2], p[2])
     elif len(p) == 6:
       p[0] = ("REPEAT", p[2], p[4])
     elif p[1]:
       p[0] = (self.token_map[p[1]],)
     else:
       p[0] = None
 
   def p_literal(self, p):
     '''literal : LITERAL'''
-    p[0] = Term('LITERAL', p[1])
+    p[0] = Term('LITERAL', ord(p[1]))
 
   def p_any(self, p):
     '''any : ANY'''
     p[0] = Term(self.token_map[p[1]])
 
   def p_class(self, p):
     '''class : CLASS_BEGIN class_content CLASS_END
              | CLASS_BEGIN NOT class_content CLASS_END'''
     if len(p) == 4:
       p[0] = Term("CLASS", p[2])
     else:
       p[0] = Term("NOT_CLASS", p[3])
 
   def p_group(self, p):
     '''group : GROUP_BEGIN start GROUP_END'''
     p[0] = p[2]
 
   def p_class_content(self, p):
     '''class_content : CLASS_LITERAL RANGE CLASS_LITERAL maybe_class_content
                      | CLASS_LITERAL maybe_class_content
                      | CHARACTER_CLASS maybe_class_content
     '''
     if len(p) == 5:
-      left = Term("RANGE", p[1], p[3])
+      left = Term("RANGE", ord(p[1]), ord(p[3]))
     else:
       if len(p[1]) == 1:
-        left = Term('LITERAL', p[1])
+        left = Term('LITERAL', ord(p[1]))
       else:
         left = Term('CHARACTER_CLASS', p[1][1:-1])
     p[0] = self.__cat(left, p[len(p)-1])
 
   def p_maybe_class_content(self, p):
     '''maybe_class_content : class_content
                            | empty'''
     p[0] = p[1]
 
   def p_empty(self, p):
     'empty :'
 
   def p_error(self, p):
     raise Exception("Syntax error in input '%s'" % str(p))
 
   @staticmethod
   def __cat(left, right):
     assert left
     return NfaBuilder.cat_terms([left] if not right else [left, right])
 
   @staticmethod
   def parse(string):
     new_lexer = lambda: RegexLexer()
     new_parser = lambda: RegexParser()
     return ParserBuilder.parse(string, "RegexParser", new_lexer, new_parser)