Experimental parser: store literals as ints

tools/lexer_generator/nfa_builder.py

Index: tools/lexer_generator/nfa_builder.py
diff --git a/tools/lexer_generator/nfa_builder.py b/tools/lexer_generator/nfa_builder.py
index d2a2d566c7b82921920a80ba32bee198a23de10d..ea30c2ce0e605a6bd0c92073691e24df8cd6c2e5 100644
--- a/tools/lexer_generator/nfa_builder.py
+++ b/tools/lexer_generator/nfa_builder.py
@@ -128,7 +128,7 @@ class NfaBuilder(object):
     state.add_unclosed_transition(key)
     return (state, [state])
 
-  def __literal(self, chars):
+  def __literal(self, *chars):
     terms = map(lambda c : Term('SINGLE_CHAR', c), chars)
     return self.__process(self.cat_terms(terms))
 
@@ -320,24 +320,21 @@ class NfaBuilder(object):
 
   @staticmethod
   def __flatten_literals(terms):
-    literal = None
+    acc = ()
     for term in terms:
       assert isinstance(term, Term)
       if not term:
         continue
       if term.name() == 'LITERAL':
-        if literal:
-          literal += term.args()[0]
-        else:
-          literal = term.args()[0]
+        acc += term.args()
       else:
-        if literal:
-          yield Term('LITERAL', literal)
-          literal = None
+        if acc:
+          yield Term('LITERAL', *acc)
+          acc = ()
         if term:
           yield term
-    if literal:
-      yield Term('LITERAL', literal)
+    if acc:
+      yield Term('LITERAL', *acc)
 
   @staticmethod
   def or_terms(terms):