Experimental parser: store literals as ints

tools/lexer_generator/nfa_builder.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 110 matching lines @@
         assert sub_ends, "this creates unreachable nodes"
         self.__patch_ends(ends, sub_start)
       ends = sub_ends
     return (start, ends)
 
   def __key_state(self, key):
     state =  self.__new_state()
     state.add_unclosed_transition(key)
     return (state, [state])
 
-  def __literal(self, chars):
+  def __literal(self, *chars):
     terms = map(lambda c : Term('SINGLE_CHAR', c), chars)
     return self.__process(self.cat_terms(terms))
 
   def __single_char(self, char):
     return self.__key_state(
       TransitionKey.single_char(self.__encoding, char))
 
   def __class(self, subtree):
     return self.__key_state(TransitionKey.character_class(
       self.__encoding, Term('CLASS', subtree), self.__character_classes))
@@ skipping 171 matching lines @@
         continue
       if term.name() == name:
         for arg in term.args():
           if arg:
             yield arg
       else:
         yield term
 
   @staticmethod
   def __flatten_literals(terms):
-    literal = None
+    acc = ()
     for term in terms:
       assert isinstance(term, Term)
       if not term:
         continue
       if term.name() == 'LITERAL':
-        if literal:
-          literal += term.args()[0]
-        else:
-          literal = term.args()[0]
+        acc += term.args()
       else:
-        if literal:
-          yield Term('LITERAL', literal)
-          literal = None
+        if acc:
+          yield Term('LITERAL', *acc)
+          acc = ()
         if term:
           yield term
-    if literal:
-      yield Term('LITERAL', literal)
+    if acc:
+      yield Term('LITERAL', *acc)
 
   @staticmethod
   def or_terms(terms):
     terms = list(NfaBuilder.__flatten_terms(terms, 'OR'))
     assert terms
     return terms[0] if len(terms) == 1 else Term('OR', *terms)
 
   @staticmethod
   def cat_terms(terms):
     terms = NfaBuilder.__flatten_terms(terms, 'CAT')
     terms = list(NfaBuilder.__flatten_literals(terms))
     assert terms
     return terms[0] if len(terms) == 1 else Term('CAT', *terms)