Experimental parser: better escaping

tools/lexer_generator/generator.py

Index: tools/lexer_generator/generator.py
diff --git a/tools/lexer_generator/generator.py b/tools/lexer_generator/generator.py
index 6137a7e386864a86076a0e58d564bb23f2828814..213ee6bab28c67a36564724c9695bea30ba4c5ee 100644
--- a/tools/lexer_generator/generator.py
+++ b/tools/lexer_generator/generator.py
@@ -88,11 +88,12 @@ def process_rules(parser_state):
       if code:
         graph = NfaBuilder.add_action(graph, (precedence, code, None))
       if transition == 'continue':
-        graph = NfaBuilder.add_continue(graph)
+        if not v['default'][1][2] == 'continue':
+          graph = NfaBuilder.add_continue(graph)
       elif (transition == 'break' or
             transition == 'terminate' or
             transition == 'terminate_illegal'):
-        pass
+        NfaBuilder.add_action(graph, (-1, transition, None))
       else:
         assert k == 'default'
         graph = NfaBuilder.join_subgraph(graph, transition, rule_map[transition])
@@ -101,6 +102,9 @@ def process_rules(parser_state):
     # merge default action
     (precedence, code, transition) = v['default'][1]
     assert transition == 'continue' or transition == 'break'
+    if transition == 'continue':
+      assert k != 'default'
+      # graph = NfaBuilder.apply_modifier('*', graph)
     if code:
       graph = NfaBuilder.add_incoming_action(graph, (precedence, code, None))
     rule_map[k] = graph