Experimental parser: some cleanup and assertions for rule_parser

tools/lexer_generator/rule_parser.py

Index: tools/lexer_generator/rule_parser.py
diff --git a/tools/lexer_generator/rule_parser.py b/tools/lexer_generator/rule_parser.py
index a616fec33d1c2faa49c833e85cf83b94b11f5003..ebdd4fa556b4d5ce79ad96f5cae54e3d8ff13837 100644
--- a/tools/lexer_generator/rule_parser.py
+++ b/tools/lexer_generator/rule_parser.py
@@ -40,6 +40,7 @@ class RuleParserState:
     self.character_classes = {}
     self.current_state = None
     self.rules = {}
+    self.transitions = set()
 
   def parse(self, string):
     return RuleParser.parse(string, self)
@@ -48,6 +49,8 @@ class RuleParser:
 
   tokens = RuleLexer.tokens
   __rule_precedence_counter = 0
+  __keyword_transitions = set([
+      'continue', 'break', 'terminate', 'terminate_illegal'])
 
   def __init__(self):
     self.__state = None
@@ -95,9 +98,15 @@ class RuleParser:
     '''transition_rule : composite_regex_or_default code action
                        | composite_regex_or_default empty action
                        | composite_regex_or_default code empty'''
-    rules = self.__state.rules[self.__state.current_state]
-    rule = (p[1], RuleParser.__rule_precedence_counter, p[2], p[3])
+    transition = p[3] if p[3] else 'continue'
+    if transition == 'continue' and self.__state.current_state == 'default':
+      transition = 'break'
+    if not transition in self.__keyword_transitions:
+      assert not transition == 'default'
+      self.__state.transitions.add(transition)
+    rule = (p[1], (RuleParser.__rule_precedence_counter, p[2], transition))
     RuleParser.__rule_precedence_counter += 1
+    rules = self.__state.rules[self.__state.current_state]
     if p[1] == 'default':
       assert not rules['default']
       rules['default'] = rule
@@ -197,7 +206,8 @@ class RuleParser:
     parser.__state = parser_state
     try:
       parser.parser.parse(data, lexer=parser.lexer.lexer)
-    except Exception as e:
+    except Exception:
       RuleParser.__static_instance = None
-      raise e
+      raise
+    assert parser_state.transitions <= set(parser_state.rules.keys())
     parser.__state = None