Experimental parser: better eos handling

tools/lexer_generator/rule_parser.py

Index: tools/lexer_generator/rule_parser.py
diff --git a/tools/lexer_generator/rule_parser.py b/tools/lexer_generator/rule_parser.py
index 061c78be8390735db86a4b2f186e5b445e9f2ea5..72d4e105bb716c36603f01fecc89e407cba6c323 100644
--- a/tools/lexer_generator/rule_parser.py
+++ b/tools/lexer_generator/rule_parser.py
@@ -99,6 +99,7 @@ class RuleParser:
   def p_transition_rule(self, p):
     '''transition_rule : composite_regex action
                        | DEFAULT_ACTION default_action
+                       | EOS eos
                        | CATCH_ALL action'''
     precedence = RuleParser.__rule_precedence_counter
     RuleParser.__rule_precedence_counter += 1
@@ -112,7 +113,7 @@ class RuleParser:
       assert self.__state.current_state == 'default'
       assert not rules['default_action']
       rules['default_action'] = action
-    elif p[1] == 'catch_all':
+    elif p[1] == 'eos' or p[1] == 'catch_all':
       assert p[1] not in rules['uniques']
       rules['uniques'][p[1]] = True
       rules['regex'].append((NfaBuilder.unique_key(p[1]), precedence, action))
@@ -128,6 +129,10 @@ class RuleParser:
     'default_action : ACTION_OPEN identifier_action ACTION_CLOSE'
     p[0] = (None, p[2], None)
 
+  def p_eos(self, p):
+    'eos : ACTION_OPEN identifier_action ACTION_CLOSE'
+    p[0] = (None, p[2], None)
+
   def p_maybe_identifier_action(self, p):
     '''maybe_identifier_action : identifier_action
                          | empty'''