Experimental parser: better eos handling

tools/lexer_generator/rule_parser.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 81 matching lines @@
       }
     p[0] = state.current_state
 
   def p_transition_rules(self, p):
     '''transition_rules : transition_rule transition_rules
                         | empty'''
 
   def p_transition_rule(self, p):
     '''transition_rule : composite_regex action
                        | DEFAULT_ACTION default_action
+                       | EOS eos
                        | CATCH_ALL action'''
     precedence = RuleParser.__rule_precedence_counter
     RuleParser.__rule_precedence_counter += 1
     action = p[2]
     (entry_action, match_action, transition) = action
     if transition and not transition in self.__keyword_transitions:
       assert not transition == 'default', "can't append default graph"
       self.__state.transitions.add(transition)
     rules = self.__state.rules[self.__state.current_state]
     if p[1] == 'default_action':
       assert self.__state.current_state == 'default'
       assert not rules['default_action']
       rules['default_action'] = action
-    elif p[1] == 'catch_all':
+    elif p[1] == 'eos' or p[1] == 'catch_all':
       assert p[1] not in rules['uniques']
       rules['uniques'][p[1]] = True
       rules['regex'].append((NfaBuilder.unique_key(p[1]), precedence, action))
     else:
       regex = p[1]
       rules['regex'].append((regex, precedence, action))
 
   def p_action(self, p):
     '''action : ACTION_OPEN maybe_identifier_action OR maybe_identifier_action OR maybe_transition ACTION_CLOSE'''
     p[0] = (p[2], p[4], p[6])
 
   def p_default_action(self, p):
     'default_action : ACTION_OPEN identifier_action ACTION_CLOSE'
     p[0] = (None, p[2], None)
 
+  def p_eos(self, p):
+    'eos : ACTION_OPEN identifier_action ACTION_CLOSE'
+    p[0] = (None, p[2], None)
+
   def p_maybe_identifier_action(self, p):
     '''maybe_identifier_action : identifier_action
                          | empty'''
     p[0] = p[1]
 
   def p_maybe_transition(self, p):
     '''maybe_transition : IDENTIFIER
                         | empty'''
     p[0] = p[1]
 
@@ skipping 184 matching lines @@
       if k == 'default': continue
       process(k, v)
     process('default', parser_state.rules['default'])
     # build the automata
     for rule_name, graph in rule_map.items():
       self.__automata[rule_name] = RuleProcessor.Automata(builder, graph)
       self.__rule_trees[rule_name] = graph
     # process default_action
     default_action = parser_state.rules['default']['default_action']
     self.default_action = Action(None, default_action[1]) if default_action else None