Experimental parser: remove match actions

tools/lexer_generator/automaton.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 91 matching lines @@
 
   def visit_all_states(self, visitor, visit_state = None, state_iter = None):
     return self.visit_states(self.start_set(), visitor, visit_state, state_iter)
 
   @staticmethod
   def epsilon_closure(states):
     f = lambda state : state.epsilon_closure_iter()
     return set(chain(iter(states), *map(f, states)))
 
   @staticmethod
-  def __transition_states_for_char(valid_states, c):
-    f = lambda state : state.transition_state_iter_for_char(c)
-    return set(chain(*map(f, Automaton.epsilon_closure(valid_states))))
+  def __omega_closure(states):
+    f = lambda s : s.transition_state_iter_for_key(TransitionKey.omega())
+    new_states = set(chain(*map(f, states)))
+    return set(chain(iter(states), iter(Automaton.epsilon_closure(new_states))))
+
+  @staticmethod
+  def __transition_states_for_char(states, c):
+    f = lambda s : s.transition_state_iter_for_char(c)
+    states = set(chain(*map(f, Automaton.epsilon_closure(states))))
+    return Automaton.__omega_closure(states)
 
   def matches(self, string):
     valid_states = self.start_set()
     for c in string:
       valid_states = Automaton.__transition_states_for_char(valid_states, c)
       if not valid_states:
         return False
-    valid_states = self.epsilon_closure(valid_states)
+    valid_states = self.__omega_closure(self.epsilon_closure(valid_states))
     return len(self.terminal_set().intersection(valid_states)) > 0
 
+  @staticmethod
+  def dominant_action(states):
+    return Action.dominant_action(map(lambda s: s.action(), states))
+
   def lex(self, string, default_action):
     last_position = 0
     valid_states = self.start_set()
     for pos, c in enumerate(string):
       transitions = Automaton.__transition_states_for_char(valid_states, c)
       if transitions:
         valid_states = transitions
         continue
-      action = Action.dominant_action(valid_states)
+      # TODO(dcarney): action collection should walk omega transitions
+      action = self.dominant_action(valid_states)
       if not action:
         assert default_action
         action = default_action
       yield (action, last_position, pos)
       last_position = pos
       # lex next token
       valid_states = Automaton.__transition_states_for_char(self.start_set(), c)
       assert valid_states
-    valid_states = self.epsilon_closure(valid_states)
-    action = Action.dominant_action(valid_states)
+    valid_states = self.__omega_closure(self.epsilon_closure(valid_states))
+    action = self.dominant_action(valid_states)
     if not action:
       assert default_action
       action = default_action
     yield (action, last_position, len(string))