Experimental lexer generator: fix dfa generation when there are epsilon transitions with actions.

tools/lexer_generator/lexer_test.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 47 matching lines @@
     "<<"          { SHL }
     " "           { SPACE }
     eof           <<terminate>>'''
 
     lexer = Lexer(rules)
 
     string = '<< <\0'
     self.__verify_action_stream(
         lexer.lex(string),
         [('SHL', '<<'), ('SPACE', ' '), ('LT', '<')])
+
+
+  def test_consecutive_epsilon_transitions(self):
+    # This rule set will create a NFA where we first have an epsilon transition,
+    # then following that, an epsilon transition with an action. This will test
+    # that the action is correctly pulled forward when creating the dfa.
+    rules = '''
+    digit = [0-9];
+    number = (digit+ ("." digit+)?);
+    <default>
+    number        { NUMBER }
+    eof           <<terminate>>'''
+
+    lexer = Lexer(rules)
+
+    string = '555\0'
+    self.__verify_action_stream(
+        lexer.lex(string),
+        [('NUMBER', '555')])