Experimental parser: better actions

tools/lexer_generator/lexer_test.py

Index: tools/lexer_generator/lexer_test.py
diff --git a/tools/lexer_generator/lexer_test.py b/tools/lexer_generator/lexer_test.py
index 0596c84a14b43f66a0bc80248be74f5affb3a628..a165349547c72b5a7c81886c76bb0ea9593b002c 100644
--- a/tools/lexer_generator/lexer_test.py
+++ b/tools/lexer_generator/lexer_test.py
@@ -30,60 +30,44 @@ from generator import Generator
 
 class LexerTestCase(unittest.TestCase):
 
-  def __verify_action_stream(self, stream, expected_stream):
-    for (ix, item) in enumerate(expected_stream):
-      self.assertEquals(stream[ix][0], item[0])
-      self.assertEquals(stream[ix][4], item[1])
+  def __verify_action_stream(self, rules, string, expected_stream):
+    expected_stream.append(('terminate', '\0'))
+    for i, (action, start, stop) in enumerate(Generator(rules).lex(string)):
+      self.assertEquals(expected_stream[i][0], action)
+      self.assertEquals(expected_stream[i][1], string[start : stop])
 
   def test_simple(self):
     rules = '''
     <default>
-    "("           { LBRACE } <<continue>>
-    ")"           { RBRACE } <<continue>>
+    "("           { LBRACE }
+    ")"           { RBRACE }
 
-    "foo"         { FOO } <<continue>>
-    eof           <<terminate>>
-    default       <<break>>'''
-
-    generator = Generator(rules)
+    "foo"         { FOO }
+    eof           <<terminate>>'''
 
     string = 'foo()\0'
-    self.__verify_action_stream(
-        generator.lex(string),
+    self.__verify_action_stream(rules, string,
         [('FOO', 'foo'), ('LBRACE', '('), ('RBRACE', ')')])
 
   def test_maximal_matching(self):
     rules = '''
     <default>
-    "<"           { LT } <<continue>>
-    "<<"          { SHL } <<continue>>
-    " "           { SPACE } <<continue>>
-    eof           <<terminate>>
-    default       <<break>>'''
-
-    generator = Generator(rules)
+    "<"           { LT }
+    "<<"          { SHL }
+    " "           { SPACE }
+    eof           <<terminate>>'''
 
     string = '<< <\0'
-    self.__verify_action_stream(
-        generator.lex(string),
+    self.__verify_action_stream(rules, string,
         [('SHL', '<<'), ('SPACE', ' '), ('LT', '<')])
 
-
   def test_consecutive_epsilon_transitions(self):
-    # This rule set will create a NFA where we first have an epsilon transition,
-    # then following that, an epsilon transition with an action. This will test
-    # that the action is correctly pulled forward when creating the dfa.
     rules = '''
     digit = [0-9];
     number = (digit+ ("." digit+)?);
     <default>
-    number        { NUMBER } <<continue>>
-    eof           <<terminate>>
-    default       <<break>>'''
-
-    generator = Generator(rules)
+    number        { NUMBER }
+    eof           <<terminate>>'''
 
     string = '555\0'
-    self.__verify_action_stream(
-        generator.lex(string),
-        [('NUMBER', '555')])
+    self.__verify_action_stream(rules, string, [('NUMBER', '555')])