Experimental lexer generator: add a test for the code generator.

tools/lexer_generator/code_generator.py

Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py b/tools/lexer_generator/code_generator.py
index cafa0841bdfd7cb40988e7fd5f49a4bb349cbd29..559b8d95442a666c9f8d858a0d4c8b5321accf43 100644
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -63,10 +63,10 @@ code_%s:
 
     action = state.action()
     if action:
-      if action[1] == 'terminate':
+      if action.type() == 'terminate':
         code += 'return 0;'
         return code
-      elif action[1] == 'terminate_illegal':
+      elif action.type() == 'terminate_illegal':
         code += 'return 1;'
         return code
 
@@ -82,7 +82,7 @@ code_%s:
 ''' % s.node_number()
 
     if action:
-      code += '%s\nyych = *(--cursor_);\ngoto code_%s;\n' % (state.action()[1],
+      code += '%s\nyych = *(--cursor_);\ngoto code_%s;\n' % (action.data(),
                                                              start_node_number)
     return code