Experimental lexer generator: transfer actions across epsilon transitions correctly when constructi…

tools/lexer_generator/generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 62 matching lines @@
     loads.append(load_template % ("dfa [%s]" % name, dfa_i))
   body = "\n".join(scripts) + (load_outer_template % "\n".join(loads))
   return file_template % body
 
 def process_rules(parser_state):
   rule_map = {}
   builder = NfaBuilder()
   builder.set_character_classes(parser_state.character_classes)
   for k, v in parser_state.rules.items():
     graphs = []
-    for (graph, code, action) in v['regex']:
-      # graphs.append(NfaBuilder.add_action(graph, (action, identifier)))
-      graphs.append(graph)
+    for (graph, code, action, precedence) in v['regex']:
+      graphs.append(NfaBuilder.add_action(graph, (code, action, precedence)))
     rule_map[k] = builder.nfa(NfaBuilder.or_graphs(graphs))
   html_data = []
   for rule_name, nfa in rule_map.items():
     (start, dfa_nodes) = nfa.compute_dfa()
     dfa = Dfa(start, dfa_nodes)
     html_data.append((rule_name, nfa, dfa))
   html = generate_html(html_data)
   # print html
 
 def parse_file(file_name):
   parser_state = RuleParserState()
   with open(file_name, 'r') as f:
     RuleParser.parse(f.read(), parser_state)
   process_rules(parser_state)
 
 if __name__ == '__main__':
   parse_file('src/lexer/lexer_py.re')