Experimental parser: better generation

tools/lexer_generator/generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 72 matching lines @@
   def process(k, v):
     assert 'default' in v
     graphs = []
     for (graph, action) in v['regex']:
       (precedence, code, transition) = action
       if code:
         graph = NfaBuilder.add_action(graph, (precedence, code, None))
       if transition == 'continue':
         if not v['default'][1][2] == 'continue':
           graph = NfaBuilder.add_continue(graph)
+        else:
+          pass # TODO null key
       elif (transition == 'break' or
             transition == 'terminate' or
             transition == 'terminate_illegal'):
-        NfaBuilder.add_action(graph, (-1, transition, None))
+        graph = NfaBuilder.add_action(graph, (10000, transition, None))
       else:
         assert k == 'default'
         graph = NfaBuilder.join_subgraph(graph, transition, rule_map[transition])
       graphs.append(graph)
     graph = NfaBuilder.or_graphs(graphs)
     # merge default action
     (precedence, code, transition) = v['default'][1]
     assert transition == 'continue' or transition == 'break'
     if transition == 'continue':
       assert k != 'default'
-      # graph = NfaBuilder.apply_modifier('*', graph)
+      graph = NfaBuilder.add_incoming_action(graph, (10000, k, None))
     if code:
       graph = NfaBuilder.add_incoming_action(graph, (precedence, code, None))
     rule_map[k] = graph
   for k, v in parser_state.rules.items():
     if k == 'default': continue
     process(k, v)
   process('default', parser_state.rules['default'])
   html_data = []
   for rule_name, graph in rule_map.items():
     nfa = builder.nfa(graph)
@@ skipping 15 matching lines @@
   with open(re_file, 'r') as f:
     RuleParser.parse(f.read(), parser_state)
   html_data = process_rules(parser_state)
 
   html_file = args.html
   if html_file:
     html = generate_html(html_data)
     with open(args.html, 'w') as f:
       f.write(html)
       print "wrote html to %s" % html_file