Experimental parser: enable transition reduction

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 113 matching lines @@
       else:
         raise Exception()
     return {
       'node_number' : state.node_number(),
       'original_node_number' : state.node_number(),
       'transitions' : transitions,
       'switch_transitions' : [],
       'deferred_transitions' : [],
       'long_char_transitions' : [],
       'disjoint_keys' : disjoint_keys,
+      'has_goto_after_entry' : False,
       'inline' : None,
       'depth' : None,
       'action' : action,
       'entry_action' : entry_action,
       'match_action' : match_action,
       'class_keys' : class_keys,
       'distinct_keys' : distinct_keys,
       'ranges' : ranges
     }
 
@@ skipping 137 matching lines @@
     for i, state in enumerate(dfa_states):
       state['node_number'] = i
     def f((key, original_node_number)):
       return (key, id_map[original_node_number]['node_number'])
     for state in dfa_states:
       state['transitions'] = map(f, state['transitions'])
     assert id_map[self.__start_node_number]['node_number'] == 0
     assert len(dfa_states) == self.__dfa.node_count()
     # store states
     self.__dfa_states = dfa_states
+    return id_map
+
+  def __rewrite_gotos(self):
+    goto_map = {}
+    for state in self.__dfa_states:
+      if (state['match_action'] and
+          state['match_action'][0] == 'do_stored_token'):
+        assert not state['match_action'][1] in goto_map
+        goto_map[state['match_action'][1]] = state['node_number']
+        state['has_goto_after_entry'] = True
+    for state in self.__dfa_states:
+      if state['match_action'] and state['match_action'][0] == 'goto':
+        state['match_action'] = ('goto', goto_map[state['match_action'][1]])
+
+  def process(self):
+
+    id_map = self.__canonicalize_traversal()
+    self.__rewrite_gotos()
+
+    dfa_states = self.__dfa_states
     # set nodes to inline
     if self.__inline:
       inlined = reduce(self.__set_inline, dfa_states, 0)
       if self.__log:
         print "%s states inlined" % inlined
     elif self.__log:
       print "no inlining"
     # split transitions
     switched = reduce(self.__split_transitions, dfa_states, 0)
     if self.__log:
       print "%s states use switch (instead of if)" % switched
     # rewrite deferred transitions
     for state in dfa_states:
       self.__rewrite_deferred_transitions(state)
 
-  def process(self):
-
-    self.__canonicalize_traversal()
-
-    dfa_states = self.__dfa_states
-
     default_action = self.__default_action
     assert(default_action and default_action.match_action())
     default_action = default_action.match_action()
 
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     template_env = jinja2.Environment(
       loader = jinja2.PackageLoader('lexer_generator', '.'),
       undefined = jinja2.StrictUndefined)
     template = template_env.get_template('code_generator.jinja')
 
     encoding = self.__dfa.encoding()
     char_types = {'latin1': 'uint8_t', 'utf16': 'uint16_t', 'utf8': 'int8_t'}
     char_type = char_types[encoding.name()]
 
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
       encoding = encoding.name(),
       char_type = char_type,
       upper_bound = encoding.primary_range()[1])