Experimental parser: remove hacks from replace_tokens_with_gotos.

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 48 matching lines @@
     entry_action = None if not action else action.entry_action()
     match_action = None if not action else action.match_action()
     # generate ordered transitions
     transitions = map(lambda (k, v) : (k, v.node_number()),
                       state.transitions().items())
     def cmp(left, right):
       return TransitionKey.canonical_compare(left[0], right[0])
     transitions = sorted(transitions, cmp)
     # map transition keys to disjoint ranges and collect stats
     disjoint_keys = []
-    eos_transition = None
+    unique_transitions = {}
     old_transitions = transitions
     transitions = []
     (class_keys, distinct_keys, ranges) = (0, 0, 0)
     for key, transition_id in old_transitions:
-      keys = list(key.range_iter(encoding))
-      eos_found = False
-      for (t, r) in keys:
+      keys = []
+      for (t, r) in key.range_iter(encoding):
         if t == 'CLASS':
           class_keys += 1
+          keys.append((t, r))
         elif t == 'PRIMARY_RANGE':
           distinct_keys += r[1] - r[0] + 1
           ranges += 1
+          keys.append((t, r))
         elif t == 'UNIQUE':
-          assert r == 'eos'
-          assert len(keys) == 1
-          assert eos_transition == None
-          eos_transition = transition_id
-          eos_found = True
+          assert r == 'no_match' or r == 'eos'
+          assert r not in unique_transitions
+          unique_transitions[r] = transition_id
         else:
           raise Exception()
-      if not eos_found:
+      if keys:
         transitions.append((keys, transition_id))
     # eos_transitions is for a followup cl
-    assert not eos_transition
     return {
       'node_number' : None,
       'original_node_number' : state.node_number(),
       'transitions' : transitions,
       # flags for code generator
       'can_elide_read' : len(transitions) == 0,
       'is_eos_handler' : False,
       'inline' : None,
       # transitions for code generator
       'if_transitions' : [],
       'switch_transitions' : [],
       'deferred_transitions' : [],
       'long_char_transitions' : [],
-      'eos_transition' : eos_transition,
+      'unique_transitions' : unique_transitions,
       # state actions
       'entry_action' : entry_action,
       'match_action' : match_action,
       # statistics for states
       'class_keys' : class_keys,
       'distinct_keys' : distinct_keys,
       'ranges' : ranges,
       # record of which entry points will be needed
       'entry_points' : {
         'state_entry' : True,
@@ skipping 123 matching lines @@
 
   @staticmethod
   def __reorder(current_node_number, id_map, dfa_states):
     current_node = id_map[current_node_number]
     if current_node['node_number'] != None:
       return
     current_node['node_number'] = len(dfa_states)
     dfa_states.append(current_node)
     for (key, node_number) in current_node['transitions']:
       CodeGenerator.__reorder(node_number, id_map, dfa_states)
+    for node_number in current_node['unique_transitions'].values():
+      CodeGenerator.__reorder(node_number, id_map, dfa_states)
 
   def __build_dfa_states(self):
     dfa_states = []
     self.__dfa.visit_all_states(lambda state, acc: dfa_states.append(state))
     encoding = self.__dfa.encoding()
     f = lambda state : CodeGenerator.__transform_state(encoding, state)
     dfa_states = map(f, dfa_states)
     id_map = {x['original_node_number'] : x for x in dfa_states}
     dfa_states = []
     CodeGenerator.__reorder(self.__start_node_number, id_map, dfa_states)
@@ skipping 78 matching lines @@
     char_type = char_types[encoding.name()]
 
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
       encoding = encoding.name(),
       char_type = char_type,
       upper_bound = encoding.primary_range()[1])