Experimental parser: cleanup deferred transitions

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 87 matching lines @@
       'transitions' : transitions,
       # flags for code generator
       'can_elide_read' : len(transitions) == 0,
       'is_eos_handler' : False,
       'inline' : None,
       'must_not_inline' : False,
       # transitions for code generator
       'if_transitions' : [],
       'switch_transitions' : [],
       'deferred_transitions' : [],
-      'long_char_transitions' : [],
       'unique_transitions' : unique_transitions,
       # state actions
       'entry_action' : entry_action,
       'match_action' : match_action,
       # statistics for states
       'class_keys' : class_keys,
       'distinct_keys' : distinct_keys,
       'ranges' : ranges,
       # record of which entry points will be needed
       'entry_points' : {k : False for k in CodeGenerator.__jump_labels}
@@ skipping 58 matching lines @@
     return split_count + (0 if no_switch else 1)
 
   __call_map = {
     'non_primary_whitespace' : 'IsWhiteSpaceNotLineTerminator',
     'non_primary_letter' : 'IsLetter',
     'non_primary_identifier_part_not_letter' : 'IsIdentifierPartNotLetter',
     'non_primary_line_terminator' : 'IsLineTerminator',
   }
 
   def __rewrite_deferred_transitions(self, state):
-    assert not state['long_char_transitions']
     transitions = state['deferred_transitions']
     if not transitions:
       return
     encoding = self.__dfa.encoding()
     bom = 'byte_order_mark'
     catch_all = 'non_primary_everything_else'
     all_classes = set(encoding.class_name_iter())
-    fast_classes = set([])
-    call_classes = all_classes - fast_classes - set([bom, catch_all])
+    call_classes = all_classes - set([bom, catch_all])
     def remap_transition(class_name):
       if class_name in call_classes:
         return ('LONG_CHAR_CLASS', 'call', self.__call_map[class_name])
       if class_name == bom:
         return ('LONG_CHAR_CLASS', class_name)
       raise Exception(class_name)
-    fast_transitions = []
     long_class_transitions = []
     long_class_map = {}
     catchall_transition = None
     # loop through and remove catch_all_transitions
     for (classes, transition_node_id) in transitions:
-      ft = []
       lct = []
       has_catch_all = False
       for (class_type, class_name) in classes:
-        if class_name in fast_classes:
-          ft.append((class_type, class_name))
+        assert not class_name in long_class_map
+        long_class_map[class_name] = transition_node_id
+        if class_name == catch_all:
+          assert not has_catch_all
+          assert catchall_transition == None
+          has_catch_all = True
         else:
-          assert not class_name in long_class_map
-          long_class_map[class_name] = transition_node_id
-          if class_name == catch_all:
-            assert not has_catch_all
-            assert catchall_transition == None
-            has_catch_all = True
-          else:
-            lct.append(remap_transition(class_name))
-      if ft:
-        fast_transitions.append((ft, transition_node_id))
+          lct.append(remap_transition(class_name))
       if has_catch_all:
         catchall_transition = (lct, transition_node_id)
       elif lct:
         long_class_transitions.append((lct, transition_node_id))
-    # all transitions are fast
-    if not long_class_map:
-      return
     if catchall_transition:
-      catchall_transitions = all_classes - fast_classes
+      catchall_transitions = all_classes
       for class_name in long_class_map.iterkeys():
         catchall_transitions.remove(class_name)
       assert not catchall_transitions, "class inversion not unimplemented"
-    # split deferred transitions
-    state['deferred_transitions'] = fast_transitions
     if catchall_transition:
       catchall_transition = [
         ([('LONG_CHAR_CLASS', 'catch_all')], catchall_transition[1])]
     else:
       catchall_transition = []
-    state['long_char_transitions'] = (long_class_transitions +
-                                      catchall_transition) # must be last
+    state['deferred_transitions'] = (long_class_transitions +
+                                     catchall_transition) # must be last
 
   @staticmethod
   def __reorder(current_node_number, id_map, dfa_states):
     current_node = id_map[current_node_number]
     if current_node['node_number'] != None:
       return
     current_node['node_number'] = len(dfa_states)
     dfa_states.append(current_node)
     for (key, node_number) in current_node['transitions']:
       CodeGenerator.__reorder(node_number, id_map, dfa_states)
@@ skipping 73 matching lines @@
     state['inline'] = None
     # clear entry points
     state['entry_points'] = {k : False for k in CodeGenerator.__jump_labels}
     return state['node_number']
 
   def __rewrite_transitions_to_jumps(self, start_id, count, inline_mapping_in):
     # order here should match the order of code generation
     transition_names = [
       'switch_transitions',
       'if_transitions',
-      'long_char_transitions',
       'deferred_transitions']
     end_offset = start_id + count
     assert len(self.__dfa_states) == end_offset
     total_nodes_created = 0
     for state_id in range(start_id, end_offset):
       state = self.__dfa_states[state_id]
       # this will be ignored during code generation,
       # and it's needed as a template, so don't rewrite
       if state['inline'] == True:
         continue
@@ skipping 74 matching lines @@
 
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
       jump_table = self.__jump_table,
       encoding = encoding.name(),
       char_type = char_type,
       upper_bound = encoding.primary_range()[1])