Experimental parser: cleanup state transformation in code generator

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 35 matching lines @@
       dfa = rule_processor.default_automata().dfa()
     self.__dfa = dfa
     self.__default_action = rule_processor.default_action
     self.__debug_print = debug_print
     self.__start_node_number = self.__dfa.start_state().node_number()
     self.__log = log
     self.__inline = inline
     self.__switching = switching
 
   @staticmethod
-  def __range_cmp(left, right):
-    if left[0] == 'PRIMARY_RANGE':
-      if right[0] == 'PRIMARY_RANGE':
-        return cmp(left[1], right[1])
-      assert right[0] == 'CLASS'
-      return -1
-    assert left[0] == 'CLASS'
-    if right[0] == 'PRIMARY_RANGE':
-      return 1
-    # TODO store numeric values and cmp
-    return cmp(left[1], right[1])
-
-  @staticmethod
   def __transform_state(encoding, state):
     # action data
     action = state.action()
     entry_action = None if not action else action.entry_action()
     match_action = None if not action else action.match_action()
     # generate ordered transitions
     transitions = map(lambda (k, v) : (k, v.node_number()),
                       state.transitions().items())
     def cmp(left, right):
       return TransitionKey.canonical_compare(left[0], right[0])
     transitions = sorted(transitions, cmp)
-    # map transition keys to disjoint ranges
-    disjoint_keys = {'value' : []}
-    def f((key, state)):
-      ranges = list(key.range_iter(encoding))
-      disjoint_keys['value'] += ranges
-      return (ranges, state)
-    transitions = map(f, transitions)
-    disjoint_keys = sorted(disjoint_keys['value'], CodeGenerator.__range_cmp)
-    # dictionary object representing state
+    # map transition keys to disjoint ranges and collect stats
+    disjoint_keys = []
+    eos_transition = None
+    old_transitions = transitions
+    transitions = []
     (class_keys, distinct_keys, ranges) = (0, 0, 0)
-    for (t, r) in disjoint_keys:
-      if t == 'CLASS':
-        class_keys += 1
-      elif t == 'PRIMARY_RANGE':
-        distinct_keys += r[1] - r[0] + 1
-        ranges += 1
-      else:
-        raise Exception()
+    for key, transition_id in old_transitions:
+      keys = list(key.range_iter(encoding))
+      eos_found = False
+      for (t, r) in keys:
+        if t == 'CLASS':
+          class_keys += 1
+        elif t == 'PRIMARY_RANGE':
+          distinct_keys += r[1] - r[0] + 1
+          ranges += 1
+        elif t == 'UNIQUE':
+          assert r == 'eos'
+          assert len(keys) == 1
+          assert eos_transition == None
+          eos_transition = transition_id
+          eos_found = True
+        else:
+          raise Exception()
+      if not eos_found:
+        transitions.append((keys, transition_id))
+    # eos_transitions is for a followup cl
+    assert not eos_transition
     return {
       'node_number' : None,
       'original_node_number' : state.node_number(),
       'transitions' : transitions,
+      # flags for code generator
       'can_elide_read' : len(transitions) == 0,
+      'is_eos_handler' : False,
+      'inline' : None,
+      # transitions for code generator
+      'if_transitions' : [],
       'switch_transitions' : [],
       'deferred_transitions' : [],
       'long_char_transitions' : [],
-      'disjoint_keys' : disjoint_keys,
-      'inline' : None,
-      'action' : action,
+      'eos_transition' : eos_transition,
+      # state actions
       'entry_action' : entry_action,
       'match_action' : match_action,
+      # statistics for states
       'class_keys' : class_keys,
       'distinct_keys' : distinct_keys,
       'ranges' : ranges,
       # record of which entry points will be needed
       'entry_points' : {
         'state_entry' : True,
         'after_entry_code' : False,
         # 'before_match' : False,
         # 'before_deferred' : False,
       }
@@ skipping 36 matching lines @@
         elif no_switch:
           i.append(r)
         else:
           s.append(r[1])
       if i:
         if_transitions.append((i, node_id))
       if s:
         switch_transitions.append((s, node_id))
       if d:
         deferred_transitions.append((d, node_id))
-    state['transitions'] = if_transitions
+    state['if_transitions'] = if_transitions
     state['switch_transitions'] = switch_transitions
     state['deferred_transitions'] = deferred_transitions
     return split_count + (0 if no_switch else 1)
 
   __call_map = {
     'non_primary_whitespace' : 'IsWhiteSpaceNotLineTerminator',
     'non_primary_letter' : 'IsLetter',
     'non_primary_identifier_part_not_letter' : 'IsIdentifierPartNotLetter',
     'non_primary_line_terminator' : 'IsLineTerminator',
   }
@@ skipping 160 matching lines @@
     char_type = char_types[encoding.name()]
 
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
       encoding = encoding.name(),
       char_type = char_type,
       upper_bound = encoding.primary_range()[1])