Experimental parser: revert read_cursor

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 36 matching lines @@
       dfa = rule_processor.default_automata().dfa()
     self.__dfa = dfa
     self.__default_action = rule_processor.default_action
     self.__debug_print = debug_print
     self.__start_node_number = self.__dfa.start_state().node_number()
     self.__log = log
     self.__inline = inline
     self.__switching = switching
     self.__jump_table = []
 
-  __jump_labels = ['state_entry', 'after_entry_code', 'before_match']
+  __jump_labels = ['state_entry', 'after_entry_code']
 
   @staticmethod
   def __transform_state(encoding, state):
     # action data
     action = state.action()
     entry_action = None if not action else action.entry_action()
     match_action = None if not action else action.match_action()
     # generate ordered transitions
     transitions = map(lambda (k, v) : (k, v.node_number()),
                       state.transitions().items())
     def cmp(left, right):
       return TransitionKey.canonical_compare(left[0], right[0])
     transitions = sorted(transitions, cmp)
     # map transition keys to disjoint ranges and collect stats
     disjoint_keys = []
     unique_transitions = {}
     old_transitions = transitions
     transitions = []
     (class_keys, distinct_keys, ranges) = (0, 0, 0)
+    zero_transition = None
     for key, transition_id in old_transitions:
       keys = []
       for (t, r) in key.range_iter(encoding):
         if t == 'CLASS':
           class_keys += 1
           keys.append((t, r))
         elif t == 'PRIMARY_RANGE':
           distinct_keys += r[1] - r[0] + 1
           ranges += 1
+          # split 0 out of range, don't bother updating stats
+          assert r[0] >= 0
+          if r[0] == 0:
+            assert zero_transition == None
+            zero_transition = transition_id
+            if r[0] == r[1]:
+              continue
+            r = (r[0] + 1, r[1])
           keys.append((t, r))
         elif t == 'UNIQUE':
           assert r == 'no_match' or r == 'eos'
           assert r not in unique_transitions
           unique_transitions[r] = transition_id
         else:
           raise Exception()
       if keys:
         transitions.append((keys, transition_id))
+    # delay zero transition until last
+    if zero_transition != None:
+      transitions.append(([('PRIMARY_RANGE', (0, 0))], transition_id))
     return {
       'node_number' : None,
       'original_node_number' : state.node_number(),
       'transitions' : transitions,
       # flags for code generator
       'can_elide_read' : len(transitions) == 0,
       'is_eos_handler' : False,
-      'inline' : None,
+      'inline' : False,
       'must_not_inline' : False,
       # transitions for code generator
       'if_transitions' : [],
       'switch_transitions' : [],
       'deferred_transitions' : [],
       'unique_transitions' : unique_transitions,
       # state actions
       'entry_action' : entry_action,
       'match_action' : match_action,
       # statistics for states
       'class_keys' : class_keys,
       'distinct_keys' : distinct_keys,
       'ranges' : ranges,
       # record of which entry points will be needed
       'entry_points' : {k : False for k in CodeGenerator.__jump_labels}
     }
 
   def __register_jump(self, node_id, label):
     if label != 'inline':
       assert label in CodeGenerator.__jump_labels
       state = self.__dfa_states[node_id]['entry_points'][label] = True
     self.__jump_table.append((node_id, label))
     return len(self.__jump_table) - 1
 
   def __set_inline(self, count, state):
-    assert state['inline'] == None
     inline = False
     if state['must_not_inline']:
       inline = False
     # inline terminal states
     elif not state['transitions']:
       inline = True
     # inline next to terminal states with 1 or 2 transitions
     elif state['distinct_keys'] < 3 and state['class_keys'] == 0:
       inline = True
       # ensure state terminates in 1 step
@@ skipping 14 matching lines @@
     switch_transitions = []
     deferred_transitions = []
     for (ranges, node_id) in state['transitions']:
       i = []
       s = []
       d = []
       for r in ranges:
         # all class checks will be deferred to after all other checks
         if r[0] == 'CLASS':
           d.append(r)
-        elif no_switch:
+        # zero must assigned to an if check because of eos check
+        elif no_switch or (r[1][0] == 0):
           i.append(r)
         else:
           s.append(r[1])
       if i:
         if_transitions.append((i, node_id))
       if s:
         switch_transitions.append((s, node_id))
       if d:
         deferred_transitions.append((d, node_id))
     state['if_transitions'] = if_transitions
@@ skipping 173 matching lines @@
         else:
           assert not target_id in inline_mapping
         return (key, self.__register_jump(target_id, jump_type))
       for name in transition_names:
         state[name] = map(generate_jump, state[name])
       if 'eos' in state['unique_transitions']:
         # eos state is not inlined
         eos_state_id = state['unique_transitions']['eos']
         jump = self.__register_jump(eos_state_id, 'state_entry')
         state['unique_transitions']['eos'] = jump
-      elif state['transitions']:
-        jump = self.__register_jump(state_id, 'before_match')
-        state['jump_before_match'] = jump
       # now rewrite all nodes created
       nodes_created = len(inline_mapping) - len(inline_mapping_in)
       assert len(self.__dfa_states) == (
         end_offset + total_nodes_created + nodes_created)
       if nodes_created == 0:
         continue
       created = self.__rewrite_transitions_to_jumps(
         end_offset + total_nodes_created, nodes_created, inline_mapping)
       total_nodes_created += nodes_created + created
     return total_nodes_created
@@ skipping 39 matching lines @@
 
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
       jump_table = self.__jump_table,
       encoding = encoding.name(),
       char_type = char_type,
       upper_bound = encoding.primary_range()[1])