Experimental parser: cleanup code requiring calls to the unicode cache

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 111 matching lines @@
         distinct_keys += r[1] - r[0] + 1
         ranges += 1
       else:
         raise Exception()
     return {
       'node_number' : state.node_number(),
       'original_node_number' : state.node_number(),
       'transitions' : transitions,
       'switch_transitions' : [],
       'deferred_transitions' : [],
+      'long_char_transitions' : [],
       'disjoint_keys' : disjoint_keys,
       'inline' : None,
       'depth' : None,
       'action' : action,
       'entry_action' : entry_action,
       'match_action' : match_action,
       'class_keys' : class_keys,
       'distinct_keys' : distinct_keys,
       'ranges' : ranges
     }
@@ skipping 49 matching lines @@
         if_transitions.append((i, node_id))
       if s:
         switch_transitions.append((s, node_id))
       if d:
         deferred_transitions.append((d, node_id))
     state['transitions'] = if_transitions
     state['switch_transitions'] = switch_transitions
     state['deferred_transitions'] = deferred_transitions
     return split_count + (0 if no_switch else 1)
 
+  __call_map = {
+    'non_primary_whitespace' : 'IsWhiteSpace',
+    'non_primary_letter' : 'IsLetter',
+    'non_primary_identifier_part_not_letter' : 'IsIdentifierPartNotLetter',
+    'non_primary_line_terminator' : 'IsLineTerminator',
+  }
+
+  def __rewrite_deferred_transitions(self, state):
+    assert not state['long_char_transitions']
+    transitions = state['deferred_transitions']
+    if not transitions:
+      return
+    encoding = self.__dfa.encoding()
+    bom = 'byte_order_mark'
+    catch_all = 'non_primary_everything_else'
+    all_classes = set(encoding.class_name_iter())
+    fast_classes = set(['eos', 'zero'])
+    call_classes = all_classes - fast_classes - set([bom, catch_all])
+    def remap_transition(class_name):
+      if class_name in call_classes:
+        return ('LONG_CHAR_CLASS', 'call', self.__call_map[class_name])
+      if class_name == bom:
+        return ('LONG_CHAR_CLASS', class_name)
+      raise Exception(class_name)
+    fast_transitions = []
+    long_class_transitions = []
+    long_class_map = {}
+    catchall_transition = None
+    # loop through and remove catch_all_transitions
+    for (classes, transition_node_id) in transitions:
+      ft = []
+      lct = []
+      has_catch_all = False
+      for (class_type, class_name) in classes:
+        if class_name in fast_classes:
+          ft.append((class_type, class_name))
+        else:
+          assert not class_name in long_class_map
+          long_class_map[class_name] = transition_node_id
+          if class_name == catch_all:
+            assert not has_catch_all
+            assert catchall_transition == None
+            has_catch_all = True
+          else:
+            lct.append(remap_transition(class_name))
+      if ft:
+        fast_transitions.append((ft, transition_node_id))
+      if has_catch_all:
+        catchall_transition = (lct, transition_node_id)
+      elif lct:
+        long_class_transitions.append((lct, transition_node_id))
+    # all transitions are fast
+    if not long_class_map:
+      return
+    if catchall_transition:
+      catchall_transitions = all_classes - fast_classes
+      for class_name in long_class_map.iterkeys():
+        catchall_transitions.remove(class_name)
+      assert not catchall_transitions, "class inversion not unimplemented"
+    # split deferred transitions
+    state['deferred_transitions'] = fast_transitions
+    if catchall_transition:
+      catchall_transition = [
+        ([('LONG_CHAR_CLASS', 'catch_all')], catchall_transition[1])]
+    else:
+      catchall_transition = []
+    state['long_char_transitions'] = (long_class_transitions +
+                                      catchall_transition) # must be last
+
   def __canonicalize_traversal(self):
     dfa_states = []
     self.__dfa.visit_all_states(lambda state, acc: dfa_states.append(state))
     encoding = self.__dfa.encoding()
     f = lambda state : CodeGenerator.__transform_state(encoding, state)
     dfa_states = map(f, dfa_states)
     id_map = {x['original_node_number'] : x for x in dfa_states}
     CodeGenerator.__compute_depths(self.__start_node_number, 1, id_map)
     dfa_states = sorted(dfa_states, cmp=self.__state_cmp)
     # remap all node numbers
@@ skipping 11 matching lines @@
     if self.__inline:
       inlined = reduce(self.__set_inline, dfa_states, 0)
       if self.__log:
         print "%s states inlined" % inlined
     elif self.__log:
       print "no inlining"
     # split transitions
     switched = reduce(self.__split_transitions, dfa_states, 0)
     if self.__log:
       print "%s states use switch (instead of if)" % switched
+    # rewrite deferred transitions
+    for state in dfa_states:
+      self.__rewrite_deferred_transitions(state)
 
   def process(self):
 
     self.__canonicalize_traversal()
 
     dfa_states = self.__dfa_states
 
     default_action = self.__default_action
     assert(default_action and default_action.match_action())
     default_action = default_action.match_action()
 
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     template_env = jinja2.Environment(
       loader = jinja2.PackageLoader('lexer_generator', '.'),
       undefined = jinja2.StrictUndefined)
     template = template_env.get_template('code_generator.jinja')
 
     encoding = self.__dfa.encoding()
     char_types = {'latin1': 'uint8_t', 'utf16': 'uint16_t', 'utf8': 'int8_t'}
     char_type = char_types[encoding.name()]
 
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
       encoding = encoding.name(),
       char_type = char_type,
       upper_bound = encoding.primary_range()[1])