Experimental parser: KeyEncoding class added

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 17 matching lines @@
 import os
 import sys
 import jinja2
 from dfa import Dfa
 from transition_keys import TransitionKey
 
 class CodeGenerator:
 
   def __init__(self,
                rule_processor,
-               encoding = 'latin1',
                minimize_default = True,
                inline = True,
                switching = True,
                debug_print = False,
                log = False):
     if minimize_default:
       dfa = rule_processor.default_automata().minimal_dfa()
     else:
       dfa = rule_processor.default_automata().dfa()
     self.__dfa = dfa
     self.__default_action = rule_processor.default_action
     self.__debug_print = debug_print
     self.__start_node_number = self.__dfa.start_state().node_number()
     self.__log = log
     self.__inline = inline
     self.__switching = switching
-    self.__encoding = encoding
 
   def __state_cmp(self, left, right):
     if left['original_node_number'] == self.__start_node_number:
       return -1
     if right['original_node_number'] == self.__start_node_number:
       return 1
     c = cmp(len(left['disjoint_keys']), len(right['disjoint_keys']))
     if c:
       return c
     c = cmp(left['disjoint_keys'], right['disjoint_keys'])
@@ skipping 23 matching lines @@
         return cmp(left[1], right[1])
       assert right[0] == 'CLASS'
       return -1
     assert left[0] == 'CLASS'
     if right[0] == 'LATIN_1':
       return 1
     # TODO store numeric values and cmp
     return cmp(left[1], right[1])
 
   @staticmethod
-  def __transform_state(state):
+  def __transform_state(encoding, state):
     # action data
     action = state.action()
     entry_action = None if not action else action.entry_action()
     match_action = None if not action else action.match_action()
     # generate ordered transitions
     transitions = map(lambda (k, v) : (k, v.node_number()),
                       state.transitions().items())
     def cmp(left, right):
       return TransitionKey.canonical_compare(left[0], right[0])
     transitions = sorted(transitions, cmp)
     # map transition keys to disjoint ranges
     disjoint_keys = {'value' : []}
     def f((key, state)):
-      ranges = list(key.range_iter())
+      ranges = list(key.range_iter(encoding))
       disjoint_keys['value'] += ranges
       return (ranges, state)
     transitions = map(f, transitions)
     disjoint_keys = sorted(disjoint_keys['value'], CodeGenerator.__range_cmp)
     # dictionary object representing state
     (class_keys, distinct_keys, ranges) = (0, 0, 0)
     for (t, r) in disjoint_keys:
       if t == 'CLASS':
         class_keys += 1
       elif t == 'LATIN_1':
@@ skipping 72 matching lines @@
       if d:
         deferred_transitions.append((d, node_id))
     state['transitions'] = if_transitions
     state['switch_transitions'] = switch_transitions
     state['deferred_transitions'] = deferred_transitions
     return split_count + (0 if no_switch else 1)
 
   def __canonicalize_traversal(self):
     dfa_states = []
     self.__dfa.visit_all_states(lambda state, acc: dfa_states.append(state))
-    dfa_states = map(CodeGenerator.__transform_state, dfa_states)
+    encoding = self.__dfa.encoding()
+    f = lambda state : CodeGenerator.__transform_state(encoding, state)
+    dfa_states = map(f, dfa_states)
     id_map = {x['original_node_number'] : x for x in dfa_states}
     CodeGenerator.__compute_depths(self.__start_node_number, 1, id_map)
     dfa_states = sorted(dfa_states, cmp=self.__state_cmp)
     # remap all node numbers
     for i, state in enumerate(dfa_states):
       state['node_number'] = i
     def f((key, original_node_number)):
       return (key, id_map[original_node_number]['node_number'])
     for state in dfa_states:
       state['transitions'] = map(f, state['transitions'])
@@ skipping 22 matching lines @@
     default_action = self.__default_action
     assert(default_action and default_action.match_action())
     default_action = default_action.match_action()
 
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     template_env = jinja2.Environment(
       loader = jinja2.PackageLoader('lexer_generator', '.'),
       undefined = jinja2.StrictUndefined)
     template = template_env.get_template('code_generator.jinja')
 
-    if self.__encoding == 'latin1':
+    encoding = self.__dfa.encoding().name()
+    if encoding == 'latin1':
       char_type = 'uint8_t'
-    elif self.__encoding == 'utf16':
+    elif encoding == 'utf16':
       char_type = 'uint16_t'
     else:
       raise Exception('Unsupported encoding %s' % encoding)
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
-      encoding = self.__encoding,
+      encoding = encoding,
       char_type = char_type)