Experimental parser: KeyEncoding class added

tools/lexer_generator/dfa.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 60 matching lines @@
   def transition_state_for_key(self, value):
     matches = self.__matches(lambda k, v : k.is_superset_of_key(v), value)
     if not matches:
       return None
     # Since this is a dfa, we should have at most one such state. Return it.
     assert len(matches) == 1
     return matches[0]
 
 class Dfa(Automaton):
 
-  def __init__(self, start_name, mapping):
-    super(Dfa, self).__init__()
+  def __init__(self, encoding, start_name, mapping):
+    super(Dfa, self).__init__(encoding)
     self.__terminal_set = set()
     name_map = {}
     for name, node_data in mapping.items():
       node = DfaState(name, node_data['action'])
       name_map[name] = node
       if node_data['terminal']:
         self.__terminal_set.add(node)
     for name, node_data in mapping.items():
       node = name_map[name]
       inversion = {}
       for key, state in node_data['transitions'].items():
         if not state in inversion:
           inversion[state] = []
         inversion[state].append(key)
       for state, keys in inversion.items():
-        merged_key = TransitionKey.merged_key(keys)
+        merged_key = TransitionKey.merged_key(encoding, keys)
         node.add_transition(merged_key, name_map[state])
     self.__start = name_map[start_name]
     self.__node_count = len(mapping)
     self.__verify()
 
   def __verify(self):
     assert self.__terminal_set
     state_count = self.visit_all_states(lambda state, count: count + 1, 0)
     assert self.__node_count == state_count
 
@@ skipping 107 matching lines @@
     # The alphabet defines the TransitionKeys we need to check when dedicing if
     # two states of the dfa can be in the same partition. See the doc string of
     # TransitionKey.disjoint_keys.
 
     # For example, if the TransitionKeys are {[a-d], [c-h]}, the alphabet is
     # {[a-b], [c-d], [e-h]}. If state S1 has a transition to S2 with
     # TransitionKey [a-d], and state S3 has a transition to S4 with
     # TransitionKey [c-h], S1 and S3 cannot be in the same partition. This will
     # become clear when we check the transition for TransitionKey [c-d] (S1 has
     # a transition to S2, S3 has a transition to S4).
-    self.__alphabet = list(TransitionKey.disjoint_keys(chain(*all_keys)))
+    encoding = self.__dfa.encoding()
+    self.__alphabet = list(
+        TransitionKey.disjoint_keys(encoding, chain(*all_keys)))
 
     # For each state and each TransitionKey in the alphabet, find out which
     # transition we take from the state with the TransitionKey.
     transitions = {}
     for state_id, state in id_to_state.items():
       def f((key_id, key)):
         transition_state = state.transition_state_for_key(key)
         if transition_state:
           return state_to_id[transition_state]
         return None
@@ skipping 144 matching lines @@
             working_set.add(difference)
           elif len(intersection) <= len(difference):
             working_set.add(intersection)
           else:
             working_set.add(difference)
         if old_partitions:
           partitions -= old_partitions
         if new_partitions:
           partitions |= new_partitions
     (start_name, mapping) = self.__create_states_from_partitions(partitions)
-    return Dfa(start_name, mapping)
+    return Dfa(self.__dfa.encoding(), start_name, mapping)