Experimental parser: KeyEncoding class added

tools/lexer_generator/nfa_builder.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 13 matching lines @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from types import TupleType
 from inspect import getmembers
 from nfa import *
 
 class NfaBuilder(object):
 
-  def __init__(self):
+  def __init__(self, encoding, character_classes = {}):
     self.__node_number = 0
     self.__operation_map = {}
     self.__members = getmembers(self)
-    self.__character_classes = {}
+    self.__encoding = encoding
+    self.__character_classes = character_classes
     self.__states = []
 
-  def set_character_classes(self, classes):
-    self.__character_classes = classes
-
   def __new_state(self):
     self.__node_number += 1
     return NfaState()
 
   def __or(self, graph):
     start = self.__new_state()
     ends = []
     for x in [self.__process(graph[1]), self.__process(graph[2])]:
       start.add_epsilon_transition(x[0])
       ends += x[1]
@@ skipping 46 matching lines @@
     (left, right) = (self.__process(graph[1]), self.__process(graph[2]))
     self.__patch_ends(left[1], right[0])
     return (left[0], right[1])
 
   def __key_state(self, key):
     state =  self.__new_state()
     state.add_unclosed_transition(key)
     return (state, [state])
 
   def __literal(self, graph):
-    return self.__key_state(TransitionKey.single_char(graph[1]))
+    return self.__key_state(
+      TransitionKey.single_char(self.__encoding, graph[1]))
 
   def __class(self, graph):
-    return self.__key_state(
-      TransitionKey.character_class(graph, self.__character_classes))
+    return self.__key_state(TransitionKey.character_class(
+      self.__encoding, graph, self.__character_classes))
 
   def __not_class(self, graph):
-    return self.__key_state(
-      TransitionKey.character_class(graph, self.__character_classes))
+    return self.__key_state(TransitionKey.character_class(
+      self.__encoding, graph, self.__character_classes))
 
   def __any(self, graph):
-    return self.__key_state(TransitionKey.any())
+    return self.__key_state(TransitionKey.any(self.__encoding))
 
   def __epsilon(self, graph):
     start = self.__new_state()
     end = self.__new_state()
     start.close(end)
     return (start, [end])
 
   def __action(self, graph):
     (start, ends) = self.__process(graph[1])
     action = graph[2]
@@ skipping 77 matching lines @@
         return closure
       is_epsilon = lambda k: k == TransitionKey.epsilon()
       state_iter = lambda node : node.state_iter(key_filter = is_epsilon)
       edge = set(state_iter(node))
       closure = Automaton.visit_states(
           edge, inner, state_iter=state_iter, visit_state=set())
       node.set_epsilon_closure(closure)
     Automaton.visit_states(set([start_state]), outer)
 
   @staticmethod
-  def __replace_catch_all(state):
+  def __replace_catch_all(encoding, state):
     catch_all = TransitionKey.unique('catch_all')
     transitions = state.transitions()
     if not catch_all in transitions:
       return
     f = lambda acc, state: acc | set(state.epsilon_closure_iter())
     reachable_states = reduce(f, transitions[catch_all], set())
     f = lambda acc, state: acc | set(state.transitions().keys())
     keys = reduce(f, reachable_states, set())
     keys.discard(TransitionKey.epsilon())
     keys.discard(catch_all)
-    inverse_key = TransitionKey.inverse_key(keys)
+    inverse_key = TransitionKey.inverse_key(encoding, keys)
     if inverse_key:
       transitions[inverse_key] = transitions[catch_all]
     del transitions[catch_all]
 
   def nfa(self, graph):
     (start, end, nodes_created) = self.__nfa(graph)
     end.close(None)
     self.__compute_epsilon_closures(start)
-    f = lambda node, state: self.__replace_catch_all(node)
+    f = lambda node, state: self.__replace_catch_all(self.__encoding, node)
     Automaton.visit_states(set([start]), f)
-    return Nfa(start, end, nodes_created)
+    return Nfa(self.__encoding, start, end, nodes_created)
 
   @staticmethod
   def add_action(graph, action):
     return ('ACTION', graph, action)
 
   @staticmethod
   def add_continue(graph):
     return ('CONTINUE', graph)
 
   @staticmethod
@@ skipping 18 matching lines @@
 
   __modifer_map = {
     '+': 'ONE_OR_MORE',
     '?': 'ZERO_OR_ONE',
     '*': 'ZERO_OR_MORE',
   }
 
   @staticmethod
   def apply_modifier(modifier, graph):
     return (NfaBuilder.__modifer_map[modifier], graph)