Experimental parser: KeyEncoding class added

tools/lexer_generator/rule_parser.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 13 matching lines @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import ply.yacc as yacc
 from automaton import Action
 from rule_lexer import RuleLexer
 from regex_parser import RegexParser
 from nfa_builder import NfaBuilder
 from dfa import Dfa
-from transition_keys import TransitionKey
+from transition_keys import TransitionKey, KeyEncoding
 
 class RuleParserState:
 
-  def __init__(self):
+  def __init__(self, encoding):
     self.aliases = {}
     self.character_classes = {}
     self.current_state = None
     self.rules = {}
     self.transitions = set()
+    self.encoding = encoding
 
   def parse(self, string):
     return RuleParser.parse(string, self)
 
 class RuleParser:
 
   tokens = RuleLexer.tokens
   __rule_precedence_counter = 0
   __keyword_transitions = set(['continue'])
 
   def __init__(self):
     self.__state = None
 
   def p_statements(self, p):
     'statements : aliases rules'
 
   def p_aliases(self, p):
     '''aliases : alias_rule aliases
                | empty'''
 
   def p_alias_rule(self, p):
     'alias_rule : IDENTIFIER EQUALS composite_regex SEMICOLON'
     state = self.__state
     assert not p[1] in state.aliases
     graph = p[3]
     state.aliases[p[1]] = graph
     if graph[0] == 'CLASS' or graph[0] == 'NOT_CLASS':
       classes = state.character_classes
       assert not p[1] in classes
-      classes[p[1]] = TransitionKey.character_class(graph, classes)
+      encoding = state.encoding
+      classes[p[1]] = TransitionKey.character_class(encoding, graph, classes)
 
   def p_rules(self, p):
     '''rules : state_change transition_rules rules
              | empty'''
 
   def p_state_change(self, p):
     'state_change : GRAPH_OPEN IDENTIFIER GRAPH_CLOSE'
     state = self.__state
     state.current_state = p[2]
     assert state.current_state
@@ skipping 169 matching lines @@
     parser.__state = None
     assert parser_state.transitions <= set(parser_state.rules.keys())
 
 class RuleProcessor(object):
 
   def __init__(self, parser_state):
     self.__automata = {}
     self.__process_parser_state(parser_state)
 
   @staticmethod
-  def parse(string):
-    parser_state = RuleParserState()
+  def parse(string, encoding_name):
+    parser_state = RuleParserState(KeyEncoding.get(encoding_name))
     RuleParser.parse(string, parser_state)
     return RuleProcessor(parser_state)
 
   def automata_iter(self):
     return iter(self.__automata.items())
 
   def default_automata(self):
     return self.__automata['default']
 
   class Automata(object):
 
     def __init__(self, builder, graph):
       self.__builder = builder
       self.__graph = graph
       self.__nfa = None
       self.__dfa = None
       self.__minimial_dfa = None
 
     def nfa(self):
       if not self.__nfa:
         self.__nfa = self.__builder.nfa(self.__graph)
       return self.__nfa
 
     def dfa(self):
       if not self.__dfa:
         (start, dfa_nodes) = self.nfa().compute_dfa()
-        self.__dfa = Dfa(start, dfa_nodes)
+        self.__dfa = Dfa(self.nfa().encoding(), start, dfa_nodes)
       return self.__dfa
 
     def minimal_dfa(self):
       if not self.__minimial_dfa:
         self.__minimial_dfa = self.dfa().minimize()
       return self.__minimial_dfa
 
   def __process_parser_state(self, parser_state):
     rule_map = {}
-    builder = NfaBuilder()
-    builder.set_character_classes(parser_state.character_classes)
+    builder = NfaBuilder(parser_state.encoding, parser_state.character_classes)
     assert 'default' in parser_state.rules
     def process(subgraph, v):
       graphs = []
       continues = 0
       for graph, precedence, action in v['regex']:
         (entry_action, match_action, transition) = action
         if entry_action or match_action:
           action = Action(entry_action, match_action, precedence)
           graph = NfaBuilder.add_action(graph, action)
         if not transition:
@@ skipping 19 matching lines @@
     for k, v in parser_state.rules.items():
       if k == 'default': continue
       process(k, v)
     process('default', parser_state.rules['default'])
     # build the automata
     for rule_name, graph in rule_map.items():
       self.__automata[rule_name] = RuleProcessor.Automata(builder, graph)
     # process default_action
     default_action = parser_state.rules['default']['default_action']
     self.default_action = Action(None, default_action[1]) if default_action else None