Experimental parser: better actions

tools/lexer_generator/generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 72 matching lines @@
       loads.append(load_template % ("dfa [%s]" % name, dfa_i))
       body = "\n".join(scripts) + (load_outer_template % "\n".join(loads))
     return file_template % body
 
   def process_rules(self, parser_state):
     rule_map = {}
     builder = NfaBuilder()
     builder.set_character_classes(parser_state.character_classes)
     assert 'default' in parser_state.rules
     def process(k, v):
-      assert 'default' in v
       graphs = []
-      for (graph, action) in v['regex']:
-        (precedence, code, transition) = action
-        if code:
-          graph = NfaBuilder.add_action(graph, (precedence, code, None))
+      for (graph, (precedence, code, transition)) in v['regex']:
+        default_code = v['default_action']
+        action = code if code else default_code
+        if action:
+          graph = NfaBuilder.add_action(graph, (precedence, action))
         if transition == 'continue':
-          if not v['default'][1][2] == 'continue':
-            graph = NfaBuilder.add_continue(graph)
-          else:
-            pass # TODO null key
-        elif (transition == 'break' or
-              transition == 'terminate' or
+          assert not k == 'default'
+          graph = NfaBuilder.add_continue(graph)
+        elif transition == 'break':
+          pass
+        elif (transition == 'terminate' or
               transition == 'terminate_illegal'):
-          graph = NfaBuilder.add_action(graph, (10000, transition, None))
+          assert not code
+          graph = NfaBuilder.add_action(graph, (-1, transition))
         else:
           assert k == 'default'
-          graph = NfaBuilder.join_subgraph(graph, transition, rule_map[transition])
+          subgraph_modifier = '*' if code else None
+          graph = NfaBuilder.join_subgraph(
+            graph, transition, rule_map[transition], subgraph_modifier)
         graphs.append(graph)
       graph = NfaBuilder.or_graphs(graphs)
-      # merge default action
-      (precedence, code, transition) = v['default'][1]
-      assert transition == 'continue' or transition == 'break'
-      if transition == 'continue':
-        assert k != 'default'
-        graph = NfaBuilder.add_incoming_action(graph, (10000, k, None))
-      if code:
-        graph = NfaBuilder.add_incoming_action(graph, (precedence, code, None))
       rule_map[k] = graph
+    # process first the subgraphs, then the default graph
     for k, v in parser_state.rules.items():
       if k == 'default': continue
       process(k, v)
     process('default', parser_state.rules['default'])
+    # build the automata
     for rule_name, graph in rule_map.items():
       nfa = builder.nfa(graph)
       (start, dfa_nodes) = nfa.compute_dfa()
       dfa = Dfa(start, dfa_nodes)
       self.__automata[rule_name] = (nfa, dfa)
 
   # Lexes strings with the help of DFAs procuded by the grammar. For sanity
   # checking the automata.
   def lex(self, string):
-    (nfa, dfa) = self.__automata['default'] # FIXME
-
-    action_stream = []
-    terminate_seen = False
-    offset = 0
-    while not terminate_seen and string:
-      result = list(dfa.lex(string))
-      last_position = 0
-      for (action, position) in result:
-        action_stream.append((action[1], action[2], last_position + offset, position + 1 + offset, string[last_position:(position + 1)]))
-        last_position = position
-        if action[2] == 'terminate':
-          terminate_seen = True
-      string = string[(last_position + 1):]
-      offset += last_position
-    return action_stream
+    (nfa, dfa) = self.__automata['default']
+    return dfa.lex(string)
 
 if __name__ == '__main__':
 
   parser = argparse.ArgumentParser()
   parser.add_argument('--html')
   parser.add_argument('--re', default='src/lexer/lexer_py.re')
   parser.add_argument('--input')
   args = parser.parse_args()
 
   re_file = args.re
   parser_state = RuleParserState()
   print "parsing %s" % re_file
   with open(re_file, 'r') as f:
     generator = Generator(f.read())
 
   html_file = args.html
   if html_file:
     html = generator.generate_html()
     with open(args.html, 'w') as f:
       f.write(html)
       print "wrote html to %s" % html_file
 
   input_file = args.input
   if input_file:
     with open(input_file, 'r') as f:
       input_text = f.read() + '\0'
     for t in generator.lex(input_text):
       print t