Experimental parser: add push_token syntax

tools/lexer_generator/rule_parser.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 80 matching lines @@
         'catch_all' : None,
         'regex' : []
       }
     p[0] = state.current_state
 
   def p_transition_rules(self, p):
     '''transition_rules : transition_rule transition_rules
                         | empty'''
 
   def p_transition_rule(self, p):
-    '''transition_rule : composite_regex code action
+    '''transition_rule : composite_regex code_or_token action
                        | composite_regex empty action
-                       | composite_regex code empty
-                       | DEFAULT_ACTION code empty
+                       | composite_regex code_or_token empty
+                       | DEFAULT_ACTION code_or_token empty
                        | CATCH_ALL empty action'''
     transition = p[3]
     if transition and not transition in self.__keyword_transitions:
       assert not transition == 'default'
       self.__state.transitions.add(transition)
     RuleParser.__rule_precedence_counter += 1
     rules = self.__state.rules[self.__state.current_state]
     code = p[2]
     if p[1] == 'default_action':
       assert not rules['default_action']
       rules['default_action'] = code
     elif p[1] == 'catch_all':
       assert not rules['catch_all']
       rules['catch_all'] = transition
     else:
       rule = (p[1], RuleParser.__rule_precedence_counter, code, transition)
       rules['regex'].append(rule)
 
+  def p_code_or_token(self, p):
+    '''code_or_token : code
+                     | push_token'''
+    p[0] = p[1]
+
+  def p_push_token(self, p):
+    'push_token : PUSH_TOKEN LEFT_PARENTHESIS IDENTIFIER RIGHT_PARENTHESIS'
+    p[0] = (p[1], p[3])
+
   def p_action(self, p):
     'action : ACTION_OPEN IDENTIFIER ACTION_CLOSE'
     p[0] = p[2]
 
   def p_composite_regex(self, p):
     '''composite_regex : regex_parts OR regex_parts
                        | regex_parts'''
     if len(p) == 2:
       p[0] = p[1]
     else:
@@ skipping 38 matching lines @@
 
   def p_modifier(self, p):
     '''modifier : PLUS
                 | QUESTION_MARK
                 | STAR
                 | empty'''
     p[0] = p[1]
 
   def p_code(self, p):
     'code : LEFT_BRACKET code_fragments RIGHT_BRACKET'
-    p[0] = p[2].strip()
+    p[0] = ('code', p[2].strip())
 
   def p_code_fragments(self, p):
     '''code_fragments : CODE_FRAGMENT code_fragments
                       | empty'''
     p[0] = p[1]
     if len(p) == 3 and p[2]:
       p[0] = p[1] + p[2]
 
   def p_empty(self, p):
     'empty :'
@@ skipping 70 matching lines @@
     rule_map = {}
     builder = NfaBuilder()
     builder.set_character_classes(parser_state.character_classes)
     assert 'default' in parser_state.rules
     def process(k, v):
       graphs = []
       continues = 0
       for (graph, precedence, code, transition) in v['regex']:
         default_code = v['default_action']
         if code or default_code:
-          action = Action('code', code if code else default_code, precedence)
+          (code_type, code_value) = code if code else default_code
+          action = Action(code_type, code_value, precedence)
           graph = NfaBuilder.add_action(graph, action)
         if not transition or transition == 'break':
           pass
         elif transition == 'continue':
           assert not k == 'default'
           continues += 1
           graph = NfaBuilder.add_continue(graph)
         elif (transition == 'terminate' or
               transition == 'terminate_illegal'):
           assert not code
@@ skipping 13 matching lines @@
       rule_map[k] = graph
     # process first the subgraphs, then the default graph
     for k, v in parser_state.rules.items():
       if k == 'default': continue
       process(k, v)
     process('default', parser_state.rules['default'])
     # build the automata
     for rule_name, graph in rule_map.items():
       self.__automata[rule_name] = RuleProcessor.Automata(builder, graph)
     self.default_action = parser_state.rules['default']['default_action']