Experimental parser: add push_token syntax

tools/lexer_generator/rule_lexer.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 15 matching lines @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import ply.lex as lex
 
 class RuleLexer:
 
   tokens = (
     'DEFAULT',
     'DEFAULT_ACTION',
     'CATCH_ALL',
+    'PUSH_TOKEN',
 
     'IDENTIFIER',
     'STRING',
     'REGEX',
     'CHARACTER_CLASS_REGEX',
 
     'PLUS',
     'QUESTION_MARK',
     'EQUALS',
     'OR',
@@ skipping 17 matching lines @@
   )
 
   t_ignore = " \t\n\r"
   t_code_ignore = ""
 
   def t_COMMENT(self, t):
     r'\#.*[\n\r]+'
     pass
 
   __special_identifiers = set(map(lambda s: s.lower(),
-    ['DEFAULT', 'DEFAULT_ACTION', 'CATCH_ALL',]))
+    ['DEFAULT', 'DEFAULT_ACTION', 'CATCH_ALL', 'PUSH_TOKEN']))
 
   def t_IDENTIFIER(self, t):
     r'[a-zA-Z][a-zA-Z0-9_]*'
     if t.value in self.__special_identifiers:
       t.type = t.value.upper()
     return t
 
   t_STRING = r'"((\\("|\w|\\))|[^\\"])+"'
   t_REGEX = r'/[^\/]+/'
   t_CHARACTER_CLASS_REGEX = r'\[([^\]]|\\\])+\]'
@@ skipping 32 matching lines @@
       t.type = 'CODE_FRAGMENT'
     else:
       self.lexer.pop_state()
     return t
 
   def t_ANY_error(self, t):
     raise Exception("Illegal character '%s'" % t.value[0])
 
   def build(self, **kwargs):
     self.lexer = lex.lex(module=self, **kwargs)