Experimental parser: remove inline code from lexer rules

tools/lexer_generator/rule_lexer.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 33 matching lines @@
     'OR',
     'STAR',
     'LEFT_PARENTHESIS',
     'RIGHT_PARENTHESIS',
     'GRAPH_OPEN',
     'GRAPH_CLOSE',
     'SEMICOLON',
     'ACTION_OPEN',
     'ACTION_CLOSE',
 
-    'LEFT_BRACKET',
-    'RIGHT_BRACKET',
-
-    'CODE_FRAGMENT',
-
     'COMMA',
   )
 
-  states = (
-    ('code','exclusive'),
-  )
-
   t_ignore = " \t\n\r"
-  t_code_ignore = ""
 
   def t_COMMENT(self, t):
     r'\#.*[\n\r]+'
     pass
 
   __special_identifiers = set(map(lambda s: s.lower(),
     ['DEFAULT_ACTION', 'CATCH_ALL']))
 
   def t_IDENTIFIER(self, t):
     r'[a-zA-Z0-9_]+'
@@ skipping 18 matching lines @@
   t_ACTION_OPEN = '<'
   t_ACTION_CLOSE = '>'
   t_COMMA = ','
 
   def t_LEFT_BRACKET(self, t):
     r'{'
     self.lexer.push_state('code')
     self.nesting = 1
     return t
 
-  t_code_CODE_FRAGMENT = r'[^{}]+'
-
-  def t_code_LEFT_BRACKET(self, t):
-    r'{'
-    self.nesting += 1
-    t.type = 'CODE_FRAGMENT'
-    return t
-
-  def t_code_RIGHT_BRACKET(self, t):
-    r'}'
-    self.nesting -= 1
-    if self.nesting:
-      t.type = 'CODE_FRAGMENT'
-    else:
-      self.lexer.pop_state()
-    return t
-
   def t_ANY_error(self, t):
     raise Exception("Illegal character '%s'" % t.value[0])
 
   def build(self, **kwargs):
     self.lexer = lex.lex(module=self, **kwargs)