Experimental parser: remove inline code from lexer rules

tools/lexer_generator/rule_parser.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 102 matching lines @@
       assert not rules['default_action']
       rules['default_action'] = action
     elif p[1] == 'catch_all':
       assert not rules['catch_all']
       rules['catch_all'] = (precedence, action)
     else:
       regex = p[1]
       rules['regex'].append((regex, precedence, action))
 
   def p_action(self, p):
-    '''action : ACTION_OPEN maybe_action_part OR maybe_action_part OR maybe_transition ACTION_CLOSE'''
+    '''action : ACTION_OPEN maybe_identifier_action OR maybe_identifier_action OR maybe_transition ACTION_CLOSE'''
     p[0] = (p[2], p[4], p[6])
 
   def p_default_action(self, p):
-    'default_action : ACTION_OPEN action_part ACTION_CLOSE'
+    'default_action : ACTION_OPEN identifier_action ACTION_CLOSE'
     p[0] = (None, p[2], None)
 
-  def p_maybe_action_part(self, p):
-    '''maybe_action_part : action_part
+  def p_maybe_identifier_action(self, p):
+    '''maybe_identifier_action : identifier_action
                          | empty'''
     p[0] = p[1]
 
-  def p_action_part(self, p):
-    '''action_part : code
-                   | identifier_action'''
-    p[0] = p[1]
-
   def p_maybe_transition(self, p):
     '''maybe_transition : IDENTIFIER
                         | empty'''
     p[0] = p[1]
 
   def p_identifier_action(self, p):
     '''identifier_action : IDENTIFIER
                          | IDENTIFIER LEFT_PARENTHESIS RIGHT_PARENTHESIS
                          | IDENTIFIER LEFT_PARENTHESIS action_params RIGHT_PARENTHESIS'''
-    assert p[1] != 'code'
     if len(p) == 2 or len(p) == 4:
       p[0] = (p[1], None)
     elif len(p) == 5:
       if len(p[3]) == 1:
         p[0] = (p[1], p[3][0])
       else:
         p[0] = (p[1], p[3])
     else:
       raise Exception()
 
@@ skipping 53 matching lines @@
     'regex_alias : IDENTIFIER'
     p[0] = self.__state.aliases[p[1]]
 
   def p_modifier(self, p):
     '''modifier : PLUS
                 | QUESTION_MARK
                 | STAR
                 | empty'''
     p[0] = p[1]
 
-  def p_code(self, p):
-    'code : LEFT_BRACKET code_fragments RIGHT_BRACKET'
-    p[0] = ('code', p[2].strip())
-
-  def p_code_fragments(self, p):
-    '''code_fragments : CODE_FRAGMENT code_fragments
-                      | empty'''
-    p[0] = p[1]
-    if len(p) == 3 and p[2]:
-      p[0] = p[1] + p[2]
-
   def p_empty(self, p):
     'empty :'
 
   def p_error(self, p):
     raise Exception("Syntax error in input '%s'" % str(p))
 
   def build(self, **kwargs):
     self.parser = yacc.yacc(module=self, debug=0, write_tables=0, **kwargs)
     self.lexer = RuleLexer()
     self.lexer.build(**kwargs)
@@ skipping 97 matching lines @@
     for k, v in parser_state.rules.items():
       if k == 'default': continue
       process(k, v)
     process('default', parser_state.rules['default'])
     # build the automata
     for rule_name, graph in rule_map.items():
       self.__automata[rule_name] = RuleProcessor.Automata(builder, graph)
     # process default_action
     default_action = parser_state.rules['default']['default_action']
     self.default_action = Action(None, default_action[1]) if default_action else None