Experimental parser: remove inline code from lexer rules

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 36 matching lines @@
   /0[xX][:hex_digit:]+/ | (
   /\.[:digit:]+/ maybe_exponent |
   non_octal_whole_part /(\.[:digit:]*)?/ maybe_exponent );
 harmony_number = "0"[bBoO][:digit:]+;
 line_terminator_sequence = /[:line_terminator:]|(\r\n|\n\r)/;
 eos = [:eos:];
 
 # grammar is
 #   regex <action_on_state_entry|action_on_match|transition>
 #
-# actions can be c code enclosed in {} or identifiers to be passed to codegen
+# actions are identifiers to be passed to codegen
 # transition must be 'continue' or the name of a subgraph
 
 <<default>>
 
 "|="          <|token(ASSIGN_BIT_OR)|>
 "^="          <|token(ASSIGN_BIT_XOR)|>
 "&="          <|token(ASSIGN_BIT_AND)|>
 "+="          <|token(ASSIGN_ADD)|>
 "-="          <|token(ASSIGN_SUB)|>
 "*="          <|token(ASSIGN_MUL)|>
 "/="          <|token(ASSIGN_DIV)|>
 "%="          <|token(ASSIGN_MOD)|>
 
 "==="         <|token(EQ_STRICT)|>
 "=="          <|token(EQ)|>
 "="           <|token(ASSIGN)|>
 "!=="         <|token(NE_STRICT)|>
 "!="          <|token(NE)|>
 "!"           <|token(NOT)|>
 
 "//"          <||SingleLineComment>
 "/*"          <set_marker(2)||MultiLineComment>
 "<!--"        <||SingleLineComment>
 
-"<!-"        <|{
-  BACKWARD(2);
-  DO_TOKEN(Token::LT);
-  return;
-}|>
-
-"<!"        <|{
-  BACKWARD(1);
-  DO_TOKEN(Token::LT);
-  return;
-}|>
-
-"-->" <{
-  if (!has_line_terminator_before_next_) {
-    BACKWARD(1);
-    DO_TOKEN(Token::DEC);
-    return;
-  }
-}||SingleLineComment>
+"<!-"  <|backtrack(2, LT)|>
+"<!"   <|backtrack(1, LT)|>
+"-->"  <if_line_terminator_backtrack(1, DEC)||SingleLineComment>
 
 ">>>="        <|token(ASSIGN_SHR)|>
 ">>>"         <|token(SHR)|>
 "<<="         <|token(ASSIGN_SHL)|>
 ">>="         <|token(ASSIGN_SAR)|>
 "<="          <|token(LTE)|>
 ">="          <|token(GTE)|>
 "<<"          <|token(SHL)|>
 ">>"          <|token(SAR)|>
 "<"           <|token(LT)|>
@@ skipping 82 matching lines @@
 "true"        <|token(TRUE_LITERAL)|>
 "try"         <|token(TRY)|>
 "typeof"      <|token(TYPEOF)|>
 "var"         <|token(VAR)|>
 "void"        <|token(VOID)|>
 "while"       <|token(WHILE)|>
 "with"        <|token(WITH)|>
 "yield"       <|token(YIELD)|>
 
 identifier_start <|token(IDENTIFIER)|Identifier>
-/\\u[:hex_digit:]{4}/ <{
-  if (V8_UNLIKELY(!ValidIdentifierStart())) {
-    goto default_action;
-  }
-  next_.has_escapes = true;
-}|token(IDENTIFIER)|Identifier>
+/\\u[:hex_digit:]{4}/ <check_escaped_identifier_start|token(IDENTIFIER)|Identifier>
 
 eos             <|terminate|>
 default_action  <do_token_and_go_forward(ILLEGAL)>
 
 <<DoubleQuoteString>>
 "\\" line_terminator_sequence <set_has_escapes||continue>
 /\\[x][:hex_digit:]{2}/       <set_has_escapes||continue>
 /\\[u][:hex_digit:]{4}/       <set_has_escapes||continue>
 /\\[1-7]/                     <octal_inside_string||continue>
 /\\[0-7][0-7]+/               <octal_inside_string||continue>
@@ skipping 15 matching lines @@
 "\\0"                         <set_has_escapes||continue>
 /\\[^xu0-7:line_terminator:]/ <set_has_escapes||continue>
 "\\"                          <|token(ILLEGAL)|>
 line_terminator               <|token(ILLEGAL)|>
 "'"                           <|token(STRING)|>
 eos                           <|terminate_illegal|>
 catch_all                     <||continue>
 
 <<Identifier>>
 identifier_char <|token(IDENTIFIER)|continue>
-/\\u[:hex_digit:]{4}/ <{
-  if (V8_UNLIKELY(!ValidIdentifierPart())) {
-    goto default_action;
-  }
-  next_.has_escapes = true;
-}|token(IDENTIFIER)|continue>
+/\\u[:hex_digit:]{4}/ <check_escaped_identifier_part|token(IDENTIFIER)|continue>
 
 <<SingleLineComment>>
 line_terminator  <|line_terminator|>
 eos              <|skip_and_terminate|>
 catch_all        <||continue>
 
 <<MultiLineComment>>
 /\*+\//          <|skip|>
 # TODO find a way to generate the below rule
 /\*+[^\/*]/      <||continue>
 line_terminator  <line_terminator_in_multiline_comment||continue>
 eos              <|terminate_illegal|>
 catch_all        <||continue>