Experimental parser: add back last re2c rules

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 47 matching lines @@
 "=="          <|push_token(EQ)|>
 "="           <|push_token(ASSIGN)|>
 "!=="         <|push_token(NE_STRICT)|>
 "!="          <|push_token(NE)|>
 "!"           <|push_token(NOT)|>
 
 "//"          <||SingleLineComment>
 "/*"          <||MultiLineComment>
 "<!--"        <||HtmlComment>
 
-#whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITION(kConditionSingleLineComment); goto yyc_SingleLineComment; } else { --cursor_; send(Token::DEC); start_ = cursor_; goto yyc_Normal; } }
+whitespace? "-->" <{
+  if (!just_seen_line_terminator_) {
+    PUSH_TOKEN(Token::DEC);
+    start_ = cursor_ - 1;
+    goto code_start;
+  }
+}||SingleLineComment>
 
 ">>>="        <|push_token(ASSIGN_SHR)|>
 ">>>"         <|push_token(SHR)|>
 "<<="         <|push_token(ASSIGN_SHL)|>
 ">>="         <|push_token(ASSIGN_SAR)|>
 "<="          <|push_token(LTE)|>
 ">="          <|push_token(GTE)|>
 "<<"          <|push_token(SHL)|>
 ">>"          <|push_token(SAR)|>
 "<"           <|push_token(LT)|>
@@ skipping 79 matching lines @@
 "typeof"      <|push_token(TYPEOF)|>
 "var"         <|push_token(VAR)|>
 "void"        <|push_token(VOID)|>
 "while"       <|push_token(WHILE)|>
 "with"        <|push_token(WITH)|>
 "yield"       <|push_token(YIELD)|>
 
 identifier_start <|push_token(IDENTIFIER)|Identifier>
 /\\u[0-9a-fA-F]{4}/ <{
   if (V8_UNLIKELY(!ValidIdentifierStart())) {
-    PUSH_TOKEN(Token::ILLEGAL);
-    // need to goto something here
+    goto default_action;
   }
 }|push_token(IDENTIFIER)|Identifier>
 
 eof             <|terminate|>
 default_action  <push_token(ILLEGAL)>
 
 <<DoubleQuoteString>>
 /\\\n\r?/ <||continue>
 /\\\r\n?/ <||continue>
 /\\./     <||continue>
 /\n|\r/   <|push_token(ILLEGAL)|>
 "\""      <|push_token(STRING)|>
 eof       <|terminate_illegal|>
 catch_all <||continue>
 
 <<SingleQuoteString>>
 /\\\n\r?/ <||continue>
 /\\\r\n?/ <||continue>
 /\\./     <||continue>
 /\n|\r/   <|push_token(ILLEGAL)|>
 "'"       <|push_token(STRING)|>
 eof       <|terminate_illegal|>
 catch_all <||continue>
 
 <<Identifier>>
 identifier_char <|push_token(IDENTIFIER)|continue>
 /\\u[0-9a-fA-F]{4}/ <{
-  if (V8_UNLIKELY(!ValidIdentifierStart())) {
-    PUSH_TOKEN(Token::ILLEGAL);
-    // need to goto something here
+  if (V8_UNLIKELY(!ValidIdentifierPart())) {
+    goto default_action;
   }
 }|push_token(IDENTIFIER)|continue>
 
 <<SingleLineComment>>
 line_terminator  <|push_line_terminator|>
 catch_all <||continue>
 
 <<MultiLineComment>>
 "*/"             <|skip|>
 # TODO find a way to generate the below rule
 /\*[^\/]/        <||continue>
 line_terminator  <push_line_terminator||continue>
 catch_all        <||continue>
 
 <<HtmlComment>>
 "-->"            <|skip|>
 # TODO find a way to generate the below rules
 /--./            <||continue>
 /-./             <||continue>
 line_terminator  <push_line_terminator||continue>
 catch_all <||continue>