Experimental lexer generator: Fix the rest of the failures.

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 47 matching lines @@
 "%="          <|push_token(ASSIGN_MOD)|>
 
 "==="         <|push_token(EQ_STRICT)|>
 "=="          <|push_token(EQ)|>
 "="           <|push_token(ASSIGN)|>
 "!=="         <|push_token(NE_STRICT)|>
 "!="          <|push_token(NE)|>
 "!"           <|push_token(NOT)|>
 
 "//"          <||SingleLineComment>
-"/*"          <||MultiLineComment>
+"/*"          <{marker_ = cursor_ - 2;}||MultiLineComment>
 "<!--"        <||SingleLineComment>
 
 "<!-"        <|{
   cursor_ -= 2;
   yych = *(cursor_);
   PUSH_TOKEN(Token::LT);
 }|>
 
 "<!"        <|{
   cursor_ -= 1;
@@ skipping 109 matching lines @@
   if (V8_UNLIKELY(!ValidIdentifierStart())) {
     goto default_action;
   }
 }|push_token(IDENTIFIER)|Identifier>
 
 eof             <|terminate|>
 default_action  <push_token_and_go_forward(ILLEGAL)>
 
 <<DoubleQuoteString>>
 "\\" line_terminator_sequence <||continue>
-/\\[xX][:hex_digit:]{2}/      <||continue>
+/\\[x][:hex_digit:]{2}/       <||continue>
 /\\[u][:hex_digit:]{4}/       <||continue>
-/\\[^xXu\r\n]/                <||continue>
+/\\[^xu\r\n]/                 <||continue>
 "\\"                          <|push_token(ILLEGAL)|>
 /\n|\r/                       <|push_token(ILLEGAL)|>
 "\""                          <|push_token(STRING)|>
 eof                           <|terminate_illegal|>
 catch_all                     <||continue>
 
 <<SingleQuoteString>>
 # TODO subgraph for '\'
 "\\" line_terminator_sequence <||continue>
-/\\[xX][:hex_digit:]{2}/      <||continue>
+/\\[x][:hex_digit:]{2}/       <||continue>
 /\\[u][:hex_digit:]{4}/       <||continue>
-/\\[^xXu\r\n]/                <||continue>
+/\\[^xu\r\n]/                 <||continue>
 "\\"                          <|push_token(ILLEGAL)|>
 /\n|\r/                       <|push_token(ILLEGAL)|>
 "'"                           <|push_token(STRING)|>
 eof                           <|terminate_illegal|>
 catch_all                     <||continue>
 
 <<Identifier>>
 identifier_char <|push_token(IDENTIFIER)|continue>
 /\\u[:hex_digit:]{4}/ <{
   if (V8_UNLIKELY(!ValidIdentifierPart())) {
     goto default_action;
   }
 }|push_token(IDENTIFIER)|continue>
 
 <<SingleLineComment>>
 line_terminator  <|push_line_terminator|>
 eof <|skip_and_terminate|>
 catch_all <||continue>
 
 <<MultiLineComment>>
 /\*+\//             <|skip|>
 # TODO find a way to generate the below rule
 /\*+[^\/*]/       <||continue>
 line_terminator  <push_line_terminator||continue>
-eof <|skip_and_terminate|>
+eof <|{start_ = marker_; BACKWARD(); PUSH_TOKEN(Token::ILLEGAL);}|>
 catch_all        <||continue>