Experimental parser: unify regexp handling with the baseline some more.

src/lexer/lexer.re

 // Portions of this code based on re2c:
 //   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 308 matching lines @@
 
     <Normal> identifier_start_    :=> Identifier
 
     <Normal> eof           { PUSH_EOF_AND_RETURN();}
     <Normal> any           { PUSH_TOKEN(Token::ILLEGAL); }
 
     <DoubleQuoteString> "\\\""  { goto yy0; }
     <DoubleQuoteString> '"'     { PUSH_TOKEN(Token::STRING);}
     <DoubleQuoteString> "\\" "\n" "\r"? { goto yy0; }
     <DoubleQuoteString> "\\" "\r" "\n"? { goto yy0; }
-    <DoubleQuoteString> line_terminator { TERMINATE_ILLEGAL(); }
+    <DoubleQuoteString> "\n"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
+    <DoubleQuoteString> "\r"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <DoubleQuoteString> eof     { TERMINATE_ILLEGAL(); }
     <DoubleQuoteString> any     { goto yy0; }
 
     <SingleQuoteString> "\\'"   { goto yy0; }
     <SingleQuoteString> "'"     { PUSH_TOKEN(Token::STRING);}
     <SingleQuoteString> "\\" "\n" "\r"? { goto yy0; }
     <SingleQuoteString> "\\" "\r" "\n"? { goto yy0; }
-    <SingleQuoteString> line_terminator { TERMINATE_ILLEGAL(); }
+    <SingleQuoteString> "\n"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
+    <SingleQuoteString> "\r"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <SingleQuoteString> eof     { TERMINATE_ILLEGAL(); }
     <SingleQuoteString> any     { goto yy0; }
 
     <Identifier> identifier_char+  { goto yy0; }
     <Identifier> illegal_after_identifier { PUSH_TOKEN(Token::ILLEGAL); }
     <Identifier> any               { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
 
     <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> eof             { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> any             { goto yy0; }
@@ skipping 33 matching lines @@
     size_t start_offset = start_ - buffer_;
     memmove(buffer_, start_, limit_ - start_);
     marker_ -= start_offset;
     cursor_ -= start_offset;
     limit_ -= start_offset;
     start_ -= start_offset;
     real_start_ += start_offset;
   }
   return 0;
 }