Experimental scanner: more misc fixes around invalid tokens.

src/lexer/lexer.re

 // Portions of this code based on re2c:
 //   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 241 matching lines @@
     re2c:condenumprefix = kCondition;
     re2c:define:YYCONDTYPE = Condition;
     re2c:define:YYCURSOR = cursor_;
     re2c:define:YYLIMIT = limit_;
     re2c:define:YYMARKER = marker_;
 
     eof = "\000";
     any = [\000-\377];
     whitespace_char = [ \t\v\f\r];
     whitespace = whitespace_char+;
-    identifier_start_ = [$_a-zA-Z];
+    identifier_start = [$_a-zA-Z];
     identifier_char = [$_a-zA-Z0-9];
     not_identifier_char = any\identifier_char\[\\];
     line_terminator = [\n\r]+;
     digit = [0-9];
     hex_digit = [0-9a-fA-F];
     maybe_exponent = ('e' [-+]? digit+)?;
 
     <Normal> "break" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::BREAK); }
     <Normal> "case" not_identifier_char       { PUSH_TOKEN_LOOKAHEAD(Token::CASE); }
     <Normal> "catch" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::CATCH); }
@@ skipping 68 matching lines @@
     <Normal> ">="          { PUSH_TOKEN(Token::GTE); }
     <Normal> "<<"          { PUSH_TOKEN(Token::SHL); }
     <Normal> ">>"          { PUSH_TOKEN(Token::SAR); }
     <Normal> "<"           { PUSH_TOKEN(Token::LT); }
     <Normal> ">"           { PUSH_TOKEN(Token::GT); }
 
     <Normal> '0x' hex_digit+ not_identifier_char                     { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
     <Normal> "." digit+ maybe_exponent not_identifier_char           { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
     <Normal> digit+ ("." digit+)? maybe_exponent not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
 
+    <Normal> '0x' hex_digit+ "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else { YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6; send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+    <Normal> "." digit+ maybe_exponent "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else { YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6; send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+    <Normal> digit+ ("." digit+)? maybe_exponent "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else { YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6; send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+
     <Normal> "("           { PUSH_TOKEN(Token::LPAREN); }
     <Normal> ")"           { PUSH_TOKEN(Token::RPAREN); }
     <Normal> "["           { PUSH_TOKEN(Token::LBRACK); }
     <Normal> "]"           { PUSH_TOKEN(Token::RBRACK); }
     <Normal> "{"           { PUSH_TOKEN(Token::LBRACE); }
     <Normal> "}"           { PUSH_TOKEN(Token::RBRACE); }
     <Normal> ":"           { PUSH_TOKEN(Token::COLON); }
     <Normal> ";"           { PUSH_TOKEN(Token::SEMICOLON); }
     <Normal> "."           { PUSH_TOKEN(Token::PERIOD); }
     <Normal> "?"           { PUSH_TOKEN(Token::CONDITIONAL); }
@@ skipping 13 matching lines @@
     <Normal> "%"           { PUSH_TOKEN(Token::MOD); }
     <Normal> "~"           { PUSH_TOKEN(Token::BIT_NOT); }
     <Normal> ","           { PUSH_TOKEN(Token::COMMA); }
 
     <Normal> line_terminator  { PUSH_LINE_TERMINATOR(); }
     <Normal> whitespace       { SKIP(); }
 
     <Normal> ["]           :=> DoubleQuoteString
     <Normal> [']           :=> SingleQuoteString
 
-    <Normal> identifier_start_    :=> Identifier
-    <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); goto yy0; } YYSETCONDITION(kConditionIdentifierIllegal); send(Token::ILLEGAL); start_ = cursor_;  goto yy0; }
+    <Normal> identifier_start     :=> Identifier
+    <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); goto yy0; } send(Token::ILLEGAL); start_ = cursor_; goto yy0; }
     <Normal> "\\"                 { PUSH_TOKEN(Token::ILLEGAL); }
 
     <Normal> eof           { PUSH_EOF_AND_RETURN();}
-    <Normal> any           { PUSH_TOKEN(Token::ILLEGAL); }
+    <Normal> any           :=> IdentifierIllegal
 
     <DoubleQuoteString> "\\\\"  { goto yy0; }
     <DoubleQuoteString> "\\\""  { goto yy0; }
     <DoubleQuoteString> '"'     { PUSH_TOKEN(Token::STRING);}
     <DoubleQuoteString> "\\" "\n" "\r"? { goto yy0; }
     <DoubleQuoteString> "\\" "\r" "\n"? { goto yy0; }
     <DoubleQuoteString> "\n"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <DoubleQuoteString> "\r"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <DoubleQuoteString> eof     { TERMINATE_ILLEGAL(); }
     <DoubleQuoteString> any     { goto yy0; }
 
     <SingleQuoteString> "\\\\"  { goto yy0; }
     <SingleQuoteString> "\\'"   { goto yy0; }
     <SingleQuoteString> "'"     { PUSH_TOKEN(Token::STRING);}
     <SingleQuoteString> "\\" "\n" "\r"? { goto yy0; }
     <SingleQuoteString> "\\" "\r" "\n"? { goto yy0; }
     <SingleQuoteString> "\n"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <SingleQuoteString> "\r"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
     <SingleQuoteString> eof     { TERMINATE_ILLEGAL(); }
     <SingleQuoteString> any     { goto yy0; }
 
     <Identifier> identifier_char+  { goto yy0; }
-    <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) goto yy0; YYSETCONDITION(kConditionIdentifierIllegal); send(Token::ILLEGAL); }
+    <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) { YYSETCONDITION(kConditionIdentifier); goto yy0; } YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cursor_; goto yy0; }
     <Identifier> "\\"              { PUSH_TOKEN(Token::ILLEGAL); }
     <Identifier> any               { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
 
-    <IdentifierIllegal> identifier_char+  { goto yy0; }
+    <IdentifierIllegal> identifier_start  { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
+    <IdentifierIllegal> identifier_char\identifier_start  { goto yy0; }
+    <IdentifierIllegal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) { cursor_ -= 6; PUSH_TOKEN(Token::ILLEGAL); } goto yy0; }
     <IdentifierIllegal> "\\"+             { goto yy0; }
     <IdentifierIllegal> any               { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
 
     <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> eof             { PUSH_TOKEN(Token::EOS); }
     <SingleLineComment> any             { goto yy0; }
 
     <MultiLineComment> [*][//]  { PUSH_LINE_TERMINATOR();}
     <MultiLineComment> eof      { TERMINATE_ILLEGAL(); }
     <MultiLineComment> any      { goto yy0; }
@@ skipping 29 matching lines @@
     size_t start_offset = start_ - buffer_;
     memmove(buffer_, start_, limit_ - start_);
     marker_ -= start_offset;
     cursor_ -= start_offset;
     limit_ -= start_offset;
     start_ -= start_offset;
     real_start_ += start_offset;
   }
   return 0;
 }