Experimental parser: Small fixes.

src/lexer/lexer.re

 // Portions of this code based on re2c:
 //   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
 //       with the distribution.
 //     * Neither the name of Google Inc. nor the names of its
 //       contributors may be used to endorse or promote products derived
 //       from this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#include <fcntl.h>
 #include <stdio.h>
-#include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
 
 // TODO:
 // - SpiderMonkey compatibility hack: "  --> something" is treated
 //   as a single line comment.
-// - An identifier cannot start immediately after a number.
 // - Run-time lexing modifications: harmony number literals, keywords depending
 //   on harmony_modules, harmony_scoping
+// - Escaping the string literals (like the baseline does)
+// - Error recovery after illegal tokens.
 
 enum Condition {
   kConditionNormal,
   kConditionDoubleQuoteString,
   kConditionSingleQuoteString,
   kConditionIdentifier,
   kConditionSingleLineComment,
   kConditionMultiLineComment,
   kConditionHtmlComment
 };
@@ skipping 20 matching lines @@
 #endif //  defined(WIN32)
 
 #include "lexer.h"
 
 using namespace v8::internal;
 
 #define PUSH_TOKEN(T) { send(T); SKIP(); }
 #define PUSH_TOKEN_LOOKAHEAD(T) { --cursor_; send(T); SKIP(); }
 #define PUSH_EOF_AND_RETURN() { send(Token::EOS); eof_ = true; return 1;}
 #define PUSH_LINE_TERMINATOR() { SKIP(); }
-#define TERMINATE_ILLEGAL() { return 1; }
+#define TERMINATE_ILLEGAL() { send(Token::ILLEGAL); send(Token::EOS); return 1; }
 
 PushScanner::PushScanner(ExperimentalScanner* sink)
 : eof_(false),
   state_(-1),
   condition_(kConditionNormal),
   limit_(NULL),
   start_(NULL),
   cursor_(NULL),
   marker_(NULL),
   real_start_(0),
@@ skipping 67 matching lines @@
     buffer_end_ = needed + buffer_;
 
     marker_ = marker__offset + buffer_;
     cursor_ = cursor__offset + buffer_;
     start_ = buffer_ + start_offset;
     limit_ = limit__offset + buffer_;
   }
   memcpy(limit_, input, input_size);
   limit_ += input_size;
 
-  // The scanner starts here
-#define YYLIMIT     limit_
-#define YYCURSOR    cursor_
-#define YYMARKER    marker_
-#define YYCTYPE     uint8_t
+#define SKIP()                { start_ = cursor_; YYSETCONDITION(kConditionNormal); goto yy0; }
+#define YYFILL(n)             { goto fill;        }
 
-#define SKIP()     { start_ = cursor_; YYSETCONDITION(kConditionNormal); goto yy0; }
-#define YYFILL(n)    { goto fill;        }
+#define YYGETSTATE()          state_
+#define YYSETSTATE(x)         { state_ = (x); }
 
-#define YYGETSTATE()  state_
-#define YYSETSTATE(x)  { state_ = (x); }
-
-#define YYGETCONDITION() condition_
-#define YYSETCONDITION(x) { condition_ = (x); }
+#define YYGETCONDITION()      condition_
+#define YYSETCONDITION(x)     { condition_ = (x); }
 
 start_:
   if (FLAG_trace_lexer) {
     printf("Starting a round; state_: %d, condition_: %d\n", state_, condition_);
   }
 
   /*!re2c
-    re2c:indent:top   = 1;
+    re2c:indent:top = 1;
     re2c:yych:conversion = 0;
-    re2c:condenumprefix     = kCondition;
-    re2c:define:YYCONDTYPE    = Condition;
+    re2c:condenumprefix = kCondition;
+    re2c:define:YYCONDTYPE = Condition;
+    re2c:define:YYCURSOR = cursor_;
+    re2c:define:YYCTYPE = uint8_t;
+    re2c:define:YYLIMIT = limit_;
+    re2c:define:YYMARKER = marker_;
 
     eof = "\000";
     any = [\000-\377];
     whitespace_char = [ \t\v\f\r];
     whitespace = whitespace_char+;
     identifier_start_ = [$_\\a-zA-Z];
     identifier_char = [$_\\a-zA-Z0-9];
     not_identifier_char = any\identifier_char;
     line_terminator = [\n\r]+;
     digit = [0-9];
@@ skipping 69 matching lines @@
     <Normal> ">>>="        { PUSH_TOKEN(Token::ASSIGN_SHR); }
     <Normal> "<<="         { PUSH_TOKEN(Token::ASSIGN_SHL); }
     <Normal> ">>="         { PUSH_TOKEN(Token::ASSIGN_SAR); }
     <Normal> "<="          { PUSH_TOKEN(Token::LTE); }
     <Normal> ">="          { PUSH_TOKEN(Token::GTE); }
     <Normal> "<<"          { PUSH_TOKEN(Token::SHL); }
     <Normal> ">>"          { PUSH_TOKEN(Token::SAR); }
     <Normal> "<"           { PUSH_TOKEN(Token::LT); }
     <Normal> ">"           { PUSH_TOKEN(Token::GT); }
 
-    <Normal> '0x' hex_digit+                     { PUSH_TOKEN(Token::NUMBER); }
-    <Normal> "." digit+ maybe_exponent           { PUSH_TOKEN(Token::NUMBER); }
-    <Normal> digit+ ("." digit+)? maybe_exponent { PUSH_TOKEN(Token::NUMBER); }
+    <Normal> '0x' hex_digit+ not_identifier_char                     { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
+    <Normal> "." digit+ maybe_exponent not_identifier_char           { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
+    <Normal> digit+ ("." digit+)? maybe_exponent not_identifier_char { PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
 
     <Normal> "("           { PUSH_TOKEN(Token::LPAREN); }
     <Normal> ")"           { PUSH_TOKEN(Token::RPAREN); }
     <Normal> "["           { PUSH_TOKEN(Token::LBRACK); }
     <Normal> "]"           { PUSH_TOKEN(Token::RBRACK); }
     <Normal> "{"           { PUSH_TOKEN(Token::LBRACE); }
     <Normal> "}"           { PUSH_TOKEN(Token::RBRACE); }
     <Normal> ":"           { PUSH_TOKEN(Token::COLON); }
     <Normal> ";"           { PUSH_TOKEN(Token::SEMICOLON); }
     <Normal> "."           { PUSH_TOKEN(Token::PERIOD); }
     <Normal> "?"           { PUSH_TOKEN(Token::CONDITIONAL); }
     <Normal> "++"          { PUSH_TOKEN(Token::INC); }
     <Normal> "--"          { PUSH_TOKEN(Token::DEC); }
 
     <Normal> "||"          { PUSH_TOKEN(Token::OR); }
     <Normal> "&&"          { PUSH_TOKEN(Token::AND); }
 
     <Normal> "|"           { PUSH_TOKEN(Token::BIT_OR); }
     <Normal> "^"           { PUSH_TOKEN(Token::BIT_XOR); }
     <Normal> "&"           { PUSH_TOKEN(Token::BIT_AND); }
     <Normal> "+"           { PUSH_TOKEN(Token::ADD); }
     <Normal> "-"           { PUSH_TOKEN(Token::SUB); }
     <Normal> "*"           { PUSH_TOKEN(Token::MUL); }
     <Normal> "/"           { PUSH_TOKEN(Token::DIV); }
     <Normal> "%"           { PUSH_TOKEN(Token::MOD); }
     <Normal> "~"           { PUSH_TOKEN(Token::BIT_NOT); }
     <Normal> ","           { PUSH_TOKEN(Token::COMMA); }
 
-    <Normal> line_terminator+ { PUSH_LINE_TERMINATOR(); }
+    <Normal> line_terminator  { PUSH_LINE_TERMINATOR(); }
     <Normal> whitespace       { SKIP(); }
 
     <Normal> ["]           :=> DoubleQuoteString
     <Normal> [']           :=> SingleQuoteString
 
     <Normal> identifier_start_    :=> Identifier
 
     <Normal> eof           { PUSH_EOF_AND_RETURN();}
     <Normal> any           { TERMINATE_ILLEGAL(); }
 
     <DoubleQuoteString> "\\\""  { goto yy0; }
     <DoubleQuoteString> '"'     { PUSH_TOKEN(Token::STRING);}
+    <DoubleQuoteString> line_terminator { TERMINATE_ILLEGAL(); }
+    <DoubleQuoteString> eof     { TERMINATE_ILLEGAL(); }
     <DoubleQuoteString> any     { goto yy0; }
 
     <SingleQuoteString> "\\'"   { goto yy0; }
     <SingleQuoteString> "'"     { PUSH_TOKEN(Token::STRING);}
+    <SingleQuoteString> line_terminator { TERMINATE_ILLEGAL(); }
+    <SingleQuoteString> eof     { TERMINATE_ILLEGAL(); }
     <SingleQuoteString> any     { goto yy0; }
 
     <Identifier> identifier_char+  { goto yy0; }
     <Identifier> any               { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
 
     <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> eof             { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> any             { goto yy0; }
 
     <MultiLineComment> [*][//]  { PUSH_LINE_TERMINATOR();}
     <MultiLineComment> eof      { TERMINATE_ILLEGAL(); }
     <MultiLineComment> any      { goto yy0; }
 
     <HtmlComment> eof        { TERMINATE_ILLEGAL(); }
     <HtmlComment> "-->"      { PUSH_LINE_TERMINATOR();}
     <HtmlComment> any        { goto yy0; }
     */
 
 fill:
   int unfinished_size = cursor_ - start_;
   if (FLAG_trace_lexer) {
     printf(
         "scanner needs a refill. Exiting for now with:\n"
         "  saved fill state_ = %d\n"
         "  unfinished token size = %d\n",
         state_,
         unfinished_size);
-    if(0 < unfinished_size && start_ < limit_) {
+    if (0 < unfinished_size && start_ < limit_) {
       printf("  unfinished token is: ");
       fwrite(start_, 1, cursor_ - start_, stdout);
       putchar('\n');
     }
     putchar('\n');
   }
 
   if (eof_) goto start_;
 
   //  Once we get here, we can get rid of
   //  everything before start_ and after limit_.
 
   if (buffer_ < start_) {
     size_t start_offset = start_ - buffer_;
     memmove(buffer_, start_, limit_ - start_);
     marker_ -= start_offset;
     cursor_ -= start_offset;
     limit_ -= start_offset;
     start_ -= start_offset;
     real_start_ += start_offset;
   }
   return 0;
 }