Experimental scanner: HTML comments.

src/lexer/lexer.re

 // Portions of this code based on re2c:
 //   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 38 matching lines @@
 #include "parser.h"
 #include "platform.h"
 #include "preparser.h"
 #include "runtime.h"
 #include "scanner-character-streams.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
 
 
 // TODO:
-// - SpiderMonkey compatibility hack: "  --> something" is treated
-//   as a single line comment.
 // - Run-time lexing modifications: harmony number literals, keywords depending
 //   on harmony_modules, harmony_scoping
 // - Escaping the string literals (like the baseline does)
 // - Error recovery after illegal tokens.
 
 enum Condition {
   kConditionNormal,
   kConditionDoubleQuoteString,
   kConditionSingleQuoteString,
   kConditionIdentifier,
@@ skipping 37 matching lines @@
   c = (c | 0x20) - ('a' - '0');  // detect 0x11..0x16 and 0x31..0x36.
   if (static_cast<unsigned>(c) <= 5) return c + 10;
   return -1;
 }
 
 }
 
 #define PUSH_TOKEN(T) { send(T); SKIP(); }
 #define PUSH_TOKEN_LOOKAHEAD(T) { --cursor_; send(T); SKIP(); }
 #define PUSH_EOF_AND_RETURN() { send(Token::EOS); eof_ = true; return 1;}
-#define PUSH_LINE_TERMINATOR() { SKIP(); }
+#define PUSH_LINE_TERMINATOR() { just_seen_line_terminator_ = true; SKIP(); }
 #define TERMINATE_ILLEGAL() { send(Token::ILLEGAL); send(Token::EOS); return 1; }
 
 #define YYCTYPE uint8_t
 
 PushScanner::PushScanner(ExperimentalScanner* sink, UnicodeCache* unicode_cache)
 : unicode_cache_(unicode_cache),
   eof_(false),
   state_(-1),
   condition_(kConditionNormal),
   limit_(NULL),
   start_(NULL),
   cursor_(NULL),
   marker_(NULL),
   real_start_(0),
   buffer_(NULL),
   buffer_end_(NULL),
   yych(0),
   yyaccept(0),
+  just_seen_line_terminator_(true),
   sink_(sink) {
 
 }
 
 PushScanner::~PushScanner() {
 }
 
 
 uc32 PushScanner::ScanHexNumber(int length) {
   // We have seen \uXXXX, let's see what it is.
@@ skipping 20 matching lines @@
 }
 
 void PushScanner::send(Token::Value token) {
   int beg = (start_ - buffer_) + real_start_;
   int end = (cursor_ - buffer_) + real_start_;
   if (FLAG_trace_lexer) {
     printf("got %s at (%d, %d): ", Token::Name(token), beg, end);
     for (uint8_t* s = start_; s != cursor_; s++) printf("%c", (char)*s);
     printf(".\n");
   }
+  just_seen_line_terminator_ = false;
   sink_->Record(token, beg, end);
 }
 
 uint32_t PushScanner::push(const void *input, int input_size) {
   if (FLAG_trace_lexer) {
     printf(
         "scanner is receiving a new data batch of length %d\n"
         "scanner continues with saved state_ = %d\n",
         input_size,
         state_);
@@ skipping 137 matching lines @@
     <Normal> "%="          { PUSH_TOKEN(Token::ASSIGN_MOD); }
 
     <Normal> "==="         { PUSH_TOKEN(Token::EQ_STRICT); }
     <Normal> "=="          { PUSH_TOKEN(Token::EQ); }
     <Normal> "="           { PUSH_TOKEN(Token::ASSIGN); }
     <Normal> "!=="         { PUSH_TOKEN(Token::NE_STRICT); }
     <Normal> "!="          { PUSH_TOKEN(Token::NE); }
     <Normal> "!"           { PUSH_TOKEN(Token::NOT); }
 
     <Normal> "//"          :=> SingleLineComment
+    <Normal> whitespace? "-->" { if (just_seen_line_terminator_) { YYSETCONDITION(kConditionSingleLineComment); goto yy0; } else { --cursor_; send(Token::DEC); start_ = cursor_; goto yy0; } }
     <Normal> "/*"          :=> MultiLineComment
     <Normal> "<!--"        :=> HtmlComment
 
     <Normal> ">>>="        { PUSH_TOKEN(Token::ASSIGN_SHR); }
     <Normal> ">>>"         { PUSH_TOKEN(Token::SHR); }
     <Normal> "<<="         { PUSH_TOKEN(Token::ASSIGN_SHL); }
     <Normal> ">>="         { PUSH_TOKEN(Token::ASSIGN_SAR); }
     <Normal> "<="          { PUSH_TOKEN(Token::LTE); }
     <Normal> ">="          { PUSH_TOKEN(Token::GTE); }
     <Normal> "<<"          { PUSH_TOKEN(Token::SHL); }
@@ skipping 81 matching lines @@
     <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
     <SingleLineComment> eof             { PUSH_TOKEN(Token::EOS); }
     <SingleLineComment> any             { goto yy0; }
 
     <MultiLineComment> [*][//]  { PUSH_LINE_TERMINATOR();}
     <MultiLineComment> eof      { TERMINATE_ILLEGAL(); }
     <MultiLineComment> any      { goto yy0; }
 
     <HtmlComment> eof        { TERMINATE_ILLEGAL(); }
     <HtmlComment> "-->"      { PUSH_LINE_TERMINATOR();}
+    <HtmlComment> line_terminator+ { PUSH_LINE_TERMINATOR();}
     <HtmlComment> any        { goto yy0; }
     */
 
 fill:
   int unfinished_size = cursor_ - start_;
   if (FLAG_trace_lexer) {
     printf(
         "scanner needs a refill. Exiting for now with:\n"
         "  saved fill state_ = %d\n"
         "  unfinished token size = %d\n",
@@ skipping 16 matching lines @@
     size_t start_offset = start_ - buffer_;
     memmove(buffer_, start_, limit_ - start_);
     marker_ -= start_offset;
     cursor_ -= start_offset;
     limit_ -= start_offset;
     start_ -= start_offset;
     real_start_ += start_offset;
   }
   return 0;
 }