[esnext] implement frontend changes for async/await proposal

src/parsing/scanner.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Features shared by parsing and pre-parsing scanners.
 
 #include "src/parsing/scanner.h"
 
 #include <stdint.h>
 
@@ skipping 38 matching lines @@
 }
 
 
 void Scanner::Initialize(Utf16CharacterStream* source) {
   source_ = source;
   // Need to capture identifiers in order to recognize "get" and "set"
   // in object literals.
   Init();
   // Skip initial whitespace allowing HTML comment ends just like
   // after a newline and scan first token.
-  has_line_terminator_before_next_ = true;
+  has_preceding_line_terminator_ = true;
+  has_preceding_multiline_comment_ = false;
   SkipWhiteSpace();
   Scan();
 }
 
 template <bool capture_raw, bool unicode>
 uc32 Scanner::ScanHexNumber(int expected_length) {
   DCHECK(expected_length <= 4);  // prevent overflow
 
   int begin = source_pos() - 2;
   uc32 x = 0;
@@ skipping 172 matching lines @@
 
 Token::Value Scanner::Next() {
   if (next_.token == Token::EOS) {
     next_.location.beg_pos = current_.location.beg_pos;
     next_.location.end_pos = current_.location.end_pos;
   }
   current_ = next_;
   if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) {
     next_ = next_next_;
     next_next_.token = Token::UNINITIALIZED;
+    has_preceding_line_terminator_ = next_has_preceding_line_terminator_;
     return current_.token;
   }
-  has_line_terminator_before_next_ = false;
-  has_multiline_comment_before_next_ = false;
+  has_preceding_line_terminator_ = false;
+  has_preceding_multiline_comment_ = false;
   if (static_cast<unsigned>(c0_) <= 0x7f) {
     Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]);
     if (token != Token::ILLEGAL) {
       int pos = source_pos();
       next_.token = token;
       next_.location.beg_pos = pos;
       next_.location.end_pos = pos + 1;
       Advance();
       return current_.token;
     }
   }
   Scan();
   return current_.token;
 }
 
 
 Token::Value Scanner::PeekAhead() {
   if (next_next_.token != Token::UNINITIALIZED) {
     return next_next_.token;
   }
   TokenDesc prev = current_;
+  bool has_preceding_line_terminator =
+      has_preceding_line_terminator_ || has_preceding_multiline_comment_;

[Dan Ehrenberg, 2016/05/12 19:24:32]

What does has_preceding_line_terminator actually mean after this change? It
would be nice to document how it all works currently, in case something else
comes along that wants to refer to it.

[caitp (gmail), 2016/05/12 19:56:05]

There's no real difference in how it currently works, it's just renamed to
something that makes more sense to me. I can change it back (and restore the old
comments that were modified).

   Next();
+  next_has_preceding_line_terminator_ =
+      has_preceding_line_terminator_ || has_preceding_multiline_comment_;
+  has_preceding_line_terminator_ = has_preceding_line_terminator;
   Token::Value ret = next_.token;
   next_next_ = next_;
   next_ = current_;
   current_ = prev;
   return ret;
 }
 
 
 // TODO(yangguo): check whether this is actually necessary.
 static inline bool IsLittleEndianByteOrderMark(uc32 c) {
   // The Unicode value U+FFFE is guaranteed never to be assigned as a
   // Unicode character; this implies that in a Unicode context the
   // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF
   // character expressed in little-endian byte order (since it could
   // not be a U+FFFE character expressed in big-endian byte
   // order). Nevertheless, we check for it to be compatible with
   // Spidermonkey.
   return c == 0xFFFE;
 }
 
 
 bool Scanner::SkipWhiteSpace() {
   int start_position = source_pos();
-
   while (true) {
     while (true) {
       // The unicode cache accepts unsigned inputs.
       if (c0_ < 0) break;
       // Advance as long as character is a WhiteSpace or LineTerminator.
       // Remember if the latter is the case.
       if (unicode_cache_->IsLineTerminator(c0_)) {
-        has_line_terminator_before_next_ = true;
+        has_preceding_line_terminator_ = true;
       } else if (!unicode_cache_->IsWhiteSpace(c0_) &&
                  !IsLittleEndianByteOrderMark(c0_)) {
         break;
       }
       Advance();
     }
 
     // If there is an HTML comment end '-->' at the beginning of a
     // line (with only whitespace in front of it), we treat the rest
     // of the line as a comment. This is in line with the way
     // SpiderMonkey handles it.
-    if (c0_ == '-' && has_line_terminator_before_next_) {
+    if (c0_ == '-' && has_preceding_line_terminator_) {
       Advance();
       if (c0_ == '-') {
         Advance();
         if (c0_ == '>') {
           // Treat the rest of the line as a comment.
           SkipSingleLineComment();
           // Continue skipping white space after the comment.
           continue;
         }
         PushBack('-');  // undo Advance()
@@ skipping 86 matching lines @@
 Token::Value Scanner::SkipMultiLineComment() {
   DCHECK(c0_ == '*');
   Advance();
 
   while (c0_ >= 0) {
     uc32 ch = c0_;
     Advance();
     if (c0_ >= 0 && unicode_cache_->IsLineTerminator(ch)) {
       // Following ECMA-262, section 7.4, a comment containing
       // a newline will make the comment count as a line-terminator.
-      has_multiline_comment_before_next_ = true;
+      has_preceding_multiline_comment_ = true;
     }
     // If we have reached the end of the multi-line comment, we
     // consume the '/' and insert a whitespace. This way all
     // multi-line comments are treated as whitespace.
     if (ch == '*' && c0_ == '/') {
       c0_ = ' ';
       return Token::WHITESPACE;
     }
   }
 
@@ skipping 17 matching lines @@
   PushBack('!');  // undo Advance()
   DCHECK(c0_ == '!');
   return Token::LT;
 }
 
 
 void Scanner::Scan() {
   next_.literal_chars = NULL;
   next_.raw_literal_chars = NULL;
   Token::Value token;
+
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
 
     switch (c0_) {
       case ' ':
       case '\t':
         Advance();
         token = Token::WHITESPACE;
         break;
 
       case '\n':
         Advance();
-        has_line_terminator_before_next_ = true;
+        has_preceding_line_terminator_ = true;
         token = Token::WHITESPACE;
         break;
 
       case '"': case '\'':
         token = ScanString();
         break;
 
       case '<':
         // < <= << <<= <!--
         Advance();
@@ skipping 60 matching lines @@
         } else {
           token = Token::ADD;
         }
         break;
 
       case '-':
         // - -- --> -=
         Advance();
         if (c0_ == '-') {
           Advance();
-          if (c0_ == '>' && has_line_terminator_before_next_) {
+          if (c0_ == '>' && has_preceding_line_terminator_) {
             // For compatibility with SpiderMonkey, we skip lines that
             // start with an HTML comment end '-->'.
             token = SkipSingleLineComment();
           } else {
             token = Token::DEC;
           }
         } else if (c0_ == '=') {
           token = Select(Token::ASSIGN_SUB);
         } else {
           token = Token::SUB;
@@ skipping 165 matching lines @@
   int current_pos = source_pos();
   DCHECK_EQ(next_.location.end_pos, current_pos);
   // Positions inside the lookahead token aren't supported.
   DCHECK(pos >= current_pos);
   if (pos != current_pos) {
     source_->SeekForward(pos - source_->pos());
     Advance();
     // This function is only called to seek to the location
     // of the end of a function (at the "}" token). It doesn't matter
     // whether there was a line terminator in the part we skip.
-    has_line_terminator_before_next_ = false;
-    has_multiline_comment_before_next_ = false;
+    has_preceding_line_terminator_ = false;
+    has_preceding_multiline_comment_ = false;
   }
   Scan();
 }
 
 
 template <bool capture_raw, bool in_template_literal>
 bool Scanner::ScanEscape() {
   uc32 c = c0_;
   Advance<capture_raw>();
 
@@ skipping 371 matching lines @@
   const bool unicode = true;
   return ScanHexNumber<capture_raw, unicode>(4);
 }
 
 
 // ----------------------------------------------------------------------------
 // Keyword Matcher
 
 #define KEYWORDS(KEYWORD_GROUP, KEYWORD)                    \
   KEYWORD_GROUP('a')                                        \
+  KEYWORD("async", Token::ASYNC)                            \
   KEYWORD("await", Token::AWAIT)                            \
   KEYWORD_GROUP('b')                                        \
   KEYWORD("break", Token::BREAK)                            \
   KEYWORD_GROUP('c')                                        \
   KEYWORD("case", Token::CASE)                              \
   KEYWORD("catch", Token::CATCH)                            \
   KEYWORD("class", Token::CLASS)                            \
   KEYWORD("const", Token::CONST)                            \
   KEYWORD("continue", Token::CONTINUE)                      \
   KEYWORD_GROUP('d')                                        \
@@ skipping 313 matching lines @@
 }
 
 
 const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) {
   if (is_next_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(next_literal_one_byte_string());
   }
   return ast_value_factory->GetTwoByteString(next_literal_two_byte_string());
 }
 
+const AstRawString* Scanner::NextNextSymbol(
+    AstValueFactory* ast_value_factory) {
+  DCHECK(next_next_.token != Token::UNINITIALIZED);
+  LiteralBuffer* literal = next_next_.literal_chars;
+  if (literal->is_one_byte()) {
+    return ast_value_factory->GetOneByteString(literal->one_byte_literal());
+  }
+  return ast_value_factory->GetTwoByteString(literal->two_byte_literal());
+}
 
 const AstRawString* Scanner::CurrentRawSymbol(
     AstValueFactory* ast_value_factory) {
   if (is_raw_literal_one_byte()) {
     return ast_value_factory->GetOneByteString(raw_literal_one_byte_string());
   }
   return ast_value_factory->GetTwoByteString(raw_literal_two_byte_string());
 }
 
 
@@ skipping 205 matching lines @@
     backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u));
   }
   backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f));
 
   backing_store_.AddBlock(bytes);
   return backing_store_.EndSequence().start();
 }
 
 }  // namespace internal
 }  // namespace v8