[esnext] implement frontend changes for async/await proposal

src/parsing/scanner.cc

Index: src/parsing/scanner.cc
diff --git a/src/parsing/scanner.cc b/src/parsing/scanner.cc
index faec88b8a4bfb5ff37bf8dad58fefc32cd8ff03f..2453b066b940ce2896728d8fe9b436321d4c1b9e 100644
--- a/src/parsing/scanner.cc
+++ b/src/parsing/scanner.cc
@@ -56,7 +56,8 @@ void Scanner::Initialize(Utf16CharacterStream* source) {
   Init();
   // Skip initial whitespace allowing HTML comment ends just like
   // after a newline and scan first token.
-  has_line_terminator_before_next_ = true;
+  has_preceding_line_terminator_ = true;
+  has_preceding_multiline_comment_ = false;
   SkipWhiteSpace();
   Scan();
 }
@@ -249,10 +250,11 @@ Token::Value Scanner::Next() {
   if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) {
     next_ = next_next_;
     next_next_.token = Token::UNINITIALIZED;
+    has_preceding_line_terminator_ = next_has_preceding_line_terminator_;
     return current_.token;
   }
-  has_line_terminator_before_next_ = false;
-  has_multiline_comment_before_next_ = false;
+  has_preceding_line_terminator_ = false;
+  has_preceding_multiline_comment_ = false;
   if (static_cast<unsigned>(c0_) <= 0x7f) {
     Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]);
     if (token != Token::ILLEGAL) {
@@ -274,7 +276,12 @@ Token::Value Scanner::PeekAhead() {
     return next_next_.token;
   }
   TokenDesc prev = current_;
+  bool has_preceding_line_terminator =
+      has_preceding_line_terminator_ || has_preceding_multiline_comment_;

[Dan Ehrenberg, 2016/05/12 19:24:32]

What does has_preceding_line_terminator actually mean after this change? It
would be nice to document how it all works currently, in case something else
comes along that wants to refer to it.

[caitp (gmail), 2016/05/12 19:56:05]

There's no real difference in how it currently works, it's just renamed to
something that makes more sense to me. I can change it back (and restore the old
comments that were modified).

   Next();
+  next_has_preceding_line_terminator_ =
+      has_preceding_line_terminator_ || has_preceding_multiline_comment_;
+  has_preceding_line_terminator_ = has_preceding_line_terminator;
   Token::Value ret = next_.token;
   next_next_ = next_;
   next_ = current_;
@@ -298,7 +305,6 @@ static inline bool IsLittleEndianByteOrderMark(uc32 c) {
 
 bool Scanner::SkipWhiteSpace() {
   int start_position = source_pos();
-
   while (true) {
     while (true) {
       // The unicode cache accepts unsigned inputs.
@@ -306,7 +312,7 @@ bool Scanner::SkipWhiteSpace() {
       // Advance as long as character is a WhiteSpace or LineTerminator.
       // Remember if the latter is the case.
       if (unicode_cache_->IsLineTerminator(c0_)) {
-        has_line_terminator_before_next_ = true;
+        has_preceding_line_terminator_ = true;
       } else if (!unicode_cache_->IsWhiteSpace(c0_) &&
                  !IsLittleEndianByteOrderMark(c0_)) {
         break;
@@ -318,7 +324,7 @@ bool Scanner::SkipWhiteSpace() {
     // line (with only whitespace in front of it), we treat the rest
     // of the line as a comment. This is in line with the way
     // SpiderMonkey handles it.
-    if (c0_ == '-' && has_line_terminator_before_next_) {
+    if (c0_ == '-' && has_preceding_line_terminator_) {
       Advance();
       if (c0_ == '-') {
         Advance();
@@ -425,7 +431,7 @@ Token::Value Scanner::SkipMultiLineComment() {
     if (c0_ >= 0 && unicode_cache_->IsLineTerminator(ch)) {
       // Following ECMA-262, section 7.4, a comment containing
       // a newline will make the comment count as a line-terminator.
-      has_multiline_comment_before_next_ = true;
+      has_preceding_multiline_comment_ = true;
     }
     // If we have reached the end of the multi-line comment, we
     // consume the '/' and insert a whitespace. This way all
@@ -463,6 +469,7 @@ void Scanner::Scan() {
   next_.literal_chars = NULL;
   next_.raw_literal_chars = NULL;
   Token::Value token;
+
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
@@ -476,7 +483,7 @@ void Scanner::Scan() {
 
       case '\n':
         Advance();
-        has_line_terminator_before_next_ = true;
+        has_preceding_line_terminator_ = true;
         token = Token::WHITESPACE;
         break;
 
@@ -557,7 +564,7 @@ void Scanner::Scan() {
         Advance();
         if (c0_ == '-') {
           Advance();
-          if (c0_ == '>' && has_line_terminator_before_next_) {
+          if (c0_ == '>' && has_preceding_line_terminator_) {
             // For compatibility with SpiderMonkey, we skip lines that
             // start with an HTML comment end '-->'.
             token = SkipSingleLineComment();
@@ -743,8 +750,8 @@ void Scanner::SeekForward(int pos) {
     // This function is only called to seek to the location
     // of the end of a function (at the "}" token). It doesn't matter
     // whether there was a line terminator in the part we skip.
-    has_line_terminator_before_next_ = false;
-    has_multiline_comment_before_next_ = false;
+    has_preceding_line_terminator_ = false;
+    has_preceding_multiline_comment_ = false;
   }
   Scan();
 }
@@ -1136,6 +1143,7 @@ uc32 Scanner::ScanUnicodeEscape() {
 
 #define KEYWORDS(KEYWORD_GROUP, KEYWORD)                    \
   KEYWORD_GROUP('a')                                        \
+  KEYWORD("async", Token::ASYNC)                            \
   KEYWORD("await", Token::AWAIT)                            \
   KEYWORD_GROUP('b')                                        \
   KEYWORD("break", Token::BREAK)                            \
@@ -1469,6 +1477,15 @@ const AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) {
   return ast_value_factory->GetTwoByteString(next_literal_two_byte_string());
 }
 
+const AstRawString* Scanner::NextNextSymbol(
+    AstValueFactory* ast_value_factory) {
+  DCHECK(next_next_.token != Token::UNINITIALIZED);
+  LiteralBuffer* literal = next_next_.literal_chars;
+  if (literal->is_one_byte()) {
+    return ast_value_factory->GetOneByteString(literal->one_byte_literal());
+  }
+  return ast_value_factory->GetTwoByteString(literal->two_byte_literal());
+}
 
 const AstRawString* Scanner::CurrentRawSymbol(
     AstValueFactory* ast_value_factory) {