[async-iteration] add support for for-await-of loops in Async Functions

src/parsing/parser-base.h

Index: src/parsing/parser-base.h
diff --git a/src/parsing/parser-base.h b/src/parsing/parser-base.h
index b17d664b3abe099d90f65d67a15f7c3e616bbc5a..8413055c42e665afa5a1bdb636dbdf991d138cad 100644
--- a/src/parsing/parser-base.h
+++ b/src/parsing/parser-base.h
@@ -218,7 +218,8 @@ class ParserBase {
         allow_harmony_restrictive_generators_(false),
         allow_harmony_trailing_commas_(false),
         allow_harmony_class_fields_(false),
-        allow_harmony_object_rest_spread_(false) {}
+        allow_harmony_object_rest_spread_(false),
+        allow_harmony_async_iteration_(false) {}
 
 #define ALLOW_ACCESSORS(name)                           \
   bool allow_##name() const { return allow_##name##_; } \
@@ -232,6 +233,7 @@ class ParserBase {
   ALLOW_ACCESSORS(harmony_trailing_commas);
   ALLOW_ACCESSORS(harmony_class_fields);
   ALLOW_ACCESSORS(harmony_object_rest_spread);
+  ALLOW_ACCESSORS(harmony_async_iteration);
 
 #undef ALLOW_ACCESSORS
 
@@ -1290,6 +1292,8 @@ class ParserBase {
                                   bool* ok);
   StatementT ParseTryStatement(bool* ok);
   StatementT ParseForStatement(ZoneList<const AstRawString*>* labels, bool* ok);
+  StatementT ParseForAwaitStatement(ZoneList<const AstRawString*>* labels,
+                                    bool* ok);
 
   bool IsNextLetKeyword();
   bool IsTrivialExpression();
@@ -1458,6 +1462,7 @@ class ParserBase {
   bool allow_harmony_trailing_commas_;
   bool allow_harmony_class_fields_;
   bool allow_harmony_object_rest_spread_;
+  bool allow_harmony_async_iteration_;
 
   friend class DiscardableZoneScope;
 };
@@ -4633,6 +4638,10 @@ typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseStatement(
     case Token::WHILE:
       return ParseWhileStatement(labels, ok);
     case Token::FOR:
+      if (V8_UNLIKELY(allow_harmony_async_iteration() && is_async_function() &&
+                      PeekAhead() == Token::AWAIT)) {
+        return ParseForAwaitStatement(labels, ok);
+      }
       return ParseForStatement(labels, ok);
     case Token::CONTINUE:
     case Token::BREAK:
@@ -5510,6 +5519,165 @@ typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseForStatement(
   }
 }
 
+template <typename Impl>
+typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseForAwaitStatement(
+    ZoneList<const AstRawString*>* labels, bool* ok) {
+  // for await '(' ForDeclaration of AssignmentExpression ')'
+  DCHECK(is_async_function());
+  DCHECK(allow_harmony_async_iteration());
+
+  int stmt_pos = peek_position();
+  bool bound_names_are_lexical = false;
+
+  ForInfo for_info(this);
+  for_info.mode = ForEachStatement::ITERATE;
+
+  // Create an in-between scope for let-bound iteration variables.
+  BlockState for_state(zone(), &scope_state_);
+  Expect(Token::FOR, CHECK_OK);
+  Expect(Token::AWAIT, CHECK_OK);
+  Expect(Token::LPAREN, CHECK_OK);
+  for_state.set_start_position(scanner()->location().beg_pos);
+  for_state.set_is_hidden();
+
+  auto loop = factory()->NewForEachStatement(for_info.mode, labels, stmt_pos);
+  typename Types::Target target(this, loop);
+
+  ExpressionT each_variable = impl()->EmptyExpression();
+
+  bool has_declarations = false;
+
+  switch (peek()) {
+    case Token::LET:
+      if (!IsNextLetKeyword()) goto parse_lhs;
+    /* falls through */
+    case Token::VAR:
+    case Token::CONST: {
+      // The initializer contains declarations
+      // 'for' 'await' '(' ForDeclaration 'of' AssignmentExpression ')'
+      //     Statement
+      // 'for' 'await' '(' 'var' ForBinding 'of' AssignmentExpression ')'
+      //     Statement
+      has_declarations = true;
+      ParseVariableDeclarations(kForStatement, &for_info.parsing_result,
+                                nullptr, CHECK_OK);
+      bound_names_are_lexical =
+          IsLexicalVariableMode(for_info.parsing_result.descriptor.mode);
+
+      for_info.position = scanner()->location().beg_pos;
+
+      // Only a single declaration is allowed in for-await-of loops
+      if (for_info.parsing_result.declarations.length() != 1) {
+        impl()->ReportMessageAt(for_info.parsing_result.bindings_loc,
+                                MessageTemplate::kForInOfLoopMultiBindings,
+                                "for-await-of");
+        *ok = false;
+        return impl()->NullStatement();
+      }
+
+      // for-await-of's declarations do not permit initializers.
+      if (for_info.parsing_result.first_initializer_loc.IsValid()) {
+        impl()->ReportMessageAt(for_info.parsing_result.first_initializer_loc,
+                                MessageTemplate::kForInOfLoopInitializer,
+                                "for-await-of");
+        *ok = false;
+        return impl()->NullStatement();
+      }
+      break;
+    }
+
+    default:
+    parse_lhs : {
+      // The initializer does not contain declarations.
+      // 'for' 'await' '(' LeftHandSideExpression 'of' AssignmentExpression ')'
+      //     Statement
+      int lhs_beg_pos = peek_position();
+      ExpressionClassifier classifier(this);
+      ExpressionT lhs = each_variable = ParseLeftHandSideExpression(CHECK_OK);
+      int lhs_end_pos = scanner()->location().end_pos;
+
+      if (lhs->IsArrayLiteral() || lhs->IsObjectLiteral()) {
+        ValidateAssignmentPattern(CHECK_OK);
+      } else {
+        impl()->RewriteNonPattern(CHECK_OK);
+        each_variable = impl()->CheckAndRewriteReferenceExpression(
+            lhs, lhs_beg_pos, lhs_end_pos, MessageTemplate::kInvalidLhsInFor,
+            kSyntaxError, CHECK_OK);
+      }
+    } break;
+  }
+
+  ExpectContextualKeyword(CStrVector("of"), CHECK_OK);
+  int each_keyword_pos = scanner()->location().beg_pos;
+
+  const bool kAllowIn = true;
+  ExpressionT iterable = impl()->EmptyExpression();
+
+  {
+    ExpressionClassifier classifier(this);
+    iterable = ParseAssignmentExpression(kAllowIn, CHECK_OK);
+    impl()->RewriteNonPattern(CHECK_OK);
+  }
+
+  Expect(Token::RPAREN, CHECK_OK);
+
+  {
+    ReturnExprScope no_tail_calls(function_state_,
+                                  ReturnExprContext::kInsideForInOfBody);
+    BlockState block_state(zone(), &scope_state_);
+    block_state.set_start_position(scanner()->location().beg_pos);
+
+    // For legacy compat reasons, give for loops similar treatment to
+    // if statements in allowing a function declaration for a body
+    StatementT body = ParseScopedStatement(nullptr, true, CHECK_OK);
+    block_state.set_end_position(scanner()->location().end_pos);
+
+    if (has_declarations) {
+      BlockT body_block = impl()->NullBlock();
+      impl()->DesugarBindingInForEachStatement(&for_info, &body_block,
+                                               &each_variable, CHECK_OK);
+      body_block->statements()->Add(body, zone());
+      body_block->set_scope(block_state.FinalizedBlockScope());
+      for_state.set_end_position(scanner()->location().end_pos);
+
+      const bool finalize = true;
+      StatementT final_loop = impl()->InitializeForOfStatement(
+          loop, each_variable, iterable, body_block, finalize,
+          IteratorType::kAsync, each_keyword_pos);
+
+      BlockT init_block =
+          impl()->CreateForEachStatementTDZ(impl()->NullBlock(), for_info, ok);

[caitp, 2017/01/21 16:50:33]

Nickie / Marja: can you explain why CreateForEachStatementTDZ seems to work in
ForOf statements?

It looks like it always redeclares variables that were declared previously in
ParseVariableDeclaration (looks like the same scope in both functions), which is
always an error for lexical variables. Removing it seems to make this work as
expected, although it might be broken with eval(), not sure.

[marja, 2017/01/23 05:34:10]

Since I didn't know the answer upfront, I used the opportunity to poke the code
a bit to understand it better. Here's how ParseForStatement works:

- The variable is indeed declared twice, but not in the same scope, but two
different scopes:
- ParseVariableDeclarations doesn't call DeclareAndInitializeVariables, since
it's a for context.
- The first declaration is done by DesugarBindingInForEachStatement and a couple
of lines above that you see a BlockState which creates a new scope, so, it
declares the variable in that scope.
- The second declaration is done by CreateForEachStatementTDZ, and it uses the
parent scope (like explained in the comment above
Parser::DesugarBindingInForEachStatement).

Your problem seems to be that you're calling both
DesugarBindingInForEachStatement and CreateForEachStatementTDZ in the same scope
(not creating a new BlockState for the former).

+
+      Scope* for_scope = for_state.FinalizedBlockScope();
+      // Parsed for-in loop w/ variable declarations.
+      if (!impl()->IsNullStatement(init_block)) {
+        init_block->statements()->Add(final_loop, zone());
+        init_block->set_scope(for_scope);
+        return init_block;
+      } else {
+        DCHECK_NULL(for_scope);
+        return final_loop;
+      }
+    } else {
+      const bool finalize = true;
+      StatementT final_loop = impl()->InitializeForOfStatement(
+          loop, each_variable, iterable, body, finalize, IteratorType::kAsync,
+          each_keyword_pos);
+
+      Scope* for_scope = for_state.FinalizedBlockScope();
+      DCHECK_NULL(for_scope);
+      USE(for_scope);
+      Scope* block_scope = block_state.FinalizedBlockScope();
+      DCHECK_NULL(block_scope);
+      USE(block_scope);
+      return final_loop;
+    }
+
+    UNREACHABLE();
+    return impl()->NullStatement();
+  }
+}
+
 #undef CHECK_OK
 #undef CHECK_OK_CUSTOM