[async-iteration] add support for for-await-of loops in Async Functions

src/parsing/parser-base.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef V8_PARSING_PARSER_BASE_H
 #define V8_PARSING_PARSER_BASE_H
 
 #include "src/ast/ast.h"
 #include "src/ast/scopes.h"
 #include "src/bailout-reason.h"
@@ skipping 200 matching lines @@
         stack_overflow_(false),
         default_eager_compile_hint_(FunctionLiteral::kShouldLazyCompile),
         function_literal_id_(0),
         allow_natives_(false),
         allow_tailcalls_(false),
         allow_harmony_do_expressions_(false),
         allow_harmony_function_sent_(false),
         allow_harmony_restrictive_generators_(false),
         allow_harmony_trailing_commas_(false),
         allow_harmony_class_fields_(false),
-        allow_harmony_object_rest_spread_(false) {}
+        allow_harmony_object_rest_spread_(false),
+        allow_harmony_async_iteration_(false) {}
 
 #define ALLOW_ACCESSORS(name)                           \
   bool allow_##name() const { return allow_##name##_; } \
   void set_allow_##name(bool allow) { allow_##name##_ = allow; }
 
   ALLOW_ACCESSORS(natives);
   ALLOW_ACCESSORS(tailcalls);
   ALLOW_ACCESSORS(harmony_do_expressions);
   ALLOW_ACCESSORS(harmony_function_sent);
   ALLOW_ACCESSORS(harmony_restrictive_generators);
   ALLOW_ACCESSORS(harmony_trailing_commas);
   ALLOW_ACCESSORS(harmony_class_fields);
   ALLOW_ACCESSORS(harmony_object_rest_spread);
+  ALLOW_ACCESSORS(harmony_async_iteration);
 
 #undef ALLOW_ACCESSORS
 
   uintptr_t stack_limit() const { return stack_limit_; }
 
   void set_stack_limit(uintptr_t stack_limit) { stack_limit_ = stack_limit; }
 
   void set_default_eager_compile_hint(
       FunctionLiteral::EagerCompileHint eager_compile_hint) {
     default_eager_compile_hint_ = eager_compile_hint;
@@ skipping 1038 matching lines @@
                                 bool* ok);
   StatementT ParseDoWhileStatement(ZoneList<const AstRawString*>* labels,
                                    bool* ok);
   StatementT ParseWhileStatement(ZoneList<const AstRawString*>* labels,
                                  bool* ok);
   StatementT ParseThrowStatement(bool* ok);
   StatementT ParseSwitchStatement(ZoneList<const AstRawString*>* labels,
                                   bool* ok);
   StatementT ParseTryStatement(bool* ok);
   StatementT ParseForStatement(ZoneList<const AstRawString*>* labels, bool* ok);
+  StatementT ParseForAwaitStatement(ZoneList<const AstRawString*>* labels,
+                                    bool* ok);
 
   bool IsNextLetKeyword();
   bool IsTrivialExpression();
 
   // Checks if the expression is a valid reference expression (e.g., on the
   // left-hand side of assignments). Although ruled out by ECMA as early errors,
   // we allow calls for web compatibility and rewrite them to a runtime throw.
   ExpressionT CheckAndRewriteReferenceExpression(
       ExpressionT expression, int beg_pos, int end_pos,
       MessageTemplate::Template message, bool* ok);
@@ skipping 148 matching lines @@
   int function_literal_id_;
 
   bool allow_natives_;
   bool allow_tailcalls_;
   bool allow_harmony_do_expressions_;
   bool allow_harmony_function_sent_;
   bool allow_harmony_restrictive_generators_;
   bool allow_harmony_trailing_commas_;
   bool allow_harmony_class_fields_;
   bool allow_harmony_object_rest_spread_;
+  bool allow_harmony_async_iteration_;
 
   friend class DiscardableZoneScope;
 };
 
 template <typename Impl>
 ParserBase<Impl>::FunctionState::FunctionState(
     FunctionState** function_state_stack, ScopeState** scope_stack,
     DeclarationScope* scope)
     : ScopeState(scope_stack, scope),
       next_materialized_literal_index_(0),
@@ skipping 3155 matching lines @@
     case Token::SEMICOLON:
       Next();
       return factory()->NewEmptyStatement(kNoSourcePosition);
     case Token::IF:
       return ParseIfStatement(labels, ok);
     case Token::DO:
       return ParseDoWhileStatement(labels, ok);
     case Token::WHILE:
       return ParseWhileStatement(labels, ok);
     case Token::FOR:
+      if (V8_UNLIKELY(allow_harmony_async_iteration() && is_async_function() &&
+                      PeekAhead() == Token::AWAIT)) {
+        return ParseForAwaitStatement(labels, ok);
+      }
       return ParseForStatement(labels, ok);
     case Token::CONTINUE:
     case Token::BREAK:
     case Token::RETURN:
     case Token::THROW:
     case Token::TRY: {
       // These statements must have their labels preserved in an enclosing
       // block, as the corresponding AST nodes do not currently store their
       // labels.
       // TODO(nikolaos, marja): Consider adding the labels to the AST nodes.
@@ skipping 857 matching lines @@
       block->set_scope(for_scope);
       loop->Initialize(init, cond, next, body);
       return block;
     } else {
       loop->Initialize(init, cond, next, body);
       return loop;
     }
   }
 }
 
+template <typename Impl>
+typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseForAwaitStatement(
+    ZoneList<const AstRawString*>* labels, bool* ok) {
+  // for await '(' ForDeclaration of AssignmentExpression ')'
+  DCHECK(is_async_function());
+  DCHECK(allow_harmony_async_iteration());
+
+  int stmt_pos = peek_position();
+  bool bound_names_are_lexical = false;
+
+  ForInfo for_info(this);
+  for_info.mode = ForEachStatement::ITERATE;
+
+  // Create an in-between scope for let-bound iteration variables.
+  BlockState for_state(zone(), &scope_state_);
+  Expect(Token::FOR, CHECK_OK);
+  Expect(Token::AWAIT, CHECK_OK);
+  Expect(Token::LPAREN, CHECK_OK);
+  for_state.set_start_position(scanner()->location().beg_pos);
+  for_state.set_is_hidden();
+
+  auto loop = factory()->NewForEachStatement(for_info.mode, labels, stmt_pos);
+  typename Types::Target target(this, loop);
+
+  ExpressionT each_variable = impl()->EmptyExpression();
+
+  bool has_declarations = false;
+
+  switch (peek()) {
+    case Token::LET:
+      if (!IsNextLetKeyword()) goto parse_lhs;
+    /* falls through */
+    case Token::VAR:
+    case Token::CONST: {
+      // The initializer contains declarations
+      // 'for' 'await' '(' ForDeclaration 'of' AssignmentExpression ')'
+      //     Statement
+      // 'for' 'await' '(' 'var' ForBinding 'of' AssignmentExpression ')'
+      //     Statement
+      has_declarations = true;
+      ParseVariableDeclarations(kForStatement, &for_info.parsing_result,
+                                nullptr, CHECK_OK);
+      bound_names_are_lexical =
+          IsLexicalVariableMode(for_info.parsing_result.descriptor.mode);
+
+      for_info.position = scanner()->location().beg_pos;
+
+      // Only a single declaration is allowed in for-await-of loops
+      if (for_info.parsing_result.declarations.length() != 1) {
+        impl()->ReportMessageAt(for_info.parsing_result.bindings_loc,
+                                MessageTemplate::kForInOfLoopMultiBindings,
+                                "for-await-of");
+        *ok = false;
+        return impl()->NullStatement();
+      }
+
+      // for-await-of's declarations do not permit initializers.
+      if (for_info.parsing_result.first_initializer_loc.IsValid()) {
+        impl()->ReportMessageAt(for_info.parsing_result.first_initializer_loc,
+                                MessageTemplate::kForInOfLoopInitializer,
+                                "for-await-of");
+        *ok = false;
+        return impl()->NullStatement();
+      }
+      break;
+    }
+
+    default:
+    parse_lhs : {
+      // The initializer does not contain declarations.
+      // 'for' 'await' '(' LeftHandSideExpression 'of' AssignmentExpression ')'
+      //     Statement
+      int lhs_beg_pos = peek_position();
+      ExpressionClassifier classifier(this);
+      ExpressionT lhs = each_variable = ParseLeftHandSideExpression(CHECK_OK);
+      int lhs_end_pos = scanner()->location().end_pos;
+
+      if (lhs->IsArrayLiteral() || lhs->IsObjectLiteral()) {
+        ValidateAssignmentPattern(CHECK_OK);
+      } else {
+        impl()->RewriteNonPattern(CHECK_OK);
+        each_variable = impl()->CheckAndRewriteReferenceExpression(
+            lhs, lhs_beg_pos, lhs_end_pos, MessageTemplate::kInvalidLhsInFor,
+            kSyntaxError, CHECK_OK);
+      }
+    } break;
+  }
+
+  ExpectContextualKeyword(CStrVector("of"), CHECK_OK);
+  int each_keyword_pos = scanner()->location().beg_pos;
+
+  const bool kAllowIn = true;
+  ExpressionT iterable = impl()->EmptyExpression();
+
+  {
+    ExpressionClassifier classifier(this);
+    iterable = ParseAssignmentExpression(kAllowIn, CHECK_OK);
+    impl()->RewriteNonPattern(CHECK_OK);
+  }
+
+  Expect(Token::RPAREN, CHECK_OK);
+
+  {
+    ReturnExprScope no_tail_calls(function_state_,
+                                  ReturnExprContext::kInsideForInOfBody);
+    BlockState block_state(zone(), &scope_state_);
+    block_state.set_start_position(scanner()->location().beg_pos);
+
+    // For legacy compat reasons, give for loops similar treatment to
+    // if statements in allowing a function declaration for a body
+    StatementT body = ParseScopedStatement(nullptr, true, CHECK_OK);
+    block_state.set_end_position(scanner()->location().end_pos);
+
+    if (has_declarations) {
+      BlockT body_block = impl()->NullBlock();
+      impl()->DesugarBindingInForEachStatement(&for_info, &body_block,
+                                               &each_variable, CHECK_OK);
+      body_block->statements()->Add(body, zone());
+      body_block->set_scope(block_state.FinalizedBlockScope());
+      for_state.set_end_position(scanner()->location().end_pos);
+
+      const bool finalize = true;
+      StatementT final_loop = impl()->InitializeForOfStatement(
+          loop, each_variable, iterable, body_block, finalize,
+          IteratorType::kAsync, each_keyword_pos);
+
+      BlockT init_block =
+          impl()->CreateForEachStatementTDZ(impl()->NullBlock(), for_info, ok);

[caitp, 2017/01/21 16:50:33]

Nickie / Marja: can you explain why CreateForEachStatementTDZ seems to work in
ForOf statements?

It looks like it always redeclares variables that were declared previously in
ParseVariableDeclaration (looks like the same scope in both functions), which is
always an error for lexical variables. Removing it seems to make this work as
expected, although it might be broken with eval(), not sure.

[marja, 2017/01/23 05:34:10]

Since I didn't know the answer upfront, I used the opportunity to poke the code
a bit to understand it better. Here's how ParseForStatement works:

- The variable is indeed declared twice, but not in the same scope, but two
different scopes:
- ParseVariableDeclarations doesn't call DeclareAndInitializeVariables, since
it's a for context.
- The first declaration is done by DesugarBindingInForEachStatement and a couple
of lines above that you see a BlockState which creates a new scope, so, it
declares the variable in that scope.
- The second declaration is done by CreateForEachStatementTDZ, and it uses the
parent scope (like explained in the comment above
Parser::DesugarBindingInForEachStatement).

Your problem seems to be that you're calling both
DesugarBindingInForEachStatement and CreateForEachStatementTDZ in the same scope
(not creating a new BlockState for the former).

+
+      Scope* for_scope = for_state.FinalizedBlockScope();
+      // Parsed for-in loop w/ variable declarations.
+      if (!impl()->IsNullStatement(init_block)) {
+        init_block->statements()->Add(final_loop, zone());
+        init_block->set_scope(for_scope);
+        return init_block;
+      } else {
+        DCHECK_NULL(for_scope);
+        return final_loop;
+      }
+    } else {
+      const bool finalize = true;
+      StatementT final_loop = impl()->InitializeForOfStatement(
+          loop, each_variable, iterable, body, finalize, IteratorType::kAsync,
+          each_keyword_pos);
+
+      Scope* for_scope = for_state.FinalizedBlockScope();
+      DCHECK_NULL(for_scope);
+      USE(for_scope);
+      Scope* block_scope = block_state.FinalizedBlockScope();
+      DCHECK_NULL(block_scope);
+      USE(block_scope);
+      return final_loop;
+    }
+
+    UNREACHABLE();
+    return impl()->NullStatement();
+  }
+}
+
 #undef CHECK_OK
 #undef CHECK_OK_CUSTOM
 
 template <typename Impl>
 void ParserBase<Impl>::ObjectLiteralChecker::CheckDuplicateProto(
     Token::Value property) {
   if (property == Token::SMI || property == Token::NUMBER) return;
 
   if (IsProto()) {
     if (has_seen_proto_) {
@@ skipping 38 matching lines @@
     has_seen_constructor_ = true;
     return;
   }
 }
 
 
 }  // namespace internal
 }  // namespace v8
 
 #endif  // V8_PARSING_PARSER_BASE_H