WIP [esnext] implement async iteration proposal

src/parsing/parser-base.h

Index: src/parsing/parser-base.h
diff --git a/src/parsing/parser-base.h b/src/parsing/parser-base.h
index d8705a654fd535d95fbe5fdcea017852e9241f40..cf355e524b31dcf88213689296de499d371c4686 100644
--- a/src/parsing/parser-base.h
+++ b/src/parsing/parser-base.h
@@ -218,7 +218,8 @@ class ParserBase {
         allow_harmony_restrictive_generators_(false),
         allow_harmony_trailing_commas_(false),
         allow_harmony_class_fields_(false),
-        allow_harmony_object_spread_(false) {}
+        allow_harmony_object_spread_(false),
+        allow_harmony_async_iteration_(false) {}
 
 #define ALLOW_ACCESSORS(name)                           \
   bool allow_##name() const { return allow_##name##_; } \
@@ -232,6 +233,7 @@ class ParserBase {
   ALLOW_ACCESSORS(harmony_trailing_commas);
   ALLOW_ACCESSORS(harmony_class_fields);
   ALLOW_ACCESSORS(harmony_object_spread);
+  ALLOW_ACCESSORS(harmony_async_iteration);
 
 #undef ALLOW_ACCESSORS
 
@@ -942,6 +944,9 @@ class ParserBase {
   bool is_async_function() const {
     return IsAsyncFunction(function_state_->kind());
   }
+  bool is_async_generator() const {
+    return IsAsyncGeneratorFunction(function_state_->kind());
+  }
   bool is_resumable() const {
     return IsResumableFunction(function_state_->kind());
   }
@@ -1289,6 +1294,16 @@ class ParserBase {
                                   bool* ok);
   StatementT ParseTryStatement(bool* ok);
   StatementT ParseForStatement(ZoneList<const AstRawString*>* labels, bool* ok);
+  StatementT ParseForAwaitStatement(ZoneList<const AstRawString*>* labels,
+                                    bool* ok);
+  V8_INLINE StatementT ParseForStatementOrForAwaitStatement(
+      ZoneList<const AstRawString*>* labels, bool* ok) {
+    if (V8_UNLIKELY(allow_harmony_async_iteration() && is_async_function() &&
+                    PeekAhead() == Token::AWAIT)) {
+      return ParseForAwaitStatement(labels, ok);
+    }
+    return ParseForStatement(labels, ok);
+  }
 
   bool IsNextLetKeyword();
   bool IsTrivialExpression();
@@ -1457,6 +1472,7 @@ class ParserBase {
   bool allow_harmony_trailing_commas_;
   bool allow_harmony_class_fields_;
   bool allow_harmony_object_spread_;
+  bool allow_harmony_async_iteration_;
 
   friend class DiscardableZoneScope;
 };
@@ -2047,7 +2063,12 @@ typename ParserBase<Impl>::ExpressionT ParserBase<Impl>::ParsePropertyName(
       !scanner()->HasAnyLineTerminatorAfterNext()) {
     Consume(Token::ASYNC);
     token = peek();
-    if (SetPropertyKindFromToken(token, kind)) {
+    if (token == Token::MUL && allow_harmony_async_iteration() &&

[Dan Ehrenberg, 2017/01/13 19:41:09]

Style nit: Put the flag check first

[caitp, 2017/01/13 20:11:52]

Acknowledged.

+        !scanner()->HasAnyLineTerminatorBeforeNext()) {
+      Consume(Token::MUL);
+      token = peek();
+      *is_generator = true;
+    } else if (SetPropertyKindFromToken(token, kind)) {
       *name = impl()->GetSymbol();  // TODO(bakkot) specialize on 'async'
       impl()->PushLiteralName(*name);
       return factory()->NewStringLiteral(*name, pos);
@@ -2454,15 +2475,22 @@ ParserBase<Impl>::ParseObjectPropertyDefinition(ObjectLiteralChecker* checker,
       // MethodDefinition
       //    PropertyName '(' StrictFormalParameters ')' '{' FunctionBody '}'
       //    '*' PropertyName '(' StrictFormalParameters ')' '{' FunctionBody '}'
+      //    async PropertyName '(' StrictFormalParameters ')'
+      //        '{' FunctionBody '}'
+      //    async '*' PropertyName '(' StrictFormalParameters ')'
+      //        '{' FunctionBody '}'
 
       classifier()->RecordPatternError(
           Scanner::Location(next_beg_pos, scanner()->location().end_pos),
           MessageTemplate::kInvalidDestructuringTarget);
 
-      FunctionKind kind = is_generator
-                              ? FunctionKind::kConciseGeneratorMethod
-                              : is_async ? FunctionKind::kAsyncConciseMethod
-                                         : FunctionKind::kConciseMethod;
+      static const FunctionKind kMethodKinds[] = {
+          FunctionKind::kConciseMethod, FunctionKind::kConciseGeneratorMethod,
+          FunctionKind::kAsyncConciseMethod,
+          FunctionKind::kAsyncConciseGeneratorMethod};
+      int index = static_cast<int>(!!is_generator) +
+                  (static_cast<int>(!!is_async) << 1);
+      FunctionKind kind = kMethodKinds[index];

[Dan Ehrenberg, 2017/01/13 19:41:09]

Nit: Isn't this what a two-dimensional array is for?

static const FunctionKind kMethodKinds[2][2] = {
  { FunctionKind::kConciseMethod, FunctionKind::kConciseGeneratorMethod },
  { FunctionKind::kAsyncConciseMethod,
FunctionKind::kAsyncConciseGeneratorMethod }
};
FunctionKind kind = kMethodKinds[is_async][is_generator];

 
       ExpressionT value = impl()->ParseFunctionLiteral(
           name, scanner()->location(), kSkipFunctionNameCheck, kind,
@@ -2888,7 +2916,12 @@ typename ParserBase<Impl>::ExpressionT ParserBase<Impl>::ParseYieldExpression(
     return impl()->RewriteYieldStar(generator_object, expression, pos);
   }
 
-  expression = impl()->BuildIteratorResult(expression, false);
+  if (!is_async_generator()) {
+    // Async generator yield is rewritten in Ignition, and doesn't require
+    // producing an Iterator Result.
+    expression = impl()->BuildIteratorResult(expression, false);
+  }
+
   // Hackily disambiguate o from o.next and o [Symbol.iterator]().
   // TODO(verwaest): Come up with a better solution.
   ExpressionT yield = factory()->NewYield(generator_object, expression, pos,
@@ -3751,10 +3784,15 @@ ParserBase<Impl>::ParseHoistableDeclaration(
   //
   // 'function' and '*' (if present) have been consumed by the caller.
 
-  const bool is_generator = flags & ParseFunctionFlags::kIsGenerator;
+  bool is_generator = flags & ParseFunctionFlags::kIsGenerator;
   const bool is_async = flags & ParseFunctionFlags::kIsAsync;
   DCHECK(!is_generator || !is_async);
 
+  if (allow_harmony_async_iteration() && is_async && Check(Token::MUL)) {
+    // Async generator
+    is_generator = true;
+  }
+
   IdentifierT name;
   FunctionNameValidity name_validity;
   IdentifierT variable_name;
@@ -3772,12 +3810,17 @@ ParserBase<Impl>::ParseHoistableDeclaration(
 
   FuncNameInferrer::State fni_state(fni_);
   impl()->PushEnclosingName(name);
+
+  static const FunctionKind kFunctionKinds[] = {
+      FunctionKind::kNormalFunction, FunctionKind::kGeneratorFunction,
+      FunctionKind::kAsyncFunction, FunctionKind::kAsyncGeneratorFunction};
+  int index =
+      static_cast<int>(!!is_generator) + (static_cast<int>(!!is_async) << 1);
+  FunctionKind kind = kFunctionKinds[index];

[Dan Ehrenberg, 2017/01/13 22:09:02]

And given the multiple duplicated copies, seems like you should lift this into a
common definition.

+
   FunctionLiteralT function = impl()->ParseFunctionLiteral(
-      name, scanner()->location(), name_validity,
-      is_generator ? FunctionKind::kGeneratorFunction
-                   : is_async ? FunctionKind::kAsyncFunction
-                              : FunctionKind::kNormalFunction,
-      pos, FunctionLiteral::kDeclaration, language_mode(),
+      name, scanner()->location(), name_validity, kind, pos,
+      FunctionLiteral::kDeclaration, language_mode(),
       CHECK_OK_CUSTOM(NullStatement));
 
   return impl()->DeclareFunction(variable_name, function, pos, is_generator,
@@ -4231,16 +4274,23 @@ ParserBase<Impl>::ParseAsyncFunctionLiteral(bool* ok) {
   IdentifierT name = impl()->EmptyIdentifier();
   FunctionLiteral::FunctionType type = FunctionLiteral::kAnonymousExpression;
 
+  bool is_generator = allow_harmony_async_iteration() && Check(Token::MUL);
+
+  static const FunctionKind kFunctionKinds[] = {
+      FunctionKind::kAsyncFunction, FunctionKind::kAsyncGeneratorFunction,
+  };
+  FunctionKind kind = kFunctionKinds[!!is_generator];
+
   if (peek_any_identifier()) {
     type = FunctionLiteral::kNamedExpression;
-    name = ParseIdentifierOrStrictReservedWord(FunctionKind::kAsyncFunction,
-                                               &is_strict_reserved, CHECK_OK);
+    name = ParseIdentifierOrStrictReservedWord(kind, &is_strict_reserved,
+                                               CHECK_OK);
   }
   return impl()->ParseFunctionLiteral(
       name, scanner()->location(),
       is_strict_reserved ? kFunctionNameIsStrictReserved
                          : kFunctionNameValidityUnknown,
-      FunctionKind::kAsyncFunction, pos, type, language_mode(), CHECK_OK);
+      kind, pos, type, language_mode(), CHECK_OK);
 }
 
 template <typename Impl>
@@ -4601,7 +4651,7 @@ typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseStatement(
     case Token::WHILE:
       return ParseWhileStatement(labels, ok);
     case Token::FOR:
-      return ParseForStatement(labels, ok);
+      return ParseForStatementOrForAwaitStatement(labels, ok);
     case Token::CONTINUE:
     case Token::BREAK:
     case Token::RETURN:
@@ -5478,6 +5528,122 @@ typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseForStatement(
   }
 }
 
+template <typename Impl>
+typename ParserBase<Impl>::StatementT ParserBase<Impl>::ParseForAwaitStatement(
+    ZoneList<const AstRawString*>* labels, bool* ok) {
+  // for await '(' ForDeclaration of AssignmentExpression ')' Statement
+  DCHECK(is_async_function());
+
+  int stmt_pos = peek_position();
+  bool bound_names_are_lexical = false;
+
+  ForInfo for_info(this);
+  BlockState for_state(zone(), &scope_state_);
+
+  Expect(Token::FOR, CHECK_OK);
+  Expect(Token::AWAIT, CHECK_OK);
+  Expect(Token::LPAREN, CHECK_OK);
+
+  /*
+   ZoneList<const AstRawString*> bound_names;
+   ForEachStatement::VisitMode mode;
+   int position;
+   DeclarationParsingResult parsing_result;
+   */
+  for_info.mode = ForEachStatement::ASYNC_ITERATE;
+  for_state.set_start_position(scanner()->location().beg_pos);
+  for_state.set_is_hidden();
+
+  StatementT init = impl()->NullStatement();
+
+  bool has_declarations = false;
+  bool is_destructuring = false;
+
+  if (peek() == Token::VAR || peek() == Token::CONST ||
+      (peek() == Token::LET && IsNextLetKeyword())) {
+    // The initializer contains declarations.
+    has_declarations = true;
+    ParseVariableDeclarations(kForStatement, &for_info.parsing_result, nullptr,
+                              CHECK_OK);
+    bound_names_are_lexical =
+        IsLexicalVariableMode(for_info.parsing_result.descriptor.mode);
+    for_info.position = scanner()->location().beg_pos;
+
+    // Just one declaration followed by in/of.
+    if (for_info.parsing_result.declarations.length() != 1) {
+      impl()->ReportMessageAt(for_info.parsing_result.bindings_loc,
+                              MessageTemplate::kForInOfLoopMultiBindings,
+                              ForEachStatement::VisitModeString(for_info.mode));
+      *ok = false;
+      return impl()->NullStatement();
+    }
+  } else {
+    // The initializer does not contain declarations.
+    int lhs_beg_pos = peek_position();
+    ExpressionClassifier classifier(this);
+    ExpressionT expression = ParseExpressionCoverGrammar(false, CHECK_OK);
+    int lhs_end_pos = scanner()->location().end_pos;
+
+    if (expression->IsArrayLiteral() || expression->IsObjectLiteral()) {
+      ValidateAssignmentPattern(CHECK_OK);
+    } else {
+      impl()->RewriteNonPattern(CHECK_OK);
+      expression = impl()->CheckAndRewriteReferenceExpression(
+          expression, lhs_beg_pos, lhs_end_pos,
+          MessageTemplate::kInvalidLhsInFor, kSyntaxError, CHECK_OK);
+    }
+  }
+
+  ExpectContextualKeyword(CStrVector("of"), CHECK_OK);
+  int each_keyword_pos = scanner()->location().beg_pos;
+
+  ExpressionClassifier classifier(this);
+  ExpressionT iterable = ParseAssignmentExpression(true, CHECK_OK);
+  impl()->RewriteNonPattern(CHECK_OK);
+
+  BlockT init_block = impl()->RewriteForVarInLegacy(for_info);

[Dan Ehrenberg, 2017/01/13 22:09:02]

The grammar doesn't permit `for await (var i = j in z)`, so this line won't have
any effect.

+
+  auto loop = factory()->NewForEachStatement(for_info.mode, labels, stmt_pos);
+  typename Types::Target target(this, loop);
+
+  Expect(Token::RPAREN, CHECK_OK);
+
+  StatementT final_loop = impl()->NullStatement();
+  {
+    ReturnExprScope no_tail_calls(function_state_,
+                                  ReturnExprContext::kInsideForInOfBody);
+    BlockState block_state(zone(), &scope_state_);
+    block_state.set_start_position(scanner()->location().beg_pos);
+
+    StatementT body = ParseScopedStatement(nullptr, true, CHECK_OK);
+
+    BlockT body_block = impl()->NullBlock();
+    ExpressionT each_variable = impl()->EmptyExpression();
+    impl()->DesugarBindingInForEachStatement(&for_info, &body_block,
+                                             &each_variable, CHECK_OK);
+    body_block->statements()->Add(body, zone());
+    final_loop = impl()->InitializeForAwaitOfStatement(
+        loop, each_variable, iterable, body_block, each_keyword_pos);
+
+    block_state.set_end_position(scanner()->location().end_pos);
+    body_block->set_scope(block_state.FinalizedBlockScope());
+  }
+
+  init_block = impl()->CreateForEachStatementTDZ(init_block, for_info, ok);
+
+  for_state.set_end_position(scanner()->location().end_pos);
+  Scope* for_scope = for_state.FinalizedBlockScope();
+  // Parsed for-in loop w/ variable declarations.
+  if (!impl()->IsNullStatement(init_block)) {
+    init_block->statements()->Add(final_loop, zone());
+    init_block->set_scope(for_scope);
+    return init_block;
+  }
+
+  DCHECK_NULL(for_scope);
+  return final_loop;
+}
+
 #undef CHECK_OK
 #undef CHECK_OK_CUSTOM