WIP [esnext] implement async iteration proposal

src/parsing/parser.cc

Index: src/parsing/parser.cc
diff --git a/src/parsing/parser.cc b/src/parsing/parser.cc
index cabede50d3c5c72ee205ec7f2f34f2e118408d52..20f3193bf742e07753866b3da6ab65fe5f220d28 100644
--- a/src/parsing/parser.cc
+++ b/src/parsing/parser.cc
@@ -553,6 +553,7 @@ Parser::Parser(ParseInfo* info)
   set_allow_harmony_trailing_commas(FLAG_harmony_trailing_commas);
   set_allow_harmony_class_fields(FLAG_harmony_class_fields);
   set_allow_harmony_object_spread(FLAG_harmony_object_spread);
+  set_allow_harmony_async_iteration(FLAG_harmony_async_iteration);
   for (int feature = 0; feature < v8::Isolate::kUseCounterFeatureCount;
        ++feature) {
     use_counts_[feature] = 0;
@@ -1767,7 +1768,8 @@ Statement* Parser::RewriteTryStatement(Block* try_block, Block* catch_block,
 // !%_IsJSReceiver(result = iterator.next()) &&
 //     %ThrowIteratorResultNotAnObject(result)
 Expression* Parser::BuildIteratorNextResult(Expression* iterator,
-                                            Variable* result, int pos) {
+                                            Variable* result, IteratorType type,
+                                            int pos) {
   Expression* next_literal = factory()->NewStringLiteral(
       ast_value_factory()->next_string(), kNoSourcePosition);
   Expression* next_property =
@@ -1776,6 +1778,9 @@ Expression* Parser::BuildIteratorNextResult(Expression* iterator,
       new (zone()) ZoneList<Expression*>(0, zone());
   Expression* next_call =
       factory()->NewCall(next_property, next_arguments, pos);
+  if (type == IteratorType::kAsync) {
+    next_call = RewriteAwaitExpression(next_call, pos);
+  }
   Expression* result_proxy = factory()->NewVariableProxy(result);
   Expression* left =
       factory()->NewAssignment(Token::ASSIGN, result_proxy, next_call, pos);
@@ -1996,8 +2001,8 @@ Statement* Parser::InitializeForOfStatement(ForOfStatement* for_of,
   Expression* next_result;
   {
     Expression* iterator_proxy = factory()->NewVariableProxy(iterator);
-    next_result =
-        BuildIteratorNextResult(iterator_proxy, result, next_result_pos);
+    next_result = BuildIteratorNextResult(
+        iterator_proxy, result, IteratorType::kNormal, next_result_pos);
   }
 
   // result.done
@@ -2085,7 +2090,137 @@ Statement* Parser::InitializeForOfStatement(ForOfStatement* for_of,
 
   for_of->Initialize(body, iterator, assign_iterator, next_result, result_done,
                      assign_each);
-  return finalize ? FinalizeForOfStatement(for_of, completion, nopos) : for_of;
+  return finalize ? FinalizeForOfStatement(for_of, completion,
+                                           IteratorType::kNormal, nopos)
+                  : for_of;
+}
+
+Statement* Parser::InitializeForAwaitOfStatement(ForEachStatement* stmt,
+                                                 Expression* each,
+                                                 Expression* iterable,
+                                                 Statement* body, bool finalize,
+                                                 int next_result_pos) {

[Dan Ehrenberg, 2017/01/13 22:09:02]

This is really similar to  Parser::InitializeForOfStatement . Should that just
take an IteratorType argument?

[caitp, 2017/01/13 22:33:30]

I'm undecided on this.

I think the code is easier to  understand when separated, but since i opted to
pass the IteratorType for other methods to avoid duplication, it's probably fine
to reuse it for for-of

[Dan Ehrenberg, 2017/01/14 00:48:38]

For me, it's easier to understand code when it's not duplicated, at least when
it's in a large amount like this. When there's so much copy-paste, it takes me a
while to go through everything to find what the difference is between the two
sections. At least that was my experience reviewing this patch.

+  DCHECK(stmt->IsForOfStatement());
+  ForOfStatement* for_of = stmt->AsForOfStatement();
+
+  // Create the auxiliary expressions needed for iterating over the iterable,
+  // and initialize the given ForOfStatement with them.
+  // If finalize is true, also instrument the loop with code that performs the
+  // proper ES6 iterator finalization.  In that case, the result is not
+  // immediately a ForOfStatement.
+
+  const int nopos = kNoSourcePosition;
+  auto avfactory = ast_value_factory();
+
+  Variable* iterator = NewTemporary(ast_value_factory()->dot_iterator_string());
+  Variable* result = NewTemporary(ast_value_factory()->dot_result_string());
+  Variable* completion = NewTemporary(avfactory->empty_string());
+
+  // iterator = GetIterator(iterable, async)
+  Expression* assign_iterator;
+  {
+    assign_iterator = factory()->NewAssignment(
+        Token::ASSIGN, factory()->NewVariableProxy(iterator),
+        factory()->NewGetIterator(iterable, GetIterator::kAsync,
+                                  iterable->position()),
+        iterable->position());
+  }
+
+  Expression* next_result;
+  {
+    Expression* iterator_proxy = factory()->NewVariableProxy(iterator);
+    next_result = BuildIteratorNextResult(
+        iterator_proxy, result, IteratorType::kAsync, next_result_pos);
+  }
+
+  // result.done
+  Expression* result_done;
+  {
+    Expression* done_literal = factory()->NewStringLiteral(
+        ast_value_factory()->done_string(), kNoSourcePosition);
+    Expression* result_proxy = factory()->NewVariableProxy(result);
+    result_done =
+        factory()->NewProperty(result_proxy, done_literal, kNoSourcePosition);
+  }
+
+  // result.value
+  Expression* result_value;
+  {
+    Expression* value_literal =
+        factory()->NewStringLiteral(avfactory->value_string(), nopos);
+    Expression* result_proxy = factory()->NewVariableProxy(result);
+    result_value = factory()->NewProperty(result_proxy, value_literal, nopos);
+  }
+
+  // {{completion = kAbruptCompletion;}}
+  Statement* set_completion_abrupt;
+  if (finalize) {
+    Expression* proxy = factory()->NewVariableProxy(completion);
+    Expression* assignment = factory()->NewAssignment(
+        Token::ASSIGN, proxy,
+        factory()->NewSmiLiteral(Parser::kAbruptCompletion, nopos), nopos);
+
+    Block* block = factory()->NewBlock(nullptr, 1, true, nopos);
+    block->statements()->Add(
+        factory()->NewExpressionStatement(assignment, nopos), zone());
+    set_completion_abrupt = block;
+  }
+
+  // do { let tmp = #result_value; #set_completion_abrupt; tmp }
+  // Expression* result_value (gets overwritten)
+  if (finalize) {
+    Variable* var_tmp = NewTemporary(avfactory->empty_string());
+    Expression* tmp = factory()->NewVariableProxy(var_tmp);
+    Expression* assignment =
+        factory()->NewAssignment(Token::ASSIGN, tmp, result_value, nopos);
+
+    Block* block = factory()->NewBlock(nullptr, 2, false, nopos);
+    block->statements()->Add(
+        factory()->NewExpressionStatement(assignment, nopos), zone());
+    block->statements()->Add(set_completion_abrupt, zone());
+
+    result_value = factory()->NewDoExpression(block, var_tmp, nopos);
+  }
+
+  // each = #result_value;
+  Expression* assign_each;
+  {
+    assign_each =
+        factory()->NewAssignment(Token::ASSIGN, each, result_value, nopos);
+    if (each->IsArrayLiteral() || each->IsObjectLiteral()) {
+      assign_each = PatternRewriter::RewriteDestructuringAssignment(
+          this, assign_each->AsAssignment(), scope());
+    }
+  }
+
+  // {{completion = kNormalCompletion;}}
+  Statement* set_completion_normal;
+  if (finalize) {
+    Expression* proxy = factory()->NewVariableProxy(completion);
+    Expression* assignment = factory()->NewAssignment(
+        Token::ASSIGN, proxy,
+        factory()->NewSmiLiteral(Parser::kNormalCompletion, nopos), nopos);
+
+    Block* block = factory()->NewBlock(nullptr, 1, true, nopos);
+    block->statements()->Add(
+        factory()->NewExpressionStatement(assignment, nopos), zone());
+    set_completion_normal = block;
+  }
+
+  // { #loop-body; #set_completion_normal }
+  // Statement* body (gets overwritten)
+  if (finalize) {
+    Block* block = factory()->NewBlock(nullptr, 2, false, nopos);
+    block->statements()->Add(body, zone());
+    block->statements()->Add(set_completion_normal, zone());
+    body = block;
+  }
+
+  for_of->Initialize(body, iterator, assign_iterator, next_result, result_done,
+                     assign_each);
+  return finalize ? FinalizeForOfStatement(for_of, completion,
+                                           IteratorType::kAsync, nopos)
+                  : for_of;
 }
 
 Statement* Parser::DesugarLexicalBindingsInForStatement(
@@ -2750,6 +2885,7 @@ Parser::LazyParsingResult Parser::SkipFunction(
     SET_ALLOW(harmony_trailing_commas);
     SET_ALLOW(harmony_class_fields);
     SET_ALLOW(harmony_object_spread);
+    SET_ALLOW(harmony_async_iteration);
 #undef SET_ALLOW
   }
   // Aborting inner function preparsing would leave scopes in an inconsistent
@@ -3920,6 +4056,20 @@ Expression* Parser::RewriteAwaitExpression(Expression* value, int await_pos) {
 
   const int nopos = kNoSourcePosition;
 
+  if (is_async_generator()) {
+    ZoneList<Expression*>* args = new (zone()) ZoneList<Expression*>(2, zone());
+    Expression* generator_object =
+        factory()->NewVariableProxy(generator_object_variable);
+    args->Add(generator_object, zone());
+    args->Add(value, zone());
+
+    Expression* await = factory()->NewCallRuntime(
+        Context::ASYNC_GENERATOR_AWAIT_CAUGHT, args, nopos);
+
+    return factory()->NewYield(generator_object, await, nopos,
+                               Yield::kOnExceptionRethrow, Yield::kAwait);

[Dan Ehrenberg, 2017/01/13 22:09:02]

Why don't you have to apply the same sort of block thing that the rest of this
function does in order to get single stepping to work as expected? I would've
expected that your modification here would be to just switch between the two
different runtime function indices, and otherwise do all the same things? (Even
if you're not passing the Promise for now, presumably you'll need to once catch
prediction support is added.)

[caitp, 2017/01/13 22:33:30]

I haven't tested any of this with the inspector yet, but I'd like to minimize
the number of new runtime mechanics in the parser, as it doesn't seem like a
sensible place for them to me. If debugging is awful with it, it will be
improved, but hopefully not using do expressions or the parser

[Dan Ehrenberg, 2017/01/14 00:48:38]

I'm confused; if you wanted to minimize the amount of mechanics in the parser,
why not just do the exact same thing that the rest of this does? It would mean
that this patch just adds one line here instead of 12. I don't think Yang or I
were happy to put this logic in the parser, but that's where it goes in the
current system. When we refactor things to not have the logic in the parser,
then we can do it for both async functions and async generators.

+  }
+
   Block* do_block = factory()->NewBlock(nullptr, 2, false, nopos);
 
   Variable* promise = PromiseVariable();
@@ -3955,7 +4105,7 @@ Expression* Parser::RewriteAwaitExpression(Expression* value, int await_pos) {
 
   generator_object = factory()->NewVariableProxy(generator_object_variable);
   return factory()->NewYield(generator_object, do_expr, nopos,
-                             Yield::kOnExceptionRethrow);
+                             Yield::kOnExceptionRethrow, Yield::kAwait);
 }
 
 class NonPatternRewriter : public AstExpressionRewriter {
@@ -4313,6 +4463,8 @@ void Parser::SetFunctionName(Expression* value, const AstRawString* name) {
 Expression* Parser::RewriteYieldStar(Expression* generator,
                                      Expression* iterable, int pos) {
   const int nopos = kNoSourcePosition;
+  IteratorType type =
+      is_async_generator() ? IteratorType::kAsync : IteratorType::kNormal;
 
   // Forward definition for break/continue statements.
   WhileStatement* loop = factory()->NewWhileStatement(nullptr, nopos);
@@ -4355,7 +4507,11 @@ Expression* Parser::RewriteYieldStar(Expression* generator,
   Variable* var_iterator = NewTemporary(ast_value_factory()->empty_string());
   Statement* get_iterator;
   {
-    Expression* iterator = factory()->NewGetIterator(iterable, nopos);
+    GetIterator::Hint hint = GetIterator::kNormal;
+    if (type == IteratorType::kAsync) {
+      hint = GetIterator::kAsync;
+    }
+    Expression* iterator = factory()->NewGetIterator(iterable, hint, nopos);
     Expression* iterator_proxy = factory()->NewVariableProxy(var_iterator);
     Expression* assignment = factory()->NewAssignment(
         Token::ASSIGN, iterator_proxy, iterator, nopos);
@@ -4374,6 +4530,9 @@ Expression* Parser::RewriteYieldStar(Expression* generator,
     auto args = new (zone()) ZoneList<Expression*>(1, zone());
     args->Add(input_proxy, zone());
     Expression* call = factory()->NewCall(next_property, args, nopos);
+    if (type == IteratorType::kAsync) {
+      call = RewriteAwaitExpression(call, nopos);
+    }
     Expression* output_proxy = factory()->NewVariableProxy(var_output);
     Expression* assignment =
         factory()->NewAssignment(Token::ASSIGN, output_proxy, call, nopos);
@@ -4437,7 +4596,7 @@ Expression* Parser::RewriteYieldStar(Expression* generator,
     Block* then = factory()->NewBlock(nullptr, 4 + 1, false, nopos);
     BuildIteratorCloseForCompletion(
         scope(), then->statements(), var_iterator,
-        factory()->NewSmiLiteral(Parser::kNormalCompletion, nopos));
+        factory()->NewSmiLiteral(Parser::kNormalCompletion, nopos), type);
     then->statements()->Add(throw_call, zone());
     check_throw = factory()->NewIfStatement(
         condition, then, factory()->NewEmptyStatement(nopos), nopos);
@@ -4452,6 +4611,9 @@ Expression* Parser::RewriteYieldStar(Expression* generator,
     args->Add(factory()->NewVariableProxy(var_input), zone());
     Expression* call =
         factory()->NewCallRuntime(Runtime::kInlineCall, args, nopos);
+    if (type == IteratorType::kAsync) {
+      call = RewriteAwaitExpression(call, nopos);
+    }
     Expression* assignment = factory()->NewAssignment(
         Token::ASSIGN, factory()->NewVariableProxy(var_output), call, nopos);
     call_throw = factory()->NewExpressionStatement(assignment, nopos);
@@ -4510,8 +4672,9 @@ Expression* Parser::RewriteYieldStar(Expression* generator,
   Statement* yield_output;
   {
     Expression* output_proxy = factory()->NewVariableProxy(var_output);
-    Yield* yield = factory()->NewYield(generator, output_proxy, nopos,
-                                       Yield::kOnExceptionThrow);
+    Yield* yield =
+        factory()->NewYield(generator, output_proxy, nopos,
+                            Yield::kOnExceptionThrow, Yield::kDelegate);
     yield_output = factory()->NewExpressionStatement(yield, nopos);
   }
 
@@ -4624,7 +4787,7 @@ Expression* Parser::RewriteYieldStar(Expression* generator,
     case_next->Add(factory()->NewBreakStatement(switch_mode, nopos), zone());
 
     auto case_return = new (zone()) ZoneList<Statement*>(5, zone());
-    BuildIteratorClose(case_return, var_iterator, var_input, var_output);
+    BuildIteratorClose(case_return, var_iterator, var_input, var_output, type);
     case_return->Add(factory()->NewBreakStatement(switch_mode, nopos), zone());
 
     auto case_throw = new (zone()) ZoneList<Statement*>(5, zone());
@@ -4708,7 +4871,7 @@ Statement* Parser::CheckCallable(Variable* var, Expression* error, int pos) {
 
 void Parser::BuildIteratorClose(ZoneList<Statement*>* statements,
                                 Variable* iterator, Variable* input,
-                                Variable* var_output) {
+                                Variable* var_output, IteratorType type) {
   //
   // This function adds four statements to [statements], corresponding to the
   // following code:
@@ -4718,8 +4881,12 @@ void Parser::BuildIteratorClose(ZoneList<Statement*>* statements,
   //     return {value: input, done: true};
   //   }
   //   output = %_Call(iteratorReturn, iterator, input);
+  //   if (IteratorType == kAsync) {
+  //     output = Await(output)
+  //   }
   //   if (!IS_RECEIVER(output)) %ThrowIterResultNotAnObject(output);
-  //
+
+  // output = Await(%_Call(iteratorReturn, iterator, input));

[Dan Ehrenberg, 2017/01/13 19:41:09]

Comment nit: should be if (IteratorType == kAsync) { ... }

[caitp, 2017/01/13 20:11:52]

It kind of gives the impression that the desugaring conditionally Awaits based
on some runtime value, but it statically decides to Await or not depending on
whether it's an async iterator or not.

I haven't found a good way to word these yet, so I'll use whatever prose people
want here.

[Dan Ehrenberg, 2017/01/13 22:09:02]

I was just copying the way that you wrote the syntax above, which already seemed
fine to me, but good point about the compile time vs runtime mismatch. Anyway,
for a strawman way to do the syntax in these comments, what if we use something
traditional and conservative--IE conditional comments!
https://msdn.microsoft.com/en-us/library/ms537512(v=vs.85).aspx

[if IteratorType == kAsync]
  output = Await(output)
[endif]

Anyway, I don't think it's a big deal which syntax you use, but it's nicer if
some sort of conditionality is explicit.

 
   const int nopos = kNoSourcePosition;
 
@@ -4766,6 +4933,9 @@ void Parser::BuildIteratorClose(ZoneList<Statement*>* statements,
 
     Expression* call =
         factory()->NewCallRuntime(Runtime::kInlineCall, args, nopos);
+    if (type == IteratorType::kAsync) {
+      call = RewriteAwaitExpression(call, nopos);
+    }
     Expression* output_proxy = factory()->NewVariableProxy(var_output);
     Expression* assignment =
         factory()->NewAssignment(Token::ASSIGN, output_proxy, call, nopos);
@@ -4805,7 +4975,8 @@ void Parser::BuildIteratorClose(ZoneList<Statement*>* statements,
 
 void Parser::FinalizeIteratorUse(Scope* use_scope, Variable* completion,
                                  Expression* condition, Variable* iter,
-                                 Block* iterator_use, Block* target) {
+                                 Block* iterator_use, Block* target,
+                                 IteratorType type) {
   //
   // This function adds two statements to [target], corresponding to the
   // following code:
@@ -4861,8 +5032,8 @@ void Parser::FinalizeIteratorUse(Scope* use_scope, Variable* completion,
   {
     Block* block = factory()->NewBlock(nullptr, 2, true, nopos);
     Expression* proxy = factory()->NewVariableProxy(completion);
-    BuildIteratorCloseForCompletion(use_scope, block->statements(), iter,
-                                    proxy);
+    BuildIteratorCloseForCompletion(use_scope, block->statements(), iter, proxy,
+                                    type);
     DCHECK(block->statements()->length() == 2);
 
     maybe_close = factory()->NewBlock(nullptr, 1, true, nopos);
@@ -4922,7 +5093,8 @@ void Parser::FinalizeIteratorUse(Scope* use_scope, Variable* completion,
 void Parser::BuildIteratorCloseForCompletion(Scope* scope,
                                              ZoneList<Statement*>* statements,
                                              Variable* iterator,
-                                             Expression* completion) {
+                                             Expression* completion,
+                                             IteratorType type) {
   //
   // This function adds two statements to [statements], corresponding to the
   // following code:
@@ -4980,6 +5152,10 @@ void Parser::BuildIteratorCloseForCompletion(Scope* scope,
     Expression* call =
         factory()->NewCallRuntime(Runtime::kInlineCall, args, nopos);
 
+    if (type == IteratorType::kAsync) {
+      call = RewriteAwaitExpression(call, nopos);
+    }
+
     Block* try_block = factory()->NewBlock(nullptr, 1, false, nopos);
     try_block->statements()->Add(factory()->NewExpressionStatement(call, nopos),
                                  zone());
@@ -5080,7 +5256,8 @@ void Parser::BuildIteratorCloseForCompletion(Scope* scope,
 }
 
 Statement* Parser::FinalizeForOfStatement(ForOfStatement* loop,
-                                          Variable* var_completion, int pos) {
+                                          Variable* var_completion,
+                                          IteratorType type, int pos) {
   //
   // This function replaces the loop with the following wrapping:
   //
@@ -5124,7 +5301,7 @@ Statement* Parser::FinalizeForOfStatement(ForOfStatement* loop,
     DCHECK_EQ(scope()->scope_type(), BLOCK_SCOPE);
 
     FinalizeIteratorUse(loop_scope, var_completion, closing_condition,
-                        loop->iterator(), try_block, final_loop);
+                        loop->iterator(), try_block, final_loop, type);
   }
 
   return final_loop;