Implement async* functions in VM

runtime/vm/parser.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/parser.h"
 
 #include "lib/invocation_mirror.h"
 #include "platform/utils.h"
 #include "vm/ast_transformer.h"
 #include "vm/bootstrap.h"
@@ skipping 3086 matching lines @@
     AddFormalParamsToScope(&params, current_block_->scope);
     ASSERT(AbstractType::Handle(Z, func.result_type()).IsResolved());
     if (!Function::Handle(func.parent_function()).IsGetterFunction()) {
       // Parse and discard any formal parameters. They are accessed as
       // context variables.
       ParamList discarded_params;
       ParseFormalParameterList(allow_explicit_default_values,
                                false,
                                &discarded_params);
     }
+  } else if (func.IsAsyncGenClosure()) {
+    AddAsyncGenClosureParameters(&params);
+    SetupDefaultsForOptionalParams(&params, default_parameter_values);
+    AddFormalParamsToScope(&params, current_block_->scope);
+    ASSERT(AbstractType::Handle(Z, func.result_type()).IsResolved());
+    if (!Function::Handle(func.parent_function()).IsGetterFunction()) {
+      // Parse and discard any formal parameters. They are accessed as
+      // context variables.
+      ParamList discarded_params;
+      ParseFormalParameterList(allow_explicit_default_values,
+                               false,
+                               &discarded_params);
+    }
   } else {
     ParseFormalParameterList(allow_explicit_default_values, false, &params);
 
     // The number of parameters and their type are not yet set in local
     // functions, since they are not 'top-level' parsed.
     if (func.IsLocalFunction()) {
       AddFormalParamsToFunction(&params, func);
     }
     SetupDefaultsForOptionalParams(&params, default_parameter_values);
     ASSERT(AbstractType::Handle(Z, func.result_type()).IsResolved());
@@ skipping 48 matching lines @@
     ASSERT(func.is_debuggable());
     OpenAsyncClosure();
   } else if (func.IsSyncGenerator()) {
     // The code of a sync generator is synthesized. Disable debugging.
     func.set_is_debuggable(false);
     generated_body_closure = OpenSyncGeneratorFunction(func.token_pos());
   } else if (func.IsSyncGenClosure()) {
     // The closure containing the body of a sync generator is debuggable.
     ASSERT(func.is_debuggable());
     async_temp_scope_ = current_block_->scope;
+  } else if (func.IsAsyncGenerator()) {
+    func.set_is_debuggable(false);
+    generated_body_closure = OpenAsyncGeneratorFunction(func.token_pos());
+  } else if (func.IsAsyncGenClosure()) {
+    // The closure containing the body of an async* function is debuggable.
+    ASSERT(func.is_debuggable());
+    OpenAsyncGeneratorClosure();
   }
 
   BoolScope allow_await(&this->await_is_keyword_,
                         func.IsAsyncOrGenerator() || func.is_generated_body());
   intptr_t end_token_pos = 0;
   if (CurrentToken() == Token::kLBRACE) {
     ConsumeToken();
     if (String::Handle(Z, func.name()).Equals(Symbols::EqualOperator())) {
       const Class& owner = Class::Handle(Z, func.Owner());
       if (!owner.IsObjectClass()) {
@@ skipping 57 matching lines @@
   if (func.IsAsyncFunction()) {
     body = CloseAsyncFunction(generated_body_closure, body);
     generated_body_closure.set_end_token_pos(end_token_pos);
   } else if (func.IsAsyncClosure()) {
     body = CloseAsyncClosure(body);
   } else if (func.IsSyncGenerator()) {
     body = CloseSyncGenFunction(generated_body_closure, body);
     generated_body_closure.set_end_token_pos(end_token_pos);
   } else if (func.IsSyncGenClosure()) {
     body->scope()->RecursivelyCaptureAllVariables();
+  } else if (func.IsAsyncGenerator()) {
+    body = CloseAsyncGeneratorFunction(generated_body_closure, body);
+    generated_body_closure.set_end_token_pos(end_token_pos);
+  } else if (func.IsAsyncGenClosure()) {
+    body = CloseAsyncGeneratorClosure(body);
   }
   current_block_->statements->Add(body);
   innermost_function_ = saved_innermost_function.raw();
   last_used_try_index_ = saved_try_index;
   async_temp_scope_ = saved_async_temp_scope;
   parsed_function()->set_saved_try_ctx(saved_saved_try_ctx);
   parsed_function()->set_async_saved_try_ctx_name(
       saved_async_saved_try_ctx_name);
   return CloseBlock();
 }
@@ skipping 1980 matching lines @@
       ExpectSemicolon();  // Reports error.
     }
   }
 }
 
 
 RawFunction::AsyncModifier Parser::ParseFunctionModifier() {
   if (CurrentLiteral()->raw() == Symbols::Async().raw()) {
     ConsumeToken();
     if (CurrentToken() == Token::kMUL) {
-      ReportError("async* generator functions are not yet supported");
+      const bool enableAsyncStar = true;
+      if (!enableAsyncStar) {
+        ReportError("async* generator functions are not yet supported");
+      }
       ConsumeToken();
       return RawFunction::kAsyncGen;
     } else {
       return RawFunction::kAsync;
     }
   } else if ((CurrentLiteral()->raw() == Symbols::Sync().raw()) &&
       (LookaheadToken(1) == Token::kMUL)) {
     const bool enableSyncStar = true;
     if (!enableSyncStar) {
       ReportError("sync* generator functions are not yet supported");
@@ skipping 670 matching lines @@
 
 void Parser::OpenAsyncClosure() {
   TRACE_PARSER("OpenAsyncClosure");
 
   async_temp_scope_ = current_block_->scope;
 
   OpenAsyncTryBlock();
 }
 
 
-SequenceNode* Parser::CloseAsyncTryBlock(SequenceNode* try_block) {
+SequenceNode* Parser::CloseAsyncGeneratorTryBlock(SequenceNode *body) {
+  TRACE_PARSER("CloseAsyncGeneratorTryBlock");
+  // The generated try-catch-finally that wraps the async generator function
+  // body is the outermost try statement.
+  ASSERT(try_blocks_list_ != NULL);
+  ASSERT(try_blocks_list_->outer_try_block() == NULL);
+  // We only get here when parsing an async generator body.
+  ASSERT(innermost_function().IsAsyncGenClosure());
+
+  // The try-block (closure body code) has been parsed. We are now
+  // generating the code for the catch block.
   try_blocks_list_->enter_catch();
+  OpenBlock();  // Catch handler list.
+  OpenBlock();  // Catch block.
 
-  OpenBlock();
-  OpenBlock();
+  // Add the exception and stack trace parameters to the scope.
   const AbstractType& dynamic_type =
       AbstractType::ZoneHandle(Z, Type::DynamicType());
   CatchParamDesc exception_param;
   CatchParamDesc stack_trace_param;
   exception_param.token_pos = Scanner::kNoSourcePos;
   exception_param.type = &dynamic_type;
   exception_param.name = &Symbols::ExceptionParameter();
   stack_trace_param.token_pos = Scanner::kNoSourcePos;
   stack_trace_param.type = &dynamic_type;
   stack_trace_param.name = &Symbols::StackTraceParameter();
-
   AddCatchParamsToScope(
       &exception_param, &stack_trace_param, current_block_->scope);
 
+  // Generate code to save the exception object and stack trace
+  // in local variables.
   LocalVariable* context_var = current_block_->scope->LookupVariable(
       Symbols::SavedTryContextVar(), false);
   ASSERT(context_var != NULL);
   LocalVariable* exception_var = current_block_->scope->LookupVariable(
       Symbols::ExceptionVar(), false);
   if (exception_param.var != NULL) {
     // Generate code to load the exception object (:exception_var) into
     // the exception variable specified in this block.
     ASSERT(exception_var != NULL);
     current_block_->statements->Add(new(Z) StoreLocalNode(
         Scanner::kNoSourcePos,
         exception_param.var,
         new(Z) LoadLocalNode(Scanner::kNoSourcePos, exception_var)));
   }
   LocalVariable* stack_trace_var =
       current_block_->scope->LookupVariable(Symbols::StackTraceVar(), false);
   if (stack_trace_param.var != NULL) {
     // A stack trace variable is specified in this block, so generate code
     // to load the stack trace object (:stack_trace_var) into the stack
     // trace variable specified in this block.
     ASSERT(stack_trace_var != NULL);
     current_block_->statements->Add(new(Z) StoreLocalNode(
         Scanner::kNoSourcePos,
         stack_trace_param.var,
         new(Z) LoadLocalNode(Scanner::kNoSourcePos, stack_trace_var)));
   }
 
+  parsed_function()->reset_saved_try_ctx_vars();
+
+  // Catch block: add the error to the stream.
+  // :controller.AddError(:exception, :stack_trace);
+  // return;  // The finally block will close the stream.
+  LocalVariable* controller =
+      current_block_->scope->LookupVariable(Symbols::Controller(), false);
+  ASSERT(controller != NULL);
+  ArgumentListNode* args =
+      new(Z) ArgumentListNode(Scanner::kNoSourcePos);
+  args->Add(new(Z) LoadLocalNode(Scanner::kNoSourcePos, exception_param.var));
+  args->Add(new(Z) LoadLocalNode(Scanner::kNoSourcePos, stack_trace_param.var));
+  current_block_->statements->Add(
+      new(Z) InstanceCallNode(TokenPos(),
+          new(Z) LoadLocalNode(Scanner::kNoSourcePos, controller),
+          Symbols::AddError(),
+          args));
+  ReturnNode* return_node = new(Z) ReturnNode(Scanner::kNoSourcePos);
+  current_block_->statements->Add(return_node);
+  AstNode* catch_block = CloseBlock();
+  current_block_->statements->Add(catch_block);
+  SequenceNode* catch_handler_list = CloseBlock();
+
+  TryBlocks* try_block = PopTryBlock();
+  ASSERT(try_blocks_list_ == NULL);  // We popped the outermost try block.
+
+  // Finally block: closing the stream and returning. (Note: the return
+  // is necessary otherwise the back-end will append a rethrow of the
+  // current exception.)
+  // :controller.close();
+  // return;
+  // We need to inline this code in all recorded exit points.
+  intptr_t node_index = 0;
+  SequenceNode* finally_clause = NULL;
+  do {
+    OpenBlock();
+    ArgumentListNode* no_args =
+        new(Z) ArgumentListNode(Scanner::kNoSourcePos);
+    current_block_->statements->Add(
+        new(Z) InstanceCallNode(TokenPos(),
+            new(Z) LoadLocalNode(Scanner::kNoSourcePos, controller),
+            Symbols::Close(),
+            no_args));
+
+    ReturnNode* return_node = new(Z) ReturnNode(Scanner::kNoSourcePos);
+    current_block_->statements->Add(return_node);
+
+    finally_clause = CloseBlock();
+    AstNode* node_to_inline = try_block->GetNodeToInlineFinally(node_index);
+    if (node_to_inline != NULL) {
+      InlinedFinallyNode* node =
+          new(Z) InlinedFinallyNode(TokenPos(),
+                                    finally_clause,
+                                    context_var,
+                                    // No outer try statement
+                                    CatchClauseNode::kInvalidTryIndex);
+      finally_clause = NULL;
+      AddFinallyBlockToNode(node_to_inline, node);
+      node_index++;
+    }
+  } while (finally_clause == NULL);
+
+  const GrowableObjectArray& handler_types =
+      GrowableObjectArray::Handle(Z, GrowableObjectArray::New());
+  handler_types.Add(dynamic_type);  // Catch block handles all exceptions.
+
+  CatchClauseNode* catch_clause = new(Z) CatchClauseNode(
+      Scanner::kNoSourcePos,
+      catch_handler_list,
+      Array::ZoneHandle(Z, Array::MakeArray(handler_types)),
+      context_var,
+      exception_var,
+      stack_trace_var,
+      AllocateTryIndex(),
+      true);
+
+  const intptr_t try_index = try_block->try_index();
+
+  AstNode* try_catch_node =
+      new(Z) TryCatchNode(Scanner::kNoSourcePos,
+                          body,
+                          context_var,
+                          catch_clause,
+                          finally_clause,
+                          try_index);
+  current_block_->statements->Add(try_catch_node);
+  return CloseBlock();
+}
+
+
+SequenceNode* Parser::CloseAsyncTryBlock(SequenceNode* try_block) {
+  // This is the outermost try-catch of the function.
   ASSERT(try_blocks_list_ != NULL);
-  ASSERT(innermost_function().IsAsyncClosure() ||
-         innermost_function().IsAsyncFunction());
-  if ((try_blocks_list_->outer_try_block() != NULL) &&
-      (try_blocks_list_->outer_try_block()->try_block()
-          ->scope->function_level() ==
-       current_block_->scope->function_level())) {
-    // We need to unchain three scope levels: catch clause, catch
-    // parameters, and the general try block.
-    RestoreSavedTryContext(
-        current_block_->scope->parent()->parent()->parent(),
-        try_blocks_list_->outer_try_block()->try_index(),
-        current_block_->statements);
-  } else {
-    parsed_function()->reset_saved_try_ctx_vars();
+  ASSERT(try_blocks_list_->outer_try_block() == NULL);
+  ASSERT(innermost_function().IsAsyncClosure());
+
+  try_blocks_list_->enter_catch();
+
+  OpenBlock();  // Catch handler list.
+  OpenBlock();  // Catch block.
+  const AbstractType& dynamic_type =
+      AbstractType::ZoneHandle(Z, Type::DynamicType());
+  CatchParamDesc exception_param;
+  CatchParamDesc stack_trace_param;
+  exception_param.token_pos = Scanner::kNoSourcePos;
+  exception_param.type = &dynamic_type;
+  exception_param.name = &Symbols::ExceptionParameter();
+  stack_trace_param.token_pos = Scanner::kNoSourcePos;
+  stack_trace_param.type = &dynamic_type;
+  stack_trace_param.name = &Symbols::StackTraceParameter();
+
+  AddCatchParamsToScope(
+      &exception_param, &stack_trace_param, current_block_->scope);
+
+  LocalVariable* context_var = current_block_->scope->LookupVariable(
+      Symbols::SavedTryContextVar(), false);
+  ASSERT(context_var != NULL);
+  LocalVariable* exception_var = current_block_->scope->LookupVariable(
+      Symbols::ExceptionVar(), false);
+  if (exception_param.var != NULL) {
+    // Generate code to load the exception object (:exception_var) into
+    // the exception variable specified in this block.
+    ASSERT(exception_var != NULL);
+    current_block_->statements->Add(new(Z) StoreLocalNode(
+        Scanner::kNoSourcePos,
+        exception_param.var,
+        new(Z) LoadLocalNode(Scanner::kNoSourcePos, exception_var)));
+  }
+  LocalVariable* stack_trace_var =
+      current_block_->scope->LookupVariable(Symbols::StackTraceVar(), false);
+  if (stack_trace_param.var != NULL) {
+    // A stack trace variable is specified in this block, so generate code
+    // to load the stack trace object (:stack_trace_var) into the stack
+    // trace variable specified in this block.
+    ASSERT(stack_trace_var != NULL);
+    current_block_->statements->Add(new(Z) StoreLocalNode(
+        Scanner::kNoSourcePos,
+        stack_trace_param.var,
+        new(Z) LoadLocalNode(Scanner::kNoSourcePos, stack_trace_var)));
   }
 
-  // Complete the async future with an error.
-  // Since we control the catch block there is no need to generate a nested
-  // if/then/else.
+  parsed_function()->reset_saved_try_ctx_vars();
+
+  // Complete the async future with an error. This catch block executes
+  // unconditionally, there is no need to generate a type check for.
   LocalVariable* async_completer = current_block_->scope->LookupVariable(
       Symbols::AsyncCompleter(), false);
   ASSERT(async_completer != NULL);
   ArgumentListNode* completer_args =
       new (Z) ArgumentListNode(Scanner::kNoSourcePos);
   completer_args->Add(
       new (Z) LoadLocalNode(Scanner::kNoSourcePos, exception_param.var));
   completer_args->Add(
       new (Z) LoadLocalNode(Scanner::kNoSourcePos, stack_trace_param.var));
   current_block_->statements->Add(new (Z) InstanceCallNode(
@@ skipping 24 matching lines @@
       context_var,
       exception_var,
       stack_trace_var,
       CatchClauseNode::kInvalidTryIndex,
       true);
   AstNode* try_catch_node = new (Z) TryCatchNode(
       Scanner::kNoSourcePos,
       try_block,
       context_var,
       catch_clause,
-      NULL,
+      NULL,  // No finally clause.
       try_index);
   current_block_->statements->Add(try_catch_node);
   return CloseBlock();
 }
 
 
+// Wrap the body of the async or arync* closure in a try/catch block.
 void Parser::OpenAsyncTryBlock() {
-  // Manually wrapping the actual body into a try/catch block.
+  ASSERT(innermost_function().IsAsyncClosure() ||
+         innermost_function().IsAsyncGenClosure());
+
   LocalVariable* context_var =
       current_block_->scope->LocalLookupVariable(Symbols::SavedTryContextVar());
   if (context_var == NULL) {
     context_var = new(Z) LocalVariable(
         TokenPos(),
         Symbols::SavedTryContextVar(),
         Type::ZoneHandle(Z, Type::DynamicType()));
     current_block_->scope->AddVariable(context_var);
   }
   LocalVariable* exception_var =
@@ skipping 10 matching lines @@
   if (stack_trace_var == NULL) {
     stack_trace_var = new(Z) LocalVariable(
         TokenPos(),
         Symbols::StackTraceVar(),
         Type::ZoneHandle(Z, Type::DynamicType()));
     current_block_->scope->AddVariable(stack_trace_var);
   }
 
   // Open the try block.
   OpenBlock();
+  // This is the outermost try-catch in the function.
+  ASSERT(try_blocks_list_ == NULL);
   PushTryBlock(current_block_);
 
-  if (innermost_function().IsAsyncClosure() ||
-      innermost_function().IsAsyncFunction() ||
-      innermost_function().IsSyncGenClosure() ||
-      innermost_function().IsSyncGenerator()) {
-    SetupSavedTryContext(context_var);
-  }
+  SetupSavedTryContext(context_var);
 }
 
 
 void Parser::AddSyncGenClosureParameters(ParamList* params) {
   // Create the parameter list for the body closure of a sync generator:
   // 1) Implicit closure parameter;
   // 2) Iterator
   const Type& dynamic_type = Type::ZoneHandle(Z, Type::DynamicType());
   // Add implicit closure parameter if not already present.
   if (params->parameters->length() == 0) {
     params->AddFinalParameter(0, &Symbols::ClosureParameter(), &dynamic_type);
   }
   ParamDesc iterator_param;
   iterator_param.name = &Symbols::IteratorParameter();
   iterator_param.type = &dynamic_type;
   params->parameters->Add(iterator_param);
   params->num_fixed_parameters++;
 }
 
 
+void Parser::AddAsyncGenClosureParameters(ParamList* params) {
+  // Create the parameter list for the body closure of a sync generator:

[Ivan Posva, 2015/02/23 08:23:36]

sync?

[hausner, 2015/02/23 17:12:01]

Yes.

+  // The closure has no explicit parameters.
+  const Type& dynamic_type = Type::ZoneHandle(Z, Type::DynamicType());
+  // Add implicit closure parameter if not already present.
+  if (params->parameters->length() == 0) {
+    params->AddFinalParameter(0, &Symbols::ClosureParameter(), &dynamic_type);
+  }
+}
+
+
 RawFunction* Parser::OpenSyncGeneratorFunction(intptr_t func_pos) {
   Function& body = Function::Handle(Z);
   String& body_closure_name = String::Handle(Z);
   bool is_new_closure = false;
 
   AddContinuationVariables();
 
   // Check whether a function for the body of this generator
   // function has already been created by a previous
   // compilation.
@@ skipping 219 matching lines @@
   current_block_->scope->CaptureVariable(Symbols::AsyncOperation());
   async_op_var->set_is_captured();
   LocalVariable* async_completer = new(Z) LocalVariable(
       Scanner::kNoSourcePos, Symbols::AsyncCompleter(), dynamic_type);
   current_block_->scope->AddVariable(async_completer);
   current_block_->scope->CaptureVariable(Symbols::AsyncCompleter());
   async_completer->set_is_captured();
 }
 
 
+void Parser::AddAsyncGeneratorVariables() {
+  // Add to current block's scope:
+  //   var :controller;
+  // This variables is used by the nested async generator closure to
+  // store the StreamController object to which the yielded expressions
+  // are added.
+  const Type& dynamic_type = Type::ZoneHandle(Z, Type::DynamicType());
+  LocalVariable* controller_var = new(Z) LocalVariable(
+      Scanner::kNoSourcePos, Symbols::Controller(), dynamic_type);
+  current_block_->scope->AddVariable(controller_var);
+  current_block_->scope->CaptureVariable(Symbols::Controller());
+  controller_var->set_is_captured();
+}
+
+
+RawFunction* Parser::OpenAsyncGeneratorFunction(intptr_t async_func_pos) {
+  TRACE_PARSER("OpenAsyncGeneratorFunction");
+  AddContinuationVariables();
+  AddAsyncGeneratorVariables();
+
+  Function& closure = Function::Handle(Z);
+  bool is_new_closure = false;
+
+  // Check whether a function for the asynchronous function body of
+  // this async generator has already been created by a previous
+  // compilation of this function.
+  const Function& found_func = Function::Handle(
+      Z, current_class().LookupClosureFunction(async_func_pos));
+  if (!found_func.IsNull() &&
+      (found_func.token_pos() == async_func_pos) &&
+      (found_func.script() == innermost_function().script()) &&
+      (found_func.parent_function() == innermost_function().raw())) {
+    ASSERT(found_func.IsAsyncGenClosure());
+    closure = found_func.raw();
+  } else {
+    // Create the closure containing the body of this async generator function.
+    const String& async_generator_name =
+        String::Handle(Z, innermost_function().name());
+    String& closure_name = String::Handle(Z,
+        String::NewFormatted("<%s_async_body>",
+                             async_generator_name.ToCString()));
+    closure = Function::NewClosureFunction(
+        String::Handle(Z, Symbols::New(closure_name)),
+        innermost_function(),
+        async_func_pos);
+    closure.set_is_generated_body(true);
+    closure.set_result_type(AbstractType::Handle(Type::DynamicType()));
+    is_new_closure = true;
+  }
+
+  ParamList closure_params;
+  AddAsyncGenClosureParameters(&closure_params);
+
+  if (is_new_closure) {
+    // Add the parameters to the newly created closure.
+    AddFormalParamsToFunction(&closure_params, closure);
+
+    // Create and set the signature class of the closure.
+    const String& sig = String::Handle(Z, closure.Signature());
+    Class& sig_cls = Class::Handle(Z, library_.LookupLocalClass(sig));
+    if (sig_cls.IsNull()) {
+      sig_cls =
+          Class::NewSignatureClass(sig, closure, script_, closure.token_pos());
+      library_.AddClass(sig_cls);
+    }
+    closure.set_signature_class(sig_cls);
+    const Type& sig_type = Type::Handle(Z, sig_cls.SignatureType());
+    if (!sig_type.IsFinalized()) {
+      ClassFinalizer::FinalizeType(
+          sig_cls, sig_type, ClassFinalizer::kCanonicalize);
+    }
+    ASSERT(AbstractType::Handle(Z, closure.result_type()).IsResolved());
+    ASSERT(closure.NumParameters() == closure_params.parameters->length());
+  }
+
+  OpenFunctionBlock(closure);
+  AddFormalParamsToScope(&closure_params, current_block_->scope);
+  OpenBlock();
+  async_temp_scope_ = current_block_->scope;
+  return closure.raw();
+}
+
+
+// Generate the Ast nodes for the implicit code of the async* function.
+//
+// f(...) async* {
+//   var :controller;
+//   var :await_jump_var = -1;
+//   var :await_context_var;
+//   f_async_body() {
+//     ... source code of f ...
+//   }
+//   :controller = new _AsyncStarStreamController(f_async_body);
+//   return :controller.stream;
+// }
+SequenceNode* Parser::CloseAsyncGeneratorFunction(const Function& closure,
+                                                  SequenceNode* closure_body) {
+  TRACE_PARSER("CloseAsyncGeneratorFunction");
+  ASSERT(!closure.IsNull());
+  ASSERT(closure_body != NULL);
+
+  // The block for the async closure body has already been closed. Close the
+  // corresponding function block.
+  CloseBlock();
+
+  // Make sure the implicit variables of the async generator function
+  // are captured.
+  closure_body->scope()->LookupVariable(Symbols::AwaitJumpVar(), false);
+  closure_body->scope()->LookupVariable(Symbols::AwaitContextVar(), false);
+  closure_body->scope()->LookupVariable(Symbols::Controller(), false);
+
+  const Class& controller_class = Class::Handle(Z,
+      Library::LookupCoreClass(Symbols::_AsyncStarStreamController()));
+  ASSERT(!controller_class.IsNull());
+  const Function& controller_constructor = Function::ZoneHandle(Z,
+      controller_class.LookupConstructorAllowPrivate(
+          Symbols::_AsyncStarStreamControllerConstructor()));
+
+  // :await_jump_var = -1;
+  LocalVariable* jump_var =
+      current_block_->scope->LookupVariable(Symbols::AwaitJumpVar(), false);
+  LiteralNode* init_value =
+      new(Z) LiteralNode(Scanner::kNoSourcePos, Smi::ZoneHandle(Smi::New(-1)));
+  current_block_->statements->Add(
+      new(Z) StoreLocalNode(Scanner::kNoSourcePos, jump_var, init_value));
+
+  // :controller = new _AsyncStarStreamController(body_closure);
+  ArgumentListNode* arguments = new(Z) ArgumentListNode(Scanner::kNoSourcePos);
+  ClosureNode* closure_obj = new(Z) ClosureNode(
+      Scanner::kNoSourcePos, closure, NULL, closure_body->scope());
+  arguments->Add(closure_obj);
+  ConstructorCallNode* controller_constructor_call =
+      new(Z) ConstructorCallNode(Scanner::kNoSourcePos,
+                                 TypeArguments::ZoneHandle(Z),
+                                 controller_constructor,
+                                 arguments);
+  LocalVariable* controller_var =
+     current_block_->scope->LookupVariable(Symbols::Controller(), false);
+  StoreLocalNode* store_controller =
+      new(Z) StoreLocalNode(Scanner::kNoSourcePos,
+                            controller_var,
+                            controller_constructor_call);
+  current_block_->statements->Add(store_controller);
+
+  // return :controller.stream;
+  ReturnNode* return_node = new(Z) ReturnNode(Scanner::kNoSourcePos,
+      new(Z) InstanceGetterNode(Scanner::kNoSourcePos,
+          new(Z) LoadLocalNode(Scanner::kNoSourcePos,
+              controller_var),
+              Symbols::ControllerStream()));
+  current_block_->statements->Add(return_node);
+  return CloseBlock();
+}
+
+
+void Parser::OpenAsyncGeneratorClosure() {
+  async_temp_scope_ = current_block_->scope;
+  OpenAsyncTryBlock();
+}
+
+
+SequenceNode* Parser::CloseAsyncGeneratorClosure(SequenceNode* body) {
+  // TODO(hausner): Is the temporary expression necessary?
+  // We need a temporary expression to store intermediate return values.
+  parsed_function()->EnsureExpressionTemp();
+
+  SequenceNode* new_body = CloseAsyncGeneratorTryBlock(body);
+  ASSERT(new_body != NULL);
+  ASSERT(new_body->scope() != NULL);
+
+  // Implicitly mark those variables below as captured. We currently mark all
+  // variables of all scopes as captured, but as soon as we do something
+  // smarter we rely on these internal variables to be available.
+  new_body->scope()->LookupVariable(Symbols::AwaitJumpVar(), false);
+  new_body->scope()->LookupVariable(Symbols::AwaitContextVar(), false);
+  new_body->scope()->LookupVariable(Symbols::Controller(), false);
+  new_body->scope()->RecursivelyCaptureAllVariables();
+  return new_body;
+}
+
+
 SequenceNode* Parser::CloseBlock() {
   SequenceNode* statements = current_block_->statements;
   if (current_block_->scope != NULL) {
     // Record the begin and end token index of the scope.
     ASSERT(statements != NULL);
     current_block_->scope->set_begin_token_pos(statements->token_pos());
     current_block_->scope->set_end_token_pos(TokenPos());
   }
   current_block_ = current_block_->parent;
   return statements;
@@ skipping 1395 matching lines @@
 AstNode* Parser::ParseAwaitForStatement(String* label_name) {
   TRACE_PARSER("ParseAwaitForStatement");
   ASSERT(IsAwaitKeyword());
   const intptr_t await_for_pos = TokenPos();
   ConsumeToken();  // await.
   ASSERT(CurrentToken() == Token::kFOR);
   ConsumeToken();  // for.
   ExpectToken(Token::kLPAREN);
 
   if (!innermost_function().IsAsyncFunction() &&
-      !innermost_function().IsAsyncClosure()) {
+      !innermost_function().IsAsyncClosure() &&
+      !innermost_function().IsAsyncGenerator() &&
+      !innermost_function().IsAsyncGenClosure()) {
     ReportError(await_for_pos,
-                "await for loop is only allowed in async function");
+                "await for loop is only allowed in an asynchronous function");
   }
 
   // Parse loop variable.
   bool loop_var_is_final = (CurrentToken() == Token::kFINAL);
   if (CurrentToken() == Token::kCONST) {
     ReportError("Loop variable cannot be 'const'");
   }
   bool new_loop_var = false;
   AbstractType& loop_var_type =  AbstractType::ZoneHandle(Z);
   if (LookaheadToken(1) != Token::kIN) {
@@ skipping 418 matching lines @@
   TRACE_PARSER("ParseFinallyBlock");
   OpenBlock();
   ExpectToken(Token::kLBRACE);
 
   // In case of async closures we need to restore the saved try index of an
   // outer try block (if it exists).  The current try block has already been
   // removed from the stack of try blocks.
   if ((innermost_function().IsAsyncClosure() ||
        innermost_function().IsAsyncFunction() ||
        innermost_function().IsSyncGenClosure() ||
-       innermost_function().IsSyncGenerator()) &&
+       innermost_function().IsSyncGenerator() ||
+       innermost_function().IsAsyncGenerator() ||
+       innermost_function().IsAsyncGenClosure()) &&
       (try_blocks_list_ != NULL)) {
     // We need two unchain two scopes: finally clause, and the try block level.
     RestoreSavedTryContext(current_block_->scope->parent()->parent(),
                            try_blocks_list_->try_index(),
                            current_block_->statements);
   } else {
     parsed_function()->reset_saved_try_ctx_vars();
   }
 
   ParseStatementSequence();
@@ skipping 88 matching lines @@
     } else {
       exception_param.type = &AbstractType::ZoneHandle(Z, Type::DynamicType());
     }
     if (CurrentToken() == Token::kCATCH) {
       ConsumeToken();  // Consume the 'catch'.
       ExpectToken(Token::kLPAREN);
       exception_param.token_pos = TokenPos();
       exception_param.name = ExpectIdentifier("identifier expected");
       if (CurrentToken() == Token::kCOMMA) {
         ConsumeToken();
-        // TODO(hausner): Make implicit type be StackTrace, not dynamic.
         stack_trace_param.type =
             &AbstractType::ZoneHandle(Z, Type::DynamicType());
         stack_trace_param.token_pos = TokenPos();
         stack_trace_param.name = ExpectIdentifier("identifier expected");
       }
       ExpectToken(Token::kRPAREN);
     }
 
     // Create a block containing the catch clause parameters and the
     // following code:
@@ skipping 16 matching lines @@
       // A stack trace variable is specified in this block, so generate code
       // to load the stack trace object (:stack_trace_var) into the stack
       // trace variable specified in this block.
       *needs_stack_trace = true;
       ASSERT(stack_trace_var != NULL);
       current_block_->statements->Add(new(Z) StoreLocalNode(
           catch_pos, stack_trace_param.var, new(Z) LoadLocalNode(
               catch_pos, stack_trace_var)));
     }
 
-    // Add nested block with user-defined code.  This blocks allows
+    // Add nested block with user-defined code.  This block allows
     // declarations in the body to shadow the catch parameters.
     CheckToken(Token::kLBRACE);
 
     // In case of async closures we need to restore the saved try index of an
     // outer try block (if it exists).
     ASSERT(try_blocks_list_ != NULL);
     if (innermost_function().IsAsyncClosure() ||
         innermost_function().IsAsyncFunction() ||
         innermost_function().IsSyncGenClosure() ||
-        innermost_function().IsSyncGenerator()) {
+        innermost_function().IsSyncGenerator() ||
+        innermost_function().IsAsyncGenerator() ||
+        innermost_function().IsAsyncGenClosure()) {
       if ((try_blocks_list_->outer_try_block() != NULL) &&
           (try_blocks_list_->outer_try_block()->try_block()
               ->scope->function_level() ==
            current_block_->scope->function_level())) {
         // We need to unchain three scope levels: catch clause, catch
         // parameters, and the general try block.
         RestoreSavedTryContext(
             current_block_->scope->parent()->parent()->parent(),
             try_blocks_list_->outer_try_block()->try_index(),
             current_block_->statements);
@@ skipping 89 matching lines @@
   while (!type_tests.is_empty()) {
     AstNode* type_test = type_tests.RemoveLast();
     SequenceNode* catch_block = catch_blocks.RemoveLast();
 
     // In case of async closures we need to restore the saved try index of an
     // outer try block (if it exists).
     ASSERT(try_blocks_list_ != NULL);
     if (innermost_function().IsAsyncClosure() ||
         innermost_function().IsAsyncFunction() ||
         innermost_function().IsSyncGenClosure() ||
-        innermost_function().IsSyncGenerator()) {
+        innermost_function().IsSyncGenerator() ||
+        innermost_function().IsAsyncGenerator() ||
+        innermost_function().IsAsyncGenClosure()) {
       if ((try_blocks_list_->outer_try_block() != NULL) &&
           (try_blocks_list_->outer_try_block()->try_block()
               ->scope->function_level() ==
            current_block_->scope->function_level())) {
         // We need to unchain three scope levels: catch clause, catch
         // parameters, and the general try block.
         RestoreSavedTryContext(
             current_block_->scope->parent()->parent(),
             try_blocks_list_->outer_try_block()->try_index(),
             current_block_->statements);
@@ skipping 118 matching lines @@
   }
 
   // Now parse the 'try' block.
   OpenBlock();
   PushTryBlock(current_block_);
   ExpectToken(Token::kLBRACE);
 
   if (innermost_function().IsAsyncClosure() ||
       innermost_function().IsAsyncFunction() ||
       innermost_function().IsSyncGenClosure() ||
-      innermost_function().IsSyncGenerator()) {
+      innermost_function().IsSyncGenerator() ||
+      innermost_function().IsAsyncGenerator() ||
+      innermost_function().IsAsyncGenClosure()) {
     SetupSavedTryContext(context_var);
   }
 
   ParseStatementSequence();
   ExpectToken(Token::kRBRACE);
   SequenceNode* try_block = CloseBlock();
 
   if ((CurrentToken() != Token::kCATCH) && !IsLiteral("on") &&
       (CurrentToken() != Token::kFINALLY)) {
     ReportError("catch or finally clause expected");
@@ skipping 123 matching lines @@
     ReportError(jump_pos, "'break' to case clause label is illegal");
   }
   if (target->FunctionLevel() != current_block_->scope->function_level()) {
     ReportError(jump_pos, "'%s' target must be in same function context",
                 Token::Str(jump_kind));
   }
   return new(Z) JumpNode(jump_pos, jump_kind, target);
 }
 
 
+AstNode* Parser::ParseYieldStatement() {
+  bool is_yield_each = false;
+  const intptr_t yield_pos = TokenPos();
+  ConsumeToken();  // yield reserved word.
+  ASSERT(innermost_function().IsGenerator() ||
+         innermost_function().IsSyncGenClosure() ||
+         innermost_function().IsAsyncGenerator() ||
+         innermost_function().IsAsyncGenClosure());
+  if (CurrentToken() == Token::kMUL) {
+    is_yield_each = true;
+    ConsumeToken();
+  }
+  AstNode* expr = ParseAwaitableExpr(kAllowConst, kConsumeCascades, NULL);
+
+  LetNode* yield = new(Z) LetNode(yield_pos);
+  if (innermost_function().IsSyncGenerator() ||
+      innermost_function().IsSyncGenClosure()) {
+    // Yield statement in sync* function.
+
+    LocalVariable* iterator_param =
+        LookupLocalScope(Symbols::IteratorParameter());
+    ASSERT(iterator_param != NULL);
+    // Generate :iterator.current = expr;
+    AstNode* iterator =
+        new(Z) LoadLocalNode(Scanner::kNoSourcePos, iterator_param);
+    AstNode* store_current =
+        new(Z) InstanceSetterNode(Scanner::kNoSourcePos,
+                                  iterator,
+                                  String::ZoneHandle(Symbols::Current().raw()),
+                                  expr);
+    yield->AddNode(store_current);
+    if (is_yield_each) {
+      // Generate :iterator.isYieldEach = true;
+      AstNode* set_is_yield_each =
+          new(Z) InstanceSetterNode(Scanner::kNoSourcePos,
+              iterator,
+              String::ZoneHandle(Symbols::IsYieldEach().raw()),
+              new(Z) LiteralNode(TokenPos(), Bool::True()));
+      yield->AddNode(set_is_yield_each);
+    }
+    AwaitMarkerNode* await_marker = new(Z) AwaitMarkerNode();
+    await_marker->set_scope(current_block_->scope);
+    yield->AddNode(await_marker);
+    // Return true to indicate that a value has been generated.
+    ReturnNode* return_true = new(Z) ReturnNode(yield_pos,
+        new(Z) LiteralNode(TokenPos(), Bool::True()));
+    return_true->set_return_type(ReturnNode::kContinuationTarget);
+    yield->AddNode(return_true);
+
+    // If this expression is part of a try block, also append the code for
+    // restoring the saved try context that lives on the stack.
+    const String& async_saved_try_ctx_name =
+        String::Handle(Z, parsed_function()->async_saved_try_ctx_name());
+    if (!async_saved_try_ctx_name.IsNull()) {
+      LocalVariable* async_saved_try_ctx =
+          current_block_->scope->LookupVariable(async_saved_try_ctx_name,
+                                                false);
+      ASSERT(async_saved_try_ctx != NULL);
+      yield->AddNode(new (Z) StoreLocalNode(
+          Scanner::kNoSourcePos,
+          parsed_function()->saved_try_ctx(),
+          new (Z) LoadLocalNode(Scanner::kNoSourcePos, async_saved_try_ctx)));
+    }
+  } else {
+    // yield statement in async* function.
+    ASSERT(innermost_function().IsAsyncGenerator() ||
+           innermost_function().IsAsyncGenClosure());
+
+    LocalVariable* controller_var = LookupLocalScope(Symbols::Controller());
+    ASSERT(controller_var != NULL);
+    // :controller.add[Stream](expr);
+    ArgumentListNode* add_args = new(Z) ArgumentListNode(yield_pos);
+    add_args->Add(expr);
+    AstNode* add_call =
+        new(Z) InstanceCallNode(yield_pos,
+            new(Z) LoadLocalNode(Scanner::kNoSourcePos, controller_var),
+            is_yield_each ? Symbols::AddStream() : Symbols::add(),
+            add_args);
+
+    // if (:controller.isPaused) {
+    //    await_marker;
+    //    continuation_return;
+    // }
+    /*
+    AstNode* is_paused = new(Z) InstanceGetterNode(
+         Scanner::kNoSourcePos,
+         new(Z) LoadLocalNode(Scanner::kNoSourcePos, controller_var),
+         Symbols::isPaused());*/
+
+    SequenceNode* true_branch =
+        new(Z) SequenceNode(Scanner::kNoSourcePos, current_block_->scope);
+    AwaitMarkerNode* await_marker = new(Z) AwaitMarkerNode();
+    await_marker->set_scope(current_block_->scope);
+    true_branch->Add(await_marker);
+    ReturnNode* continuation_return = new(Z) ReturnNode(yield_pos);
+    continuation_return->set_return_type(ReturnNode::kContinuationTarget);
+    true_branch->Add(continuation_return);
+
+    AstNode* if_is_paused =
+        new(Z) IfNode(Scanner::kNoSourcePos, add_call, true_branch, NULL);
+    yield->AddNode(if_is_paused);
+  }
+  return yield;
+}
+
+
 AstNode* Parser::ParseStatement() {
   TRACE_PARSER("ParseStatement");
   AstNode* statement = NULL;
   intptr_t label_pos = 0;
   String* label_name = NULL;
   if (IsIdentifier()) {
     if (LookaheadToken(1) == Token::kCOLON) {
       // Statement starts with a label.
       label_name = CurrentLiteral();
       label_pos = TokenPos();
@@ skipping 19 matching lines @@
     statement = ParseTryStatement(label_name);
   } else if (token == Token::kRETURN) {
     const intptr_t return_pos = TokenPos();
     ConsumeToken();
     if (CurrentToken() != Token::kSEMICOLON) {
       const intptr_t expr_pos = TokenPos();
       if (current_function().IsGenerativeConstructor() &&
           (current_block_->scope->function_level() == 0)) {
         ReportError(expr_pos,
                     "return of a value is not allowed in constructors");
-      } else if (current_function().IsGenerator()) {
+      } else if (current_function().IsGeneratorClosure() &&
+          (current_block_->scope->function_level() == 0)) {
         ReportError(expr_pos, "generator functions may not return a value");
       }
       AstNode* expr = ParseAwaitableExpr(kAllowConst, kConsumeCascades, NULL);
       statement = new(Z) ReturnNode(statement_pos, expr);
     } else {
       if (current_function().IsSyncGenClosure() &&
           (current_block_->scope->function_level() == 0)) {
         // In a synchronous generator, return without an expression
         // returns false, signaling that the iterator terminates and
         // did not yield a value.
         statement = new(Z) ReturnNode(statement_pos,
             new(Z) LiteralNode(return_pos, Bool::False()));
       } else {
         statement = new(Z) ReturnNode(statement_pos);
       }
     }
     AddNodeForFinallyInlining(statement);
     ExpectSemicolon();
   } else if (IsYieldKeyword()) {
-    bool is_yield_each = false;
-    ConsumeToken();
-    ASSERT(innermost_function().IsGenerator() ||
-           innermost_function().IsSyncGenClosure());
-    if (CurrentToken() == Token::kMUL) {
-      is_yield_each = true;
-      ConsumeToken();
-    }
-    AstNode* expr = ParseAwaitableExpr(kAllowConst, kConsumeCascades, NULL);
-    LocalVariable* iterator_param =
-        LookupLocalScope(Symbols::IteratorParameter());
-    ASSERT(iterator_param != NULL);
-    // Generate :iterator.current = expr;
-    AstNode* iterator =
-        new(Z) LoadLocalNode(Scanner::kNoSourcePos, iterator_param);
-    AstNode* store_current =
-        new(Z) InstanceSetterNode(Scanner::kNoSourcePos,
-                                  iterator,
-                                  String::ZoneHandle(Symbols::Current().raw()),
-                                  expr);
-    LetNode* yield = new(Z) LetNode(statement_pos);
-    yield->AddNode(store_current);
-    if (is_yield_each) {
-      // Generate :iterator.isYieldEach = true;
-      AstNode* set_is_yield_each =
-          new(Z) InstanceSetterNode(Scanner::kNoSourcePos,
-              iterator,
-              String::ZoneHandle(Symbols::IsYieldEach().raw()),
-              new(Z) LiteralNode(TokenPos(), Bool::True()));
-      yield->AddNode(set_is_yield_each);
-    }
-    AwaitMarkerNode* await_marker = new(Z) AwaitMarkerNode();
-    await_marker->set_scope(current_block_->scope);
-    yield->AddNode(await_marker);
-    // Return true to indicate that a value has been generated.
-    ReturnNode* return_true = new(Z) ReturnNode(statement_pos,
-        new(Z) LiteralNode(TokenPos(), Bool::True()));
-    return_true->set_return_type(ReturnNode::kContinuationTarget);
-    yield->AddNode(return_true);
-
-    // If this expression is part of a try block, also append the code for
-    // restoring the saved try context that lives on the stack.
-    const String& async_saved_try_ctx_name =
-        String::Handle(Z, parsed_function()->async_saved_try_ctx_name());
-    if (!async_saved_try_ctx_name.IsNull()) {
-      LocalVariable* async_saved_try_ctx =
-          current_block_->scope->LookupVariable(async_saved_try_ctx_name,
-                                                false);
-      ASSERT(async_saved_try_ctx != NULL);
-      yield->AddNode(new (Z) StoreLocalNode(
-          Scanner::kNoSourcePos,
-          parsed_function()->saved_try_ctx(),
-          new (Z) LoadLocalNode(Scanner::kNoSourcePos, async_saved_try_ctx)));
-    }
-
-    statement = yield;
+    statement = ParseYieldStatement();
     ExpectSemicolon();
   } else if (token == Token::kIF) {
     statement = ParseIfStatement(label_name);
   } else if (token == Token::kASSERT) {
     statement = ParseAssertStatement();
     ExpectSemicolon();
   } else if (IsVariableDeclaration()) {
     statement = ParseVariableDeclarationList();
     ExpectSemicolon();
   } else if (IsFunctionDeclaration()) {
@@ skipping 830 matching lines @@
 }
 
 
 AstNode* Parser::ParseUnaryExpr() {
   TRACE_PARSER("ParseUnaryExpr");
   AstNode* expr = NULL;
   const intptr_t op_pos = TokenPos();
   if (IsAwaitKeyword()) {
     TRACE_PARSER("ParseAwaitExpr");
     if (!innermost_function().IsAsyncFunction() &&
-        !innermost_function().IsAsyncClosure()) {
-      ReportError("await operator is only allowed in async function");
+        !innermost_function().IsAsyncClosure() &&
+        !innermost_function().IsAsyncGenerator() &&
+        !innermost_function().IsAsyncGenClosure()) {
+      ReportError("await operator is only allowed in an asynchronous function");
     }
     ConsumeToken();
     parsed_function()->record_await();
     expr = new (Z) AwaitNode(op_pos, ParseUnaryExpr());
   } else if (IsPrefixOperator(CurrentToken())) {
     Token::Kind unary_op = CurrentToken();
     if (unary_op == Token::kSUB) {
       unary_op = Token::kNEGATE;
     }
     ConsumeToken();
@@ skipping 2816 matching lines @@
 void Parser::SkipQualIdent() {
   ASSERT(IsIdentifier());
   ConsumeToken();
   if (CurrentToken() == Token::kPERIOD) {
     ConsumeToken();  // Consume the kPERIOD token.
     ExpectIdentifier("identifier expected after '.'");
   }
 }
 
 }  // namespace dart