Transform functions marked as async

runtime/vm/parser.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/parser.h"
 
 #include "lib/invocation_mirror.h"
 #include "platform/utils.h"
 #include "vm/bootstrap.h"
 #include "vm/class_finalizer.h"
@@ skipping 21 matching lines @@
 #include "vm/tags.h"
 #include "vm/timer.h"
 #include "vm/zone.h"
 
 namespace dart {
 
 DEFINE_FLAG(bool, enable_asserts, false, "Enable assert statements.");
 DEFINE_FLAG(bool, enable_type_checks, false, "Enable type checks.");
 DEFINE_FLAG(bool, trace_parser, false, "Trace parser operations.");
 DEFINE_FLAG(bool, warn_mixin_typedef, true, "Warning on legacy mixin typedef.");
+DEFINE_FLAG(bool, enable_async, false, "Enable async operations.");
 DECLARE_FLAG(bool, error_on_bad_type);
 DECLARE_FLAG(bool, throw_on_javascript_int_overflow);
 DECLARE_FLAG(bool, warn_on_javascript_compatibility);
 
 static void CheckedModeHandler(bool value) {
   FLAG_enable_asserts = value;
   FLAG_enable_type_checks = value;
 }
 
 // --enable-checked-mode and --checked both enable checked mode which is
@@ skipping 2818 matching lines @@
     AstNode* guarded_block_statements =
         new IfNode(Scanner::kNoSourcePos, comparison, ctor_block, NULL);
     current_block_->statements->Add(guarded_block_statements);
   }
   current_block_->statements->Add(new ReturnNode(func.end_token_pos()));
   SequenceNode* statements = CloseBlock();
   return statements;
 }
 
 
+SequenceNode* Parser::BuildAsyncFuture(
+    SequenceNode* body, ParamList* params, intptr_t saved_pos) {
+  const Class& future = Class::ZoneHandle(I,
+      library_.LookupClass(String::Handle(String::New("Future"))));

[hausner, 2014/07/11 20:18:40]

As mentioned in another place, we need to make sure we get the right Future
class. I would factor this out into a function that does all the necessary
checks. Also, please make the strings symbols.

[Michael Lippautz (Google), 2014/07/11 23:40:45]

Done.

+  ASSERT(!future.IsNull());
+  const Function& constructor = Function::ZoneHandle(I,
+      future.LookupFunction(String::Handle(String::New("Future."))));
+  ASSERT(!constructor.IsNull());
+  // Create the closure containing the body of this function.
+  Class& sig_cls = Class::ZoneHandle(I);
+  Type& sig_type = Type::ZoneHandle(I);
+  Function& closure = Function::ZoneHandle(I);
+  String& sig = String::ZoneHandle(I);
+  ParamList closure_params;
+  closure_params.AddFinalParameter(
+      saved_pos,
+      &Symbols::ClosureParameter(),
+      &Type::ZoneHandle(I, Type::DynamicType()));
+  closure = Function::NewClosureFunction(
+      Symbols::AnonymousClosure(),
+      innermost_function(),
+      saved_pos);
+  AddFormalParamsToFunction(&closure_params, closure);
+  closure.set_is_synthetic_container(true);

[hausner, 2014/07/11 20:18:40]

Thinking about how to name things: isn't the "outer" function, the one that
contains the synthesized "return new Future(..)" the synthesized container? It
contains the closure you are constructing here.

[Michael Lippautz (Google), 2014/07/11 23:40:45]

The outer function contains the synthesized body, yes. The outer function does
not need a tag though.

As for the closure (which is only partly synthesized), .skip_parameters()?
Suggestions?

[hausner, 2014/07/12 00:05:24]

I can't think of a better name, don't like skip_parameters either, sorry. Maybe
async_closure()

On 2014/07/11 23:40:45, Michael Lippautz wrote:

[Michael Lippautz (Google), 2014/07/14 20:22:48]

async_closure() it is

+  // TODO(mlippautz): Result type should be the originally declared result type.

[hausner, 2014/07/11 20:18:40]

Not sure what the result type really should be. The "correct" result type of the
outer function is Future<T>, and the result type of the closure is T.

[Michael Lippautz (Google), 2014/07/11 23:40:45]

Clarified the TODO a bit. Leaving for now as we need to decide on what to do
with malformed (i.e. non-Future<T>) result types for the outer functions.

+  closure.set_result_type(AbstractType::Handle(Type::DynamicType()));
+  sig = closure.Signature();
+  sig_cls = library_.LookupLocalClass(sig);
+  if (sig_cls.IsNull()) {
+    sig_cls = Class::NewSignatureClass(sig, closure, script_, saved_pos);
+    library_.AddClass(sig_cls);
+  }
+  closure.set_signature_class(sig_cls);
+  sig_type = sig_cls.SignatureType();
+  if (!sig_type.IsFinalized()) {
+    ClassFinalizer::FinalizeType(
+        sig_cls, sig_type, ClassFinalizer::kCanonicalize);
+  }
+  ASSERT(AbstractType::Handle(I, closure.result_type()).IsResolved());
+  ASSERT(closure.NumParameters() == closure_params.parameters->length());
+  OpenFunctionBlock(closure);
+  AddFormalParamsToScope(&closure_params, current_block_->scope);
+  LocalScope* cur_scope = current_block_->scope;
+  String& var_name = String::ZoneHandle(I);
+  bool var_captured = false;
+  // Capture parameters of the enclosing function.
+  for (intptr_t i = 0; i < params->parameters->length(); i++) {
+    ParamDesc& param_desc = (*params->parameters)[i];
+    var_name = param_desc.var->name().raw();
+    // Ignore implicit closure parameter, but capture 'this' from any enclosing
+    // methods.
+    if (var_name.Equals(Symbols::ClosureParameter())) {
+      continue;
+    }
+    var_captured = cur_scope->CaptureVariable(var_name);
+    ASSERT(var_captured);
+  }
+  LocalScope* body_scope = body->scope();
+  // Capture all variables that have already been captured by the enclosing

[hausner, 2014/07/11 20:18:40]

Captured _by_ the enclosing function, or variables that have been captured in
the enclosing function (by nested functions and closures)?

[Michael Lippautz (Google), 2014/07/11 23:40:45]

We re-capture variables (into the scope of the closure) that have been captured
by parsing the body of the enclosing method (the one we generate the synthesized
Future in).

+  // function.
+  for (intptr_t j = 0; j < body->scope()->num_variables(); j++) {
+    if (body_scope->VariableAt(j)->is_captured()) {
+      var_captured =
+          cur_scope->CaptureVariable(body_scope->VariableAt(j)->name());
+      ASSERT(var_captured);
+    }
+  }
+  ClosureNode* cn = new(I) ClosureNode(saved_pos, closure, NULL, cur_scope);
+  ArgumentListNode* arguments = new (I) ArgumentListNode(saved_pos);
+  arguments->Add(cn);
+  ConstructorCallNode* ccn = new (I) ConstructorCallNode(
+      saved_pos, TypeArguments::ZoneHandle(I), constructor, arguments);
+  ReturnNode* rn = new (I) ReturnNode(saved_pos, ccn);

[hausner, 2014/07/11 20:18:40]

You should use Scanner::kNoSourcePos for synthesized code, so that the debugger
won't break on these locations or won't show the synthesized code.

[Michael Lippautz (Google), 2014/07/11 23:40:45]

Done.

+  current_block_->statements->Add(rn);
+  return CloseBlock();
+}
+
+
 // Parser is at the opening parenthesis of the formal parameter
 // declaration of the function or constructor.
 // Parse the formal parameters and code.
 SequenceNode* Parser::ParseFunc(const Function& func,
                                 Array* default_parameter_values) {
   TRACE_PARSER("ParseFunc");
   Function& saved_innermost_function =
       Function::Handle(I, innermost_function().raw());
   innermost_function_ = func.raw();
 
   // Save current try index. Try index starts at zero for each function.
   intptr_t saved_try_index = last_used_try_index_;
   last_used_try_index_ = 0;
 
+  intptr_t saved_pos = TokenPos();

[hausner, 2014/07/11 20:18:40]

Nit: please name the variable after what it points to, e.g. formal_params_pos.

[Michael Lippautz (Google), 2014/07/11 23:40:45]

Done.

   // TODO(12455) : Need better validation mechanism.
 
   if (func.IsConstructor()) {
     SequenceNode* statements = ParseConstructor(func, default_parameter_values);
     innermost_function_ = saved_innermost_function.raw();
     last_used_try_index_ = saved_try_index;
     return statements;
   }
 
   ASSERT(!func.IsConstructor());
@@ skipping 14 matching lines @@
     ASSERT(current_class().raw() == func.Owner());
     params.AddReceiver(ReceiverType(current_class()), func.token_pos());
   } else if (func.IsFactory()) {
     // The first parameter of a factory is the TypeArguments vector of
     // the type of the instance to be allocated.
     params.AddFinalParameter(
         TokenPos(),
         &Symbols::TypeArgumentsParameter(),
         &Type::ZoneHandle(I, Type::DynamicType()));
   }
-  ASSERT((CurrentToken() == Token::kLPAREN) || func.IsGetterFunction());
+  ASSERT((CurrentToken() == Token::kLPAREN) || func.IsGetterFunction() ||
+      func.is_synthetic_container());
   const bool allow_explicit_default_values = true;
   if (func.IsGetterFunction()) {
     // Populate function scope with the formal parameters. Since in this case
     // we are compiling a getter this will at most populate the receiver.
     AddFormalParamsToScope(&params, current_block_->scope);
+  } else if (func.is_synthetic_container() && Function::Handle(
+        func.parent_function()).IsGetterFunction()) {
+    AddFormalParamsToScope(&params, current_block_->scope);
+    ASSERT(AbstractType::Handle(I, func.result_type()).IsResolved());
+    ASSERT(func.NumParameters() == params.parameters->length());
+  } else if (func.is_synthetic_container()) {
+    AddFormalParamsToScope(&params, current_block_->scope);
+    ASSERT(AbstractType::Handle(I, func.result_type()).IsResolved());
+    ASSERT(func.NumParameters() == params.parameters->length());
+    ParseFormalParameterList(allow_explicit_default_values, false, &params);
+    // Parse them away.
   } else {
     ParseFormalParameterList(allow_explicit_default_values, false, &params);
 
     // The number of parameters and their type are not yet set in local
     // functions, since they are not 'top-level' parsed.
     if (func.IsLocalFunction()) {
       AddFormalParamsToFunction(&params, func);
     }
     SetupDefaultsForOptionalParams(&params, default_parameter_values);
     ASSERT(AbstractType::Handle(I, func.result_type()).IsResolved());
@@ skipping 20 matching lines @@
       if (IsInstantiatorRequired()) {
         // Make sure that the receiver of the enclosing instance function
         // (or implicit first parameter of an enclosing factory) is marked as
         // captured if type checks are enabled, because they may access it to
         // instantiate types.
         CaptureInstantiator();
       }
     }
   }
 
+  intptr_t async_type = ParseAsyncSpecifier();
+  if (async_type == 1) {
+    func.set_modifier(RawFunction::kAsync);
+  }
+
   OpenBlock();  // Open a nested scope for the outermost function block.
   intptr_t end_token_pos = 0;
   if (CurrentToken() == Token::kLBRACE) {
     ConsumeToken();
     if (String::Handle(I, func.name()).Equals(
         Symbols::EqualOperator())) {
       const Class& owner = Class::Handle(I, func.Owner());
       if (!owner.IsObjectClass()) {
         AddEqualityNullCheck();
       }
@@ skipping 41 matching lines @@
                                &func));  // Unpatched external function.
     end_token_pos = TokenPos();
   } else {
     UnexpectedToken();
   }
 
   ASSERT(func.end_token_pos() == func.token_pos() ||
          func.end_token_pos() == end_token_pos);
   func.set_end_token_pos(end_token_pos);
   SequenceNode* body = CloseBlock();
+  if (func.IsAsyncFunction() && !func.is_synthetic_container()) {
+    body = BuildAsyncFuture(body, &params, saved_pos);
+  }
   current_block_->statements->Add(body);
   innermost_function_ = saved_innermost_function.raw();
   last_used_try_index_ = saved_try_index;
   return CloseBlock();
 }
 
 
 void Parser::AddEqualityNullCheck() {
   AstNode* argument =
       new LoadLocalNode(Scanner::kNoSourcePos,
@@ skipping 293 matching lines @@
   ASSERT((method->redirect_name == NULL) || method->IsConstructor());
 
   if (method->IsConstructor() &&
       method->has_external &&
       method->params.has_field_initializer) {
     ReportError(method->name_pos,
                 "external constructor '%s' may not have field initializers",
                 method->name->ToCString());
   }
 
+  ParseAsyncSpecifier();
+
   intptr_t method_end_pos = TokenPos();
   if ((CurrentToken() == Token::kLBRACE) ||
       (CurrentToken() == Token::kARROW)) {
     if (method->has_abstract) {
       ReportError(TokenPos(),
                   "abstract method '%s' may not have a function body",
                   method->name->ToCString());
     } else if (method->has_external) {
       ReportError(TokenPos(),
                   "external %s '%s' may not have a function body",
@@ skipping 1414 matching lines @@
     } else if (CurrentToken() == Token::kSEMICOLON) {
       ConsumeToken();
       break;
     } else {
       ExpectSemicolon();  // Reports error.
     }
   }
 }
 
 
+intptr_t Parser::ParseAsyncSpecifier() {

[hausner, 2014/07/11 20:18:41]

What does the integer return value mean. Should probably use bool or an
enumeration type eventually.

[Michael Lippautz (Google), 2014/07/11 23:40:45]

Return value is now RawFunction::Modifier. This can be easily extended to
support async* and sync*.

+  if (FLAG_enable_async) {
+    if (IsLiteral("async")) {
+      const Class& future = Class::Handle(library_.LookupClass(
+          String::Handle(String::New("Future"))));
+      if (future.IsNull()) {
+        ReportError("async modifier requires dart:async to be imported");

[hausner, 2014/07/11 20:18:41]

I think we need to have a discussion with Gilad whether the user must explicitly
import dart:async.

Also, is it really necessarily an error if the program does not know the type
"Future"?

Third, what does it mean if "Future" is found but it's not the one from
dart:async? If we do require that dart:async is imported (explicitly or
implicitly) we need to make sure that we find the right Future class.

 

[Michael Lippautz (Google), 2014/07/11 23:40:45]

Factored out. We still bomb until we know what to actually do...

+      }
+      ConsumeToken();
+      return 1;
+    }
+  }
+  return 0;
+}
+
+
 void Parser::ParseTopLevelFunction(TopLevel* top_level,
                                    intptr_t metadata_pos) {
   TRACE_PARSER("ParseTopLevelFunction");
   const intptr_t decl_begin_pos = TokenPos();
   AbstractType& result_type = Type::Handle(I, Type::DynamicType());
   const bool is_static = true;
   bool is_external = false;
   bool is_patch = false;
   if (is_patch_source() &&
       (CurrentToken() == Token::kIDENT) &&
@@ skipping 33 matching lines @@
   }
   // A setter named x= may co-exist with a function named x, thus we do
   // not need to check setters.
 
   CheckToken(Token::kLPAREN);
   const intptr_t function_pos = TokenPos();
   ParamList params;
   const bool allow_explicit_default_values = true;
   ParseFormalParameterList(allow_explicit_default_values, false, &params);
 
+  intptr_t async_type = ParseAsyncSpecifier();
+
   intptr_t function_end_pos = function_pos;
   bool is_native = false;
   if (is_external) {
     function_end_pos = TokenPos();
     ExpectSemicolon();
   } else if (CurrentToken() == Token::kLBRACE) {
     SkipBlock();
     function_end_pos = TokenPos();
     ExpectToken(Token::kRBRACE);
   } else if (CurrentToken() == Token::kARROW) {
@@ skipping 14 matching lines @@
                     RawFunction::kRegularFunction,
                     is_static,
                     /* is_const = */ false,
                     /* is_abstract = */ false,
                     is_external,
                     is_native,
                     current_class(),
                     decl_begin_pos));
   func.set_result_type(result_type);
   func.set_end_token_pos(function_end_pos);
+  if (async_type == 1) {
+    func.set_modifier(RawFunction::kAsync);
+  }
   if (is_native && library_.is_dart_scheme() && library_.IsPrivate(func_name)) {
     func.set_is_visible(false);
   }
   AddFormalParamsToFunction(&params, func);
   top_level->functions.Add(func);
   if (!is_patch) {
     library_.AddObject(func, func_name);
   } else {
     library_.ReplaceObject(func, func_name);
   }
@@ skipping 82 matching lines @@
   if (found && !is_patch) {
     ReportError(name_pos, "%s for '%s' is already defined",
                 is_getter ? "getter" : "setter",
                 field_name->ToCString());
   } else if (!found && is_patch) {
     ReportError(name_pos, "missing %s for '%s' cannot be patched",
                 is_getter ? "getter" : "setter",
                 field_name->ToCString());
   }
 
+  intptr_t async_type = ParseAsyncSpecifier();
+
   intptr_t accessor_end_pos = accessor_pos;
   bool is_native = false;
   if (is_external) {
     accessor_end_pos = TokenPos();
     ExpectSemicolon();
   } else if (CurrentToken() == Token::kLBRACE) {
     SkipBlock();
     accessor_end_pos = TokenPos();
     ExpectToken(Token::kRBRACE);
   } else if (CurrentToken() == Token::kARROW) {
@@ skipping 15 matching lines @@
                                 RawFunction::kSetterFunction,
                     is_static,
                     /* is_const = */ false,
                     /* is_abstract = */ false,
                     is_external,
                     is_native,
                     current_class(),
                     decl_begin_pos));
   func.set_result_type(result_type);
   func.set_end_token_pos(accessor_end_pos);
+  if (async_type == 1) {
+    func.set_modifier(RawFunction::kAsync);
+  }
   if (is_native && library_.is_dart_scheme() &&
       library_.IsPrivate(accessor_name)) {
     func.set_is_visible(false);
   }
   AddFormalParamsToFunction(&params, func);
   top_level->functions.Add(func);
   if (!is_patch) {
     library_.AddObject(func, accessor_name);
   } else {
     library_.ReplaceObject(func, accessor_name);
@@ skipping 1184 matching lines @@
   } else {
     SetPosition(saved_pos);
     return false;
   }
   // Check parameter list and the following token.
   if (CurrentToken() == Token::kLPAREN) {
     SkipToMatchingParenthesis();
     if ((CurrentToken() == Token::kLBRACE) ||
         (CurrentToken() == Token::kARROW) ||
         (is_top_level_ && IsLiteral("native")) ||
-        is_external) {
+        is_external ||
+        (FLAG_enable_async && IsLiteral("async"))) {

[hausner, 2014/07/11 20:18:40]

You should create a new Symbol for "async" (in symbols.h) and do a raw pointer
comparison, which is much faster than IsLiteral().

[Michael Lippautz (Google), 2014/07/11 23:40:45]

Done.

       SetPosition(saved_pos);
       return true;
     }
   }
   SetPosition(saved_pos);
   return false;
 }
 
 
 bool Parser::IsTopLevelAccessor() {
@@ skipping 4505 matching lines @@
     }
     ASSERT(result.IsInstance());
     Instance& value = Instance::ZoneHandle(I);
     value ^= result.raw();
     value = TryCanonicalize(value, TokenPos());
     return value;
   }
 }
 
 
 void Parser::SkipFunctionLiteral() {

[hausner, 2014/07/11 20:18:40]

You probably need to handle the async specifiers here too.

[Michael Lippautz (Google), 2014/07/11 23:40:45]

Done.

   if (IsIdentifier()) {
     if (LookaheadToken(1) != Token::kLPAREN) {
       SkipType(true);
     }
     ExpectIdentifier("function name expected");
   }
   if (CurrentToken() == Token::kLPAREN) {
     const bool allow_explicit_default_values = true;
     ParamList params;
     params.skipped = true;
@@ skipping 312 matching lines @@
 void Parser::SkipQualIdent() {
   ASSERT(IsIdentifier());
   ConsumeToken();
   if (CurrentToken() == Token::kPERIOD) {
     ConsumeToken();  // Consume the kPERIOD token.
     ExpectIdentifier("identifier expected after '.'");
   }
 }
 
 }  // namespace dart