Calling a generator function returns a generator object

src/parser.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 468 matching lines @@
   ~BlockState() { parser_->top_scope_ = outer_scope_; }
 
  private:
   Parser* parser_;
   Scope* outer_scope_;
 };
 
 
 Parser::FunctionState::FunctionState(Parser* parser,
                                      Scope* scope,
-                                     bool is_generator,
                                      Isolate* isolate)
     : next_materialized_literal_index_(JSFunction::kLiteralsPrefixSize),
       next_handler_index_(0),
       expected_property_count_(0),
-      is_generator_(is_generator),
       only_simple_this_property_assignments_(false),
       this_property_assignments_(isolate->factory()->empty_fixed_array()),
       parser_(parser),
       outer_function_state_(parser->current_function_state_),
       outer_scope_(parser->top_scope_),
+      generator_iterator_variable_(NULL),
       saved_ast_node_id_(isolate->ast_node_id()),
       factory_(isolate, parser->zone()) {
   parser->top_scope_ = scope;
   parser->current_function_state_ = this;
   isolate->set_ast_node_id(BailoutId::FirstUsable().ToInt());
 }
 
 
 Parser::FunctionState::~FunctionState() {
   parser_->top_scope_ = outer_scope_;
@@ skipping 127 matching lines @@
     scope->set_start_position(0);
     scope->set_end_position(source->length());
 
     // Compute the parsing mode.
     Mode mode = (FLAG_lazy && allow_lazy_) ? PARSE_LAZILY : PARSE_EAGERLY;
     if (allow_natives_syntax_ || extension_ != NULL || scope->is_eval_scope()) {
       mode = PARSE_EAGERLY;
     }
     ParsingModeScope parsing_mode(this, mode);
 
-    bool is_generator = false;
     // Enters 'scope'.
-    FunctionState function_state(this, scope, is_generator, isolate());
+    FunctionState function_state(this, scope, isolate());
 
     top_scope_->SetLanguageMode(info->language_mode());
     ZoneList<Statement*>* body = new(zone()) ZoneList<Statement*>(16, zone());
     bool ok = true;
     int beg_loc = scanner().location().beg_pos;
     ParseSourceElements(body, Token::EOS, info->is_eval(), true, &ok);
     if (ok && !top_scope_->is_classic_mode()) {
       CheckOctalLiteral(beg_loc, scanner().location().end_pos, &ok);
     }
 
@@ skipping 93 matching lines @@
   FunctionLiteral* result = NULL;
 
   {
     // Parse the function literal.
     Scope* scope = NewScope(top_scope_, GLOBAL_SCOPE);
     info()->SetGlobalScope(scope);
     if (!info()->closure().is_null()) {
       scope = Scope::DeserializeScopeChain(info()->closure()->context(), scope,
                                            zone());
     }
-    bool is_generator = false;  // Top scope is not a generator.
-    FunctionState function_state(this, scope, is_generator, isolate());
+    FunctionState function_state(this, scope, isolate());
     ASSERT(scope->language_mode() != STRICT_MODE || !info()->is_classic_mode());
     ASSERT(scope->language_mode() != EXTENDED_MODE ||
            info()->is_extended_mode());
     ASSERT(info()->language_mode() == shared_info->language_mode());
     scope->SetLanguageMode(shared_info->language_mode());
     FunctionLiteral::Type type = shared_info->is_expression()
         ? (shared_info->is_anonymous()
               ? FunctionLiteral::ANONYMOUS_EXPRESSION
               : FunctionLiteral::NAMED_EXPRESSION)
         : FunctionLiteral::DECLARATION;
@@ skipping 253 matching lines @@
 
 
 void* Parser::ParseSourceElements(ZoneList<Statement*>* processor,
                                   int end_token,
                                   bool is_eval,
                                   bool is_global,
                                   bool* ok) {
   // SourceElements ::
   //   (ModuleElement)* <end_token>
 
+  // For generators, allocate and yield an iterator on function entry.

[Michael Starzinger, 2013/04/08 13:14:05]

As discussed offline, it scares me a little bit to model all of this at the AST
level. I agree that modeling it on some high-level IR would be great, but this
introduces AST nodes that have absolutely no correspondence to the source code.

[rossberg, 2013/04/09 09:37:44]

I think it's perfectly fine. With the more high-level features in ES6+ the AST
will generally move away further from being a direct representation of the
surface syntax. We already do similar things for other constructs, e.g. 'try',
let blocks, and modules, all of which introduce internal statements.

Perhaps the issue is less with the approach as such but with calling the thing
'AST'.

[rossberg, 2013/04/09 09:37:44]

I don't understand, though, why this code is here and not in
ParseFunctionLiteral. That seems odd.

[Michael Starzinger, 2013/04/09 10:40:14]

I am convinced, please ignore my initial comment.

+  if (is_generator()) {
+    ZoneList<Expression*>* arguments =
+        new(zone()) ZoneList<Expression*>(1, zone());
+    arguments->Add(factory()->NewThisFunction(), zone());
+    CallRuntime* allocation = factory()->NewCallRuntime(
+        isolate()->factory()->empty_string(),
+        Runtime::FunctionForId(Runtime::kCreateJSGeneratorIterator),
+        arguments);
+    VariableProxy* init_proxy = factory()->NewVariableProxy(
+        current_function_state_->generator_iterator_variable());
+    Assignment* assignment = factory()->NewAssignment(
+        Token::INIT_VAR, init_proxy, allocation, RelocInfo::kNoPosition);
+    VariableProxy* get_proxy = factory()->NewVariableProxy(
+        current_function_state_->generator_iterator_variable());
+    Yield* yield = factory()->NewYield(
+        get_proxy, assignment, false, RelocInfo::kNoPosition);
+    processor->Add(factory()->NewExpressionStatement(yield), zone());
+  }
+
   // Allocate a target stack to use for this set of source
   // elements. This way, all scripts and functions get their own
   // target stack thus avoiding illegal breaks and continues across
   // functions.
   TargetScope scope(&this->target_stack_);
 
   ASSERT(processor != NULL);
   ThisNamedPropertyAssignmentFinder this_property_assignment_finder(isolate(),
                                                                     zone());
   bool directive_prologue = true;     // Parsing directive prologue.
@@ skipping 2050 matching lines @@
   return factory()->NewAssignment(op, expression, right, pos);
 }
 
 
 Expression* Parser::ParseYieldExpression(bool* ok) {
   // YieldExpression ::
   //   'yield' '*'? AssignmentExpression
   int position = scanner().peek_location().beg_pos;
   Expect(Token::YIELD, CHECK_OK);
   bool is_yield_star = Check(Token::MUL);
+  Expression* iterator = factory()->NewVariableProxy(
+      current_function_state_->generator_iterator_variable());
   Expression* expression = ParseAssignmentExpression(false, CHECK_OK);
-  return factory()->NewYield(expression, is_yield_star, position);
+  return factory()->NewYield(iterator, expression, is_yield_star, position);
 }
 
 
 // Precedence = 3
 Expression* Parser::ParseConditionalExpression(bool accept_IN, bool* ok) {
   // ConditionalExpression ::
   //   LogicalOrExpression
   //   LogicalOrExpression '?' AssignmentExpression ':' AssignmentExpression
 
   // We start using the binary expression parser for prec >= 4 only!
@@ skipping 1263 matching lines @@
   FunctionLiteral::ParameterFlag duplicate_parameters =
       FunctionLiteral::kNoDuplicateParameters;
   FunctionLiteral::IsParenthesizedFlag parenthesized = parenthesized_function_
       ? FunctionLiteral::kIsParenthesized
       : FunctionLiteral::kNotParenthesized;
   FunctionLiteral::IsGeneratorFlag generator = is_generator
       ? FunctionLiteral::kIsGenerator
       : FunctionLiteral::kNotGenerator;
   AstProperties ast_properties;
   // Parse function body.
-  { FunctionState function_state(this, scope, is_generator, isolate());
+  { FunctionState function_state(this, scope, isolate());
     top_scope_->SetScopeName(function_name);
-    if (is_generator)
+
+    // Generators allocate their variables on the heap, to avoid copying on
+    // suspend and resume.  The iterator object also gets a local, which
+    // indicates this FunctionState is a generator, while also connecting the
+    // definition of the iterator with its uses in "yield" expressions.
+    if (is_generator) {
       top_scope_->ForceContextAllocation();
+      Handle<String> tempname = isolate()->factory()->InternalizeOneByteString(
+          STATIC_ASCII_VECTOR(".generator_iterator"));

[Michael Starzinger, 2013/04/08 13:14:05]

The current generator instance needs to be stored in the context, so I totally
see the need for having a context slot. But do we really need a temporary
variable for this and model all of this on the AST level? Can we just assign a
fixed context slot to hold the current generator instance?

[rossberg, 2013/04/09 09:37:44]

Again, I think this is fine. Hard-coding context slots seems far more hacky and
unmodular to me than introducing a temp var on the syntax level.

[Michael Starzinger, 2013/04/09 10:40:14]

I am convinced, please ignore my initial comment.

+      Variable* temp = top_scope_->DeclarationScope()->NewTemporary(tempname);
+      function_state.set_generator_iterator_variable(temp);
+    }
 
     //  FormalParameterList ::
     //    '(' (Identifier)*[','] ')'
     Expect(Token::LPAREN, CHECK_OK);
     scope->set_start_position(scanner().location().beg_pos);
     Scanner::Location name_loc = Scanner::Location::invalid();
     Scanner::Location dupe_loc = Scanner::Location::invalid();
     Scanner::Location reserved_loc = Scanner::Location::invalid();
 
     bool done = (peek() == Token::RPAREN);
@@ skipping 1597 matching lines @@
       ASSERT(info->isolate()->has_pending_exception());
     } else {
       result = parser.ParseProgram();
     }
   }
   info->SetFunction(result);
   return (result != NULL);
 }
 
 } }  // namespace v8::internal