Add parser support for generators.

src/parser.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 468 matching lines @@
   ~BlockState() { parser_->top_scope_ = outer_scope_; }
 
  private:
   Parser* parser_;
   Scope* outer_scope_;
 };
 
 
 Parser::FunctionState::FunctionState(Parser* parser,
                                      Scope* scope,
+                                     bool is_generator,
                                      Isolate* isolate)
     : next_materialized_literal_index_(JSFunction::kLiteralsPrefixSize),
       next_handler_index_(0),
       expected_property_count_(0),
+      is_generator_(is_generator),
       only_simple_this_property_assignments_(false),
       this_property_assignments_(isolate->factory()->empty_fixed_array()),
       parser_(parser),
       outer_function_state_(parser->current_function_state_),
       outer_scope_(parser->top_scope_),
       saved_ast_node_id_(isolate->ast_node_id()),
       factory_(isolate, parser->zone()) {
   parser->top_scope_ = scope;
   parser->current_function_state_ = this;
   isolate->set_ast_node_id(BailoutId::FirstUsable().ToInt());
@@ skipping 132 matching lines @@
     scope->set_start_position(0);
     scope->set_end_position(source->length());
 
     // Compute the parsing mode.
     Mode mode = (FLAG_lazy && allow_lazy_) ? PARSE_LAZILY : PARSE_EAGERLY;
     if (allow_natives_syntax_ || extension_ != NULL || scope->is_eval_scope()) {
       mode = PARSE_EAGERLY;
     }
     ParsingModeScope parsing_mode(this, mode);
 
-    FunctionState function_state(this, scope, isolate());  // Enters 'scope'.
+    bool is_generator = false;
+    // Enters 'scope'.
+    FunctionState function_state(this, scope, is_generator, isolate());
+
     top_scope_->SetLanguageMode(info->language_mode());
     ZoneList<Statement*>* body = new(zone()) ZoneList<Statement*>(16, zone());
     bool ok = true;
     int beg_loc = scanner().location().beg_pos;
     ParseSourceElements(body, Token::EOS, info->is_eval(), true, &ok);
     if (ok && !top_scope_->is_classic_mode()) {
       CheckOctalLiteral(beg_loc, scanner().location().end_pos, &ok);
     }
 
     if (ok && is_extended_mode()) {
@@ skipping 17 matching lines @@
           body,
           function_state.materialized_literal_count(),
           function_state.expected_property_count(),
           function_state.handler_count(),
           function_state.only_simple_this_property_assignments(),
           function_state.this_property_assignments(),
           0,
           FunctionLiteral::kNoDuplicateParameters,
           FunctionLiteral::ANONYMOUS_EXPRESSION,
           FunctionLiteral::kGlobalOrEval,
-          FunctionLiteral::kNotParenthesized);
+          FunctionLiteral::kNotParenthesized,
+          FunctionLiteral::kNotGenerator);
       result->set_ast_properties(factory()->visitor()->ast_properties());
     } else if (stack_overflow_) {
       isolate()->StackOverflow();
     }
   }
 
   // Make sure the target stack is empty.
   ASSERT(target_stack_ == NULL);
 
   // If there was a syntax error we have to get rid of the AST
@@ skipping 53 matching lines @@
   FunctionLiteral* result = NULL;
 
   {
     // Parse the function literal.
     Scope* scope = NewScope(top_scope_, GLOBAL_SCOPE);
     info()->SetGlobalScope(scope);
     if (!info()->closure().is_null()) {
       scope = Scope::DeserializeScopeChain(info()->closure()->context(), scope,
                                            zone());
     }
-    FunctionState function_state(this, scope, isolate());
+    bool is_generator = false;  // Top scope is not a generator.
+    FunctionState function_state(this, scope, is_generator, isolate());
     ASSERT(scope->language_mode() != STRICT_MODE || !info()->is_classic_mode());
     ASSERT(scope->language_mode() != EXTENDED_MODE ||
            info()->is_extended_mode());
     ASSERT(info()->language_mode() == shared_info->language_mode());
     scope->SetLanguageMode(shared_info->language_mode());
     FunctionLiteral::Type type = shared_info->is_expression()
         ? (shared_info->is_anonymous()
               ? FunctionLiteral::ANONYMOUS_EXPRESSION
               : FunctionLiteral::NAMED_EXPRESSION)
         : FunctionLiteral::DECLARATION;
     bool ok = true;
     result = ParseFunctionLiteral(name,
                                   false,  // Strict mode name already checked.
+                                  shared_info->is_generator(),
                                   RelocInfo::kNoPosition,
                                   type,
                                   &ok);
     // Make sure the results agree.
     ASSERT(ok == (result != NULL));
   }
 
   // Make sure the target stack is empty.
   ASSERT(target_stack_ == NULL);
 
@@ skipping 344 matching lines @@
   //    Statement
   //    FunctionDeclaration
   //
   // In harmony mode we allow additionally the following productions
   // ModuleElement:
   //    LetDeclaration
   //    ConstDeclaration
   //    ModuleDeclaration
   //    ImportDeclaration
   //    ExportDeclaration
+  //    GeneratorDeclaration
 
   switch (peek()) {
     case Token::FUNCTION:
       return ParseFunctionDeclaration(NULL, ok);
     case Token::LET:
     case Token::CONST:
       return ParseVariableStatement(kModuleElement, NULL, ok);
     case Token::IMPORT:
       return ParseImportDeclaration(ok);
     case Token::EXPORT:
@@ skipping 278 matching lines @@
 
   return block;
 }
 
 
 Statement* Parser::ParseExportDeclaration(bool* ok) {
   // ExportDeclaration:
   //    'export' Identifier (',' Identifier)* ';'
   //    'export' VariableDeclaration
   //    'export' FunctionDeclaration
+  //    'export' GeneratorDeclaration
   //    'export' ModuleDeclaration
   //
   // TODO(ES6): implement structuring ExportSpecifiers
 
   Expect(Token::EXPORT, CHECK_OK);
 
   Statement* result = NULL;
   ZoneStringList names(1, zone());
   switch (peek()) {
     case Token::IDENTIFIER: {
@@ skipping 59 matching lines @@
                                      bool* ok) {
   // (Ecma 262 5th Edition, clause 14):
   // SourceElement:
   //    Statement
   //    FunctionDeclaration
   //
   // In harmony mode we allow additionally the following productions
   // BlockElement (aka SourceElement):
   //    LetDeclaration
   //    ConstDeclaration
+  //    GeneratorDeclaration
 
   switch (peek()) {
     case Token::FUNCTION:
       return ParseFunctionDeclaration(NULL, ok);
     case Token::LET:
     case Token::CONST:
       return ParseVariableStatement(kModuleElement, NULL, ok);
     default:
       return ParseStatement(labels, ok);
   }
@@ skipping 99 matching lines @@
     }
 
     case Token::FUNCTION: {
       // FunctionDeclaration is only allowed in the context of SourceElements
       // (Ecma 262 5th Edition, clause 14):
       // SourceElement:
       //    Statement
       //    FunctionDeclaration
       // Common language extension is to allow function declaration in place
       // of any statement. This language extension is disabled in strict mode.
+      //
+      // In Harmony mode, this case also handles the extension:
+      // Statement:
+      //    GeneratorDeclaration
       if (!top_scope_->is_classic_mode()) {
         ReportMessageAt(scanner().peek_location(), "strict_function",
                         Vector<const char*>::empty());
         *ok = false;
         return NULL;
       }
       return ParseFunctionDeclaration(NULL, ok);
     }
 
     case Token::DEBUGGER:
@@ skipping 242 matching lines @@
       factory()->NewSharedFunctionInfoLiteral(shared);
   return factory()->NewExpressionStatement(
       factory()->NewAssignment(
           Token::INIT_VAR, proxy, lit, RelocInfo::kNoPosition));
 }
 
 
 Statement* Parser::ParseFunctionDeclaration(ZoneStringList* names, bool* ok) {
   // FunctionDeclaration ::
   //   'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}'
+  // GeneratorDeclaration ::
+  //   'function' '*' Identifier '(' FormalParameterListopt ')'
+  //      '{' FunctionBody '}'
   Expect(Token::FUNCTION, CHECK_OK);
   int function_token_position = scanner().location().beg_pos;
+  bool is_generator = FLAG_harmony_generators && Check(Token::MUL);
   bool is_strict_reserved = false;
   Handle<String> name = ParseIdentifierOrStrictReservedWord(
       &is_strict_reserved, CHECK_OK);
   FunctionLiteral* fun = ParseFunctionLiteral(name,
                                               is_strict_reserved,
+                                              is_generator,
                                               function_token_position,
                                               FunctionLiteral::DECLARATION,
                                               CHECK_OK);
   // Even if we're not at the top-level of the global or a function
   // scope, we treat it as such and introduce the function with its
   // initial value upon entering the corresponding scope.
   // In extended mode, a function behaves as a lexical binding, except in the
   // global scope.
   VariableMode mode =
       is_extended_mode() && !top_scope_->is_global_scope() ? LET : VAR;
@@ skipping 1087 matching lines @@
         factory()->NewBinaryOperation(Token::COMMA, result, right, position);
   }
   return result;
 }
 
 
 // Precedence = 2
 Expression* Parser::ParseAssignmentExpression(bool accept_IN, bool* ok) {
   // AssignmentExpression ::
   //   ConditionalExpression
+  //   YieldExpression
   //   LeftHandSideExpression AssignmentOperator AssignmentExpression
 
+  if (peek() == Token::YIELD && is_generator()) {
+    return ParseYieldExpression(ok);
+  }
+
   if (fni_ != NULL) fni_->Enter();
   Expression* expression = ParseConditionalExpression(accept_IN, CHECK_OK);
 
   if (!Token::IsAssignmentOp(peek())) {
     if (fni_ != NULL) fni_->Leave();
     // Parsed conditional expression only (no assignment).
     return expression;
   }
 
   // Signal a reference error if the expression is an invalid left-hand
@@ skipping 48 matching lines @@
     } else {
       fni_->RemoveLastFunction();
     }
     fni_->Leave();
   }
 
   return factory()->NewAssignment(op, expression, right, pos);
 }
 
 
+Expression* Parser::ParseYieldExpression(bool* ok) {
+  // YieldExpression ::
+  //   'yield' '*'? AssignmentExpression
+  int position = scanner().peek_location().beg_pos;
+  Expect(Token::YIELD, CHECK_OK);
+  bool is_yield_star = Check(Token::MUL);
+  Expression* expression = ParseAssignmentExpression(false, CHECK_OK);
+  return factory()->NewYield(expression, is_yield_star, position);
+}
+
+
 // Precedence = 3
 Expression* Parser::ParseConditionalExpression(bool accept_IN, bool* ok) {
   // ConditionalExpression ::
   //   LogicalOrExpression
   //   LogicalOrExpression '?' AssignmentExpression ':' AssignmentExpression
 
   // We start using the binary expression parser for prec >= 4 only!
   Expression* expression = ParseBinaryExpression(4, accept_IN, CHECK_OK);
   if (peek() != Token::CONDITIONAL) return expression;
   Consume(Token::CONDITIONAL);
@@ skipping 356 matching lines @@
                                                          bool* ok) {
   // MemberExpression ::
   //   (PrimaryExpression | FunctionLiteral)
   //     ('[' Expression ']' | '.' Identifier | Arguments)*
 
   // Parse the initial primary or function expression.
   Expression* result = NULL;
   if (peek() == Token::FUNCTION) {
     Expect(Token::FUNCTION, CHECK_OK);
     int function_token_position = scanner().location().beg_pos;
+    bool is_generator = FLAG_harmony_generators && Check(Token::MUL);
     Handle<String> name;
     bool is_strict_reserved_name = false;
     if (peek_any_identifier()) {
       name = ParseIdentifierOrStrictReservedWord(&is_strict_reserved_name,
                                                  CHECK_OK);
     }
     FunctionLiteral::Type type = name.is_null()
         ? FunctionLiteral::ANONYMOUS_EXPRESSION
         : FunctionLiteral::NAMED_EXPRESSION;
     result = ParseFunctionLiteral(name,
                                   is_strict_reserved_name,
+                                  is_generator,
                                   function_token_position,
                                   type,
                                   CHECK_OK);
   } else {
     result = ParsePrimaryExpression(CHECK_OK);
   }
 
   while (true) {
     switch (peek()) {
       case Token::LBRACK: {
@@ skipping 123 matching lines @@
       Consume(Token::TRUE_LITERAL);
       result = factory()->NewLiteral(isolate()->factory()->true_value());
       break;
 
     case Token::FALSE_LITERAL:
       Consume(Token::FALSE_LITERAL);
       result = factory()->NewLiteral(isolate()->factory()->false_value());
       break;
 
     case Token::IDENTIFIER:
+    case Token::YIELD:
     case Token::FUTURE_STRICT_RESERVED_WORD: {
       Handle<String> name = ParseIdentifier(CHECK_OK);
       if (fni_ != NULL) fni_->PushVariableName(name);
       // The name may refer to a module instance object, so its type is unknown.
 #ifdef DEBUG
       if (FLAG_print_interface_details)
         PrintF("# Variable %s ", name->ToAsciiArray());
 #endif
       Interface* interface = Interface::NewUnknown(zone());
       result = top_scope_->NewUnresolved(
@@ skipping 385 matching lines @@
       next == Token::STRING || is_keyword) {
     Handle<String> name;
     if (is_keyword) {
       name = isolate_->factory()->InternalizeUtf8String(Token::String(next));
     } else {
       name = GetSymbol(CHECK_OK);
     }
     FunctionLiteral* value =
         ParseFunctionLiteral(name,
                              false,   // reserved words are allowed here
+                             false,   // not a generator
                              RelocInfo::kNoPosition,
                              FunctionLiteral::ANONYMOUS_EXPRESSION,
                              CHECK_OK);
     // Allow any number of parameters for compatibilty with JSC.
     // Specification only allows zero parameters for get and one for set.
     return factory()->NewObjectLiteralProperty(is_getter, value);
   } else {
     ReportUnexpectedToken(next);
     *ok = false;
     return NULL;
@@ skipping 281 matching lines @@
   int properties_;
   LanguageMode mode_;
   // For error messages.
   const char* message_;
   const char* argument_opt_;
 };
 
 
 FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
                                               bool name_is_strict_reserved,
+                                              bool is_generator,
                                               int function_token_position,
                                               FunctionLiteral::Type type,
                                               bool* ok) {
   // Function ::
   //   '(' FormalParameterList? ')' '{' FunctionBody '}'
 
   // Anonymous functions were passed either the empty symbol or a null
   // handle as the function name.  Remember if we were passed a non-empty
   // handle to decide whether to invoke function name inference.
   bool should_infer_name = function_name.is_null();
@@ skipping 14 matching lines @@
   int materialized_literal_count = -1;
   int expected_property_count = -1;
   int handler_count = 0;
   bool only_simple_this_property_assignments;
   Handle<FixedArray> this_property_assignments;
   FunctionLiteral::ParameterFlag duplicate_parameters =
       FunctionLiteral::kNoDuplicateParameters;
   FunctionLiteral::IsParenthesizedFlag parenthesized = parenthesized_function_
       ? FunctionLiteral::kIsParenthesized
       : FunctionLiteral::kNotParenthesized;
+  FunctionLiteral::IsGeneratorFlag generator = is_generator
+      ? FunctionLiteral::kIsGenerator
+      : FunctionLiteral::kNotGenerator;
   AstProperties ast_properties;
   // Parse function body.
-  { FunctionState function_state(this, scope, isolate());
+  { FunctionState function_state(this, scope, is_generator, isolate());
     top_scope_->SetScopeName(function_name);
 
     //  FormalParameterList ::
     //    '(' (Identifier)*[','] ')'
     Expect(Token::LPAREN, CHECK_OK);
     scope->set_start_position(scanner().location().beg_pos);
     Scanner::Location name_loc = Scanner::Location::invalid();
     Scanner::Location dupe_loc = Scanner::Location::invalid();
     Scanner::Location reserved_loc = Scanner::Location::invalid();
 
@@ skipping 217 matching lines @@
                                     body,
                                     materialized_literal_count,
                                     expected_property_count,
                                     handler_count,
                                     only_simple_this_property_assignments,
                                     this_property_assignments,
                                     num_parameters,
                                     duplicate_parameters,
                                     type,
                                     FunctionLiteral::kIsFunction,
-                                    parenthesized);
+                                    parenthesized,
+                                    generator);
   function_literal->set_function_token_position(function_token_position);
   function_literal->set_ast_properties(&ast_properties);
 
   if (fni_ != NULL && should_infer_name) fni_->AddFunction(function_literal);
   return function_literal;
 }
 
 
 preparser::PreParser::PreParseResult Parser::LazyParseFunctionLiteral(
     SingletonLogger* logger) {
   HistogramTimerScope preparse_scope(isolate()->counters()->pre_parse());
   ASSERT_EQ(Token::LBRACE, scanner().current_token());
 
   if (reusable_preparser_ == NULL) {
     intptr_t stack_limit = isolate()->stack_guard()->real_climit();
     bool do_allow_lazy = true;
     reusable_preparser_ = new preparser::PreParser(&scanner_,
                                                    NULL,
                                                    stack_limit,
                                                    do_allow_lazy,
                                                    allow_natives_syntax_,
-                                                   allow_modules_);
+                                                   allow_modules_,
+                                                   FLAG_harmony_generators);
   }
   preparser::PreParser::PreParseResult result =
       reusable_preparser_->PreParseLazyFunction(top_scope_->language_mode(),
+                                                is_generator(),
                                                 logger);
   return result;
 }
 
 
 Expression* Parser::ParseV8Intrinsic(bool* ok) {
   // CallRuntime ::
   //   '%' Identifier Arguments
 
   Expect(Token::MOD, CHECK_OK);
@@ skipping 42 matching lines @@
 
   // We have a valid intrinsics call or a call to a builtin.
   return factory()->NewCallRuntime(name, function, args);
 }
 
 
 bool Parser::peek_any_identifier() {
   Token::Value next = peek();
   return next == Token::IDENTIFIER ||
          next == Token::FUTURE_RESERVED_WORD ||
-         next == Token::FUTURE_STRICT_RESERVED_WORD;
+         next == Token::FUTURE_STRICT_RESERVED_WORD ||
+         next == Token::YIELD;
 }
 
 
 void Parser::Consume(Token::Value token) {
   Token::Value next = Next();
   USE(next);
   USE(token);
   ASSERT(next == token);
 }
 
@@ skipping 51 matching lines @@
 
 
 Literal* Parser::GetLiteralTheHole() {
   return factory()->NewLiteral(isolate()->factory()->the_hole_value());
 }
 
 
 // Parses an identifier that is valid for the current scope, in particular it
 // fails on strict mode future reserved keywords in a strict scope.
 Handle<String> Parser::ParseIdentifier(bool* ok) {
-  if (!top_scope_->is_classic_mode()) {
-    Expect(Token::IDENTIFIER, ok);
-  } else if (!Check(Token::IDENTIFIER)) {
-    Expect(Token::FUTURE_STRICT_RESERVED_WORD, ok);
+  Token::Value next = Next();
+  if (next == Token::IDENTIFIER ||
+      (top_scope_->is_classic_mode() &&
+       (next == Token::FUTURE_STRICT_RESERVED_WORD ||
+        (next == Token::YIELD && !is_generator())))) {
+    return GetSymbol(ok);
+  } else {
+    ReportUnexpectedToken(next);
+    *ok = false;
+    return Handle<String>();
   }
-  if (!*ok) return Handle<String>();
-  return GetSymbol(ok);
 }
 
 
 // Parses and identifier or a strict mode future reserved word, and indicate
 // whether it is strict mode future reserved.
 Handle<String> Parser::ParseIdentifierOrStrictReservedWord(
     bool* is_strict_reserved, bool* ok) {
-  *is_strict_reserved = false;
-  if (!Check(Token::IDENTIFIER)) {
-    Expect(Token::FUTURE_STRICT_RESERVED_WORD, ok);
+  Token::Value next = Next();
+  if (next == Token::IDENTIFIER) {
+    *is_strict_reserved = false;
+  } else if (next == Token::FUTURE_STRICT_RESERVED_WORD ||
+             (next == Token::YIELD && !is_generator())) {
     *is_strict_reserved = true;
+  } else {
+    ReportUnexpectedToken(next);
+    *ok = false;
+    return Handle<String>();
   }
-  if (!*ok) return Handle<String>();
   return GetSymbol(ok);
 }
 
 
 Handle<String> Parser::ParseIdentifierName(bool* ok) {
   Token::Value next = Next();
   if (next != Token::IDENTIFIER &&
       next != Token::FUTURE_RESERVED_WORD &&
       next != Token::FUTURE_STRICT_RESERVED_WORD &&
       !Token::IsKeyword(next)) {
@@ skipping 1091 matching lines @@
 }
 
 
 ScriptDataImpl* ParserApi::PreParse(Utf16CharacterStream* source,
                                     v8::Extension* extension,
                                     int flags) {
   Handle<Script> no_script;
   if (FLAG_lazy && (extension == NULL)) {
     flags |= kAllowLazy;
   }
+  if (FLAG_harmony_generators) {
+    flags |= kAllowGenerators;
+  }
   CompleteParserRecorder recorder;
   return DoPreParse(source, flags, &recorder);
 }
 
 
 bool RegExpParser::ParseRegExp(FlatStringReader* input,
                                bool multiline,
                                RegExpCompileData* result,
                                Zone* zone) {
   ASSERT(result != NULL);
@@ skipping 54 matching lines @@
       ASSERT(info->isolate()->has_pending_exception());
     } else {
       result = parser.ParseProgram();
     }
   }
   info->SetFunction(result);
   return (result != NULL);
 }
 
 } }  // namespace v8::internal