Add parser support for generators.

src/preparser.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 35 matching lines @@
 
 #ifdef _MSC_VER
 // Usually defined in math.h, but not in MSVC.
 // Abstracted to work
 int isfinite(double value);
 #endif
 
 namespace preparser {
 
 PreParser::PreParseResult PreParser::PreParseLazyFunction(
-    i::LanguageMode mode, i::ParserRecorder* log) {
+    i::LanguageMode mode, bool is_generator, i::ParserRecorder* log) {
   log_ = log;
   // Lazy functions always have trivial outer scopes (no with/catch scopes).
   Scope top_scope(&scope_, kTopLevelScope);
   set_language_mode(mode);
   Scope function_scope(&scope_, kFunctionScope);
+  function_scope.set_is_generator(is_generator);
   ASSERT_EQ(i::Token::LBRACE, scanner_->current_token());
   bool ok = true;
   int start_position = scanner_->peek_location().beg_pos;
   ParseLazyFunctionLiteralBody(&ok);
   if (stack_overflow_) return kPreParseStackOverflow;
   if (!ok) {
     ReportUnexpectedToken(scanner_->current_token());
   } else {
     ASSERT_EQ(i::Token::RBRACE, scanner_->peek());
     if (!is_classic_mode()) {
@@ skipping 75 matching lines @@
 PreParser::Statement PreParser::ParseSourceElement(bool* ok) {
   // (Ecma 262 5th Edition, clause 14):
   // SourceElement:
   //    Statement
   //    FunctionDeclaration
   //
   // In harmony mode we allow additionally the following productions
   // SourceElement:
   //    LetDeclaration
   //    ConstDeclaration
+  //    GeneratorDeclaration
 
   switch (peek()) {
     case i::Token::FUNCTION:
       return ParseFunctionDeclaration(ok);
     case i::Token::LET:
     case i::Token::CONST:
       return ParseVariableStatement(kSourceElement, ok);
     default:
       return ParseStatement(ok);
   }
@@ skipping 120 matching lines @@
 
     default:
       return ParseExpressionOrLabelledStatement(ok);
   }
 }
 
 
 PreParser::Statement PreParser::ParseFunctionDeclaration(bool* ok) {
   // FunctionDeclaration ::
   //   'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}'
+  // GeneratorDeclaration ::
+  //   'function' '*' Identifier '(' FormalParameterListopt ')'
+  //      '{' FunctionBody '}'
   Expect(i::Token::FUNCTION, CHECK_OK);
 
+  bool is_generator = allow_generators_ && Check(i::Token::MUL);
   Identifier identifier = ParseIdentifier(CHECK_OK);
   i::Scanner::Location location = scanner_->location();
 
-  Expression function_value = ParseFunctionLiteral(CHECK_OK);
+  Expression function_value = ParseFunctionLiteral(is_generator, CHECK_OK);
 
   if (function_value.IsStrictFunction() &&
       !identifier.IsValidStrictVariable()) {
     // Strict mode violation, using either reserved word or eval/arguments
     // as name of strict function.
     const char* type = "strict_function_name";
-    if (identifier.IsFutureStrictReserved()) {
+    if (identifier.IsFutureStrictReserved() || identifier.IsYield()) {
       type = "strict_reserved_word";
     }
     ReportMessageAt(location, type, NULL);
     *ok = false;
   }
   return Statement::FunctionDeclaration();
 }
 
 
 PreParser::Statement PreParser::ParseBlock(bool* ok) {
@@ skipping 148 matching lines @@
 
 
 PreParser::Statement PreParser::ParseExpressionOrLabelledStatement(bool* ok) {
   // ExpressionStatement | LabelledStatement ::
   //   Expression ';'
   //   Identifier ':' Statement
 
   Expression expr = ParseExpression(true, CHECK_OK);
   if (expr.IsRawIdentifier()) {
     ASSERT(!expr.AsIdentifier().IsFutureReserved());
-    ASSERT(is_classic_mode() || !expr.AsIdentifier().IsFutureStrictReserved());
+    ASSERT(is_classic_mode() ||
+           (!expr.AsIdentifier().IsFutureStrictReserved() &&
+            !expr.AsIdentifier().IsYield()));
     if (peek() == i::Token::COLON) {
       Consume(i::Token::COLON);
       return ParseStatement(ok);
     }
     // Preparsing is disabled for extensions (because the extension details
     // aren't passed to lazily compiled functions), so we don't
     // accept "native function" in the preparser.
   }
   // Parsed expression statement.
   ExpectSemicolon(CHECK_OK);
@@ skipping 314 matching lines @@
   }
   return result;
 }
 
 
 // Precedence = 2
 PreParser::Expression PreParser::ParseAssignmentExpression(bool accept_IN,
                                                            bool* ok) {
   // AssignmentExpression ::
   //   ConditionalExpression
+  //   YieldExpression
   //   LeftHandSideExpression AssignmentOperator AssignmentExpression
 
+  if (scope_->is_generator() && peek() == i::Token::YIELD) {
+    return ParseYieldExpression(ok);
+  }
+
   i::Scanner::Location before = scanner_->peek_location();
   Expression expression = ParseConditionalExpression(accept_IN, CHECK_OK);
 
   if (!i::Token::IsAssignmentOp(peek())) {
     // Parsed conditional expression only (no assignment).
     return expression;
   }
 
   if (!is_classic_mode() &&
       expression.IsIdentifier() &&
@@ skipping 10 matching lines @@
 
   if ((op == i::Token::ASSIGN) && expression.IsThisProperty()) {
     scope_->AddProperty();
   }
 
   return Expression::Default();
 }
 
 
 // Precedence = 3
+PreParser::Expression PreParser::ParseYieldExpression(bool* ok) {
+  // YieldExpression ::
+  //   'yield' '*'? AssignmentExpression
+  Consume(i::Token::YIELD);
+  Check(i::Token::MUL);
+
+  ParseAssignmentExpression(false, CHECK_OK);
+
+  return Expression::Default();
+}
+
+
+// Precedence = 3
 PreParser::Expression PreParser::ParseConditionalExpression(bool accept_IN,
                                                             bool* ok) {
   // ConditionalExpression ::
   //   LogicalOrExpression
   //   LogicalOrExpression '?' AssignmentExpression ':' AssignmentExpression
 
   // We start using the binary expression parser for prec >= 4 only!
   Expression expression = ParseBinaryExpression(4, accept_IN, CHECK_OK);
   if (peek() != i::Token::CONDITIONAL) return expression;
   Consume(i::Token::CONDITIONAL);
@@ skipping 172 matching lines @@
 PreParser::Expression PreParser::ParseMemberWithNewPrefixesExpression(
     unsigned new_count, bool* ok) {
   // MemberExpression ::
   //   (PrimaryExpression | FunctionLiteral)
   //     ('[' Expression ']' | '.' Identifier | Arguments)*
 
   // Parse the initial primary or function expression.
   Expression result = Expression::Default();
   if (peek() == i::Token::FUNCTION) {
     Consume(i::Token::FUNCTION);
+
+    bool is_generator = allow_generators_ && Check(i::Token::MUL);
     Identifier identifier = Identifier::Default();
     if (peek_any_identifier()) {
       identifier = ParseIdentifier(CHECK_OK);
     }
-    result = ParseFunctionLiteral(CHECK_OK);
+    result = ParseFunctionLiteral(is_generator, CHECK_OK);
     if (result.IsStrictFunction() && !identifier.IsValidStrictVariable()) {
       StrictModeIdentifierViolation(scanner_->location(),
                                     "strict_function_name",
                                     identifier,
                                     ok);
       return Expression::Default();
     }
   } else {
     result = ParsePrimaryExpression(CHECK_OK);
   }
@@ skipping 53 matching lines @@
   Expression result = Expression::Default();
   switch (peek()) {
     case i::Token::THIS: {
       Next();
       result = Expression::This();
       break;
     }
 
     case i::Token::FUTURE_RESERVED_WORD:
     case i::Token::FUTURE_STRICT_RESERVED_WORD:
+    case i::Token::YIELD:
     case i::Token::IDENTIFIER: {
       Identifier id = ParseIdentifier(CHECK_OK);
       result = Expression::FromIdentifier(id);
       break;
     }
 
     case i::Token::NULL_LITERAL:
     case i::Token::TRUE_LITERAL:
     case i::Token::FALSE_LITERAL:
     case i::Token::NUMBER: {
@@ skipping 125 matching lines @@
                 name != i::Token::STRING &&
                 !is_keyword) {
               *ok = false;
               return Expression::Default();
             }
             if (!is_keyword) {
               LogSymbol();
             }
             PropertyType type = is_getter ? kGetterProperty : kSetterProperty;
             CheckDuplicate(&duplicate_finder, name, type, CHECK_OK);
-            ParseFunctionLiteral(CHECK_OK);
+            ParseFunctionLiteral(false, CHECK_OK);
             if (peek() != i::Token::RBRACE) {
               Expect(i::Token::COMMA, CHECK_OK);
             }
             continue;  // restart the while
         }
         CheckDuplicate(&duplicate_finder, next, kValueProperty, CHECK_OK);
         break;
       }
       case i::Token::STRING:
         Consume(next);
@@ skipping 66 matching lines @@
     if (!done) {
       Expect(i::Token::COMMA, ok);
       if (!*ok) return -1;
     }
   }
   Expect(i::Token::RPAREN, ok);
   return argc;
 }
 
 
-PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) {
+PreParser::Expression PreParser::ParseFunctionLiteral(bool is_generator,
+                                                      bool* ok) {
   // Function ::
   //   '(' FormalParameterList? ')' '{' FunctionBody '}'
 
   // Parse function body.
   ScopeType outer_scope_type = scope_->type();
   bool inside_with = scope_->IsInsideWith();
   Scope function_scope(&scope_, kFunctionScope);
+  function_scope.set_is_generator(is_generator);
   //  FormalParameterList ::
   //    '(' (Identifier)*[','] ')'
   Expect(i::Token::LPAREN, CHECK_OK);
   int start_position = scanner_->location().beg_pos;
   bool done = (peek() == i::Token::RPAREN);
   DuplicateFinder duplicate_finder(scanner_->unicode_cache());
   while (!done) {
     Identifier id = ParseIdentifier(CHECK_OK);
     if (!id.IsValidStrictVariable()) {
       StrictModeIdentifierViolation(scanner_->location(),
@@ skipping 125 matching lines @@
 }
 
 
 PreParser::Identifier PreParser::GetIdentifierSymbol() {
   LogSymbol();
   if (scanner_->current_token() == i::Token::FUTURE_RESERVED_WORD) {
     return Identifier::FutureReserved();
   } else if (scanner_->current_token() ==
              i::Token::FUTURE_STRICT_RESERVED_WORD) {
     return Identifier::FutureStrictReserved();
+  } else if (scanner_->current_token() == i::Token::YIELD) {
+    return Identifier::Yield();
   }
   if (scanner_->is_literal_ascii()) {
     // Detect strict-mode poison words.
     if (scanner_->literal_length() == 4 &&
         !strncmp(scanner_->literal_ascii_string().start(), "eval", 4)) {
       return Identifier::Eval();
     }
     if (scanner_->literal_length() == 9 &&
         !strncmp(scanner_->literal_ascii_string().start(), "arguments", 9)) {
       return Identifier::Arguments();
     }
   }
   return Identifier::Default();
 }
 
 
 PreParser::Identifier PreParser::ParseIdentifier(bool* ok) {
   i::Token::Value next = Next();
   switch (next) {
     case i::Token::FUTURE_RESERVED_WORD: {
       i::Scanner::Location location = scanner_->location();
       ReportMessageAt(location.beg_pos, location.end_pos,
                       "reserved_word", NULL);
       *ok = false;
       return GetIdentifierSymbol();
     }
+    case i::Token::YIELD:
+      if (scope_->is_generator()) {
+        // 'yield' in a generator is only valid as part of a YieldExpression.
+        ReportMessageAt(scanner_->location(), "unexpected_token", "yield");
+        *ok = false;
+        return Identifier::Yield();
+      }
+      // FALLTHROUGH
     case i::Token::FUTURE_STRICT_RESERVED_WORD:
       if (!is_classic_mode()) {
         i::Scanner::Location location = scanner_->location();
         ReportMessageAt(location.beg_pos, location.end_pos,
                         "strict_reserved_word", NULL);
         *ok = false;
       }
       // FALLTHROUGH
     case i::Token::IDENTIFIER:
       return GetIdentifierSymbol();
@@ skipping 37 matching lines @@
 }
 
 
 void PreParser::StrictModeIdentifierViolation(i::Scanner::Location location,
                                               const char* eval_args_type,
                                               Identifier identifier,
                                               bool* ok) {
   const char* type = eval_args_type;
   if (identifier.IsFutureReserved()) {
     type = "reserved_word";
-  } else if (identifier.IsFutureStrictReserved()) {
+  } else if (identifier.IsFutureStrictReserved() || identifier.IsYield()) {
     type = "strict_reserved_word";
   }
   if (!is_classic_mode()) {
     ReportMessageAt(location, type, NULL);
     *ok = false;
     return;
   }
   strict_mode_violation_location_ = location;
   strict_mode_violation_type_ = type;
 }
@@ skipping 33 matching lines @@
     *is_get = strncmp(token, "get", 3) == 0;
     *is_set = !*is_get && strncmp(token, "set", 3) == 0;
   }
   return result;
 }
 
 bool PreParser::peek_any_identifier() {
   i::Token::Value next = peek();
   return next == i::Token::IDENTIFIER ||
          next == i::Token::FUTURE_RESERVED_WORD ||
-         next == i::Token::FUTURE_STRICT_RESERVED_WORD;
+         next == i::Token::FUTURE_STRICT_RESERVED_WORD ||
+         next == i::Token::YIELD;
 }
 
 
 int DuplicateFinder::AddAsciiSymbol(i::Vector<const char> key, int value) {
   return AddSymbol(i::Vector<const byte>::cast(key), true, value);
 }
 
 int DuplicateFinder::AddUtf16Symbol(i::Vector<const uint16_t> key, int value) {
   return AddSymbol(i::Vector<const byte>::cast(key), false, value);
 }
@@ skipping 116 matching lines @@
       backing_store_.Add(static_cast<byte>((ascii_length >> 14) | 0x80u));
     }
     backing_store_.Add(static_cast<byte>((ascii_length >> 7) | 0x80u));
   }
   backing_store_.Add(static_cast<byte>(ascii_length & 0x7f));
 
   backing_store_.AddBlock(bytes);
   return backing_store_.EndSequence().start();
 }
 } }  // v8::preparser