Add parser support for generators.

src/preparser.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1101 matching lines @@
 
     case i::Token::FUTURE_RESERVED_WORD: {
       Next();
       i::Scanner::Location location = scanner_->location();
       ReportMessageAt(location.beg_pos, location.end_pos,
                       "reserved_word", NULL);
       *ok = false;
       return Expression::Default();
     }
 
+      // FIXME!!!!
+    case i::Token::YIELD:

[wingo, 2013/03/11 12:10:53]

This FIXME is to mark that this case and the FUTURE_RESERVED_WORD case and the
FUTURE_STRICT_RESERVED_WORD case can all be elided, it seems, as ParseIdentifier
(called later) will DTRT.  Am preparing a patch for that.  Apologies for letting
a FIXME through :)

+      if (scope_->is_generator()) {
+        // 'yield' in a generator is only valid as part of a YieldExpression.
+        i::Scanner::Location location = scanner_->location();
+        ReportMessageAt(location, "unexpected_token", NULL);
+        *ok = false;
+        return Expression::Default();
+      }
+      // FALLTHROUGH
     case i::Token::FUTURE_STRICT_RESERVED_WORD:
       if (!is_classic_mode()) {
         Next();
         i::Scanner::Location location = scanner_->location();
         ReportMessageAt(location, "strict_reserved_word", NULL);
         *ok = false;
         return Expression::Default();
       }
       // FALLTHROUGH
     case i::Token::IDENTIFIER: {
@@ skipping 374 matching lines @@
 }
 
 
 PreParser::Identifier PreParser::GetIdentifierSymbol() {
   LogSymbol();
   if (scanner_->current_token() == i::Token::FUTURE_RESERVED_WORD) {
     return Identifier::FutureReserved();
   } else if (scanner_->current_token() ==
              i::Token::FUTURE_STRICT_RESERVED_WORD) {
     return Identifier::FutureStrictReserved();
+  } else if (scanner_->current_token() == i::Token::YIELD) {
+    return Identifier::Yield();
   }
   if (scanner_->is_literal_ascii()) {
     // Detect strict-mode poison words.
     if (scanner_->literal_length() == 4 &&
         !strncmp(scanner_->literal_ascii_string().start(), "eval", 4)) {
       return Identifier::Eval();
     }
     if (scanner_->literal_length() == 9 &&
         !strncmp(scanner_->literal_ascii_string().start(), "arguments", 9)) {
       return Identifier::Arguments();
     }
   }
   return Identifier::Default();
 }
 
 
 PreParser::Identifier PreParser::ParseIdentifier(bool* ok) {
   i::Token::Value next = Next();
   switch (next) {
     case i::Token::FUTURE_RESERVED_WORD: {
       i::Scanner::Location location = scanner_->location();
       ReportMessageAt(location.beg_pos, location.end_pos,
                       "reserved_word", NULL);
       *ok = false;
       return GetIdentifierSymbol();
     }
+    case i::Token::YIELD:
+      if (scope_->is_generator()) {
+        // 'yield' in a generator is only valid as part of a YieldExpression.
+        i::Scanner::Location location = scanner_->location();
+        ReportMessageAt(location.beg_pos, location.end_pos,
+                        "unexpected_token", NULL);
+        *ok = false;
+        return Identifier::Default();
+      }
+      // FALLTHROUGH
     case i::Token::FUTURE_STRICT_RESERVED_WORD:
       if (!is_classic_mode()) {
         i::Scanner::Location location = scanner_->location();
         ReportMessageAt(location.beg_pos, location.end_pos,
                         "strict_reserved_word", NULL);
         *ok = false;
       }
       // FALLTHROUGH
     case i::Token::IDENTIFIER:
       return GetIdentifierSymbol();
@@ skipping 37 matching lines @@
 }
 
 
 void PreParser::StrictModeIdentifierViolation(i::Scanner::Location location,
                                               const char* eval_args_type,
                                               Identifier identifier,
                                               bool* ok) {
   const char* type = eval_args_type;
   if (identifier.IsFutureReserved()) {
     type = "reserved_word";
-  } else if (identifier.IsFutureStrictReserved()) {
+  } else if (identifier.IsFutureStrictReserved() || identifier.IsYield()) {
     type = "strict_reserved_word";
   }
   if (!is_classic_mode()) {
     ReportMessageAt(location, type, NULL);
     *ok = false;
     return;
   }
   strict_mode_violation_location_ = location;
   strict_mode_violation_type_ = type;
 }
@@ skipping 170 matching lines @@
       backing_store_.Add(static_cast<byte>((ascii_length >> 14) | 0x80u));
     }
     backing_store_.Add(static_cast<byte>((ascii_length >> 7) | 0x80u));
   }
   backing_store_.Add(static_cast<byte>(ascii_length & 0x7f));
 
   backing_store_.AddBlock(bytes);
   return backing_store_.EndSequence().start();
 }
 } }  // v8::preparser