Merge up to 8597 to experimental/gc from the bleeding edge.

src/scanner-base.cc

 // Copyright 2011 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 59 matching lines @@
 
 
 
 // ----------------------------------------------------------------------------
 // JavaScriptScanner
 
 JavaScriptScanner::JavaScriptScanner(UnicodeCache* scanner_contants)
   : Scanner(scanner_contants), octal_pos_(Location::invalid()) { }
 
 
+void JavaScriptScanner::Initialize(UC16CharacterStream* source) {
+  source_ = source;
+  // Need to capture identifiers in order to recognize "get" and "set"
+  // in object literals.
+  Init();
+  // Skip initial whitespace allowing HTML comment ends just like
+  // after a newline and scan first token.
+  has_line_terminator_before_next_ = true;
+  SkipWhiteSpace();
+  Scan();
+}
+
 Token::Value JavaScriptScanner::Next() {
   current_ = next_;
   has_line_terminator_before_next_ = false;
+  has_multiline_comment_before_next_ = false;
   Scan();
   return current_.token;
 }
 
 
 static inline bool IsByteOrderMark(uc32 c) {
   // The Unicode value U+FFFE is guaranteed never to be assigned as a
   // Unicode character; this implies that in a Unicode context the
   // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF
   // character expressed in little-endian byte order (since it could
@@ skipping 44 matching lines @@
 }
 
 
 Token::Value JavaScriptScanner::SkipSingleLineComment() {
   Advance();
 
   // The line terminator at the end of the line is not considered
   // to be part of the single-line comment; it is recognized
   // separately by the lexical grammar and becomes part of the
   // stream of input elements for the syntactic grammar (see
-  // ECMA-262, section 7.4, page 12).
+  // ECMA-262, section 7.4).
   while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) {
     Advance();
   }
 
   return Token::WHITESPACE;
 }
 
 
 Token::Value JavaScriptScanner::SkipMultiLineComment() {
   ASSERT(c0_ == '*');
   Advance();
 
   while (c0_ >= 0) {
     char ch = c0_;
     Advance();
+    if (unicode_cache_->IsLineTerminator(ch)) {
+      // Following ECMA-262, section 7.4, a comment containing
+      // a newline will make the comment count as a line-terminator.
+      has_multiline_comment_before_next_ = true;
+    }
     // If we have reached the end of the multi-line comment, we
     // consume the '/' and insert a whitespace. This way all
-    // multi-line comments are treated as whitespace - even the ones
-    // containing line terminators. This contradicts ECMA-262, section
-    // 7.4, page 12, that says that multi-line comments containing
-    // line terminators should be treated as a line terminator, but it
-    // matches the behaviour of SpiderMonkey and KJS.
+    // multi-line comments are treated as whitespace.
     if (ch == '*' && c0_ == '/') {
       c0_ = ' ';
       return Token::WHITESPACE;
     }
   }
 
   // Unterminated multi-line comment.
   return Token::ILLEGAL;
 }
 
@@ skipping 261 matching lines @@
   ASSERT_EQ(next_.location.end_pos, current_pos);
   // Positions inside the lookahead token aren't supported.
   ASSERT(pos >= current_pos);
   if (pos != current_pos) {
     source_->SeekForward(pos - source_->pos());
     Advance();
     // This function is only called to seek to the location
     // of the end of a function (at the "}" token). It doesn't matter
     // whether there was a line terminator in the part we skip.
     has_line_terminator_before_next_ = false;
+    has_multiline_comment_before_next_ = false;
   }
   Scan();
 }
 
 
 void JavaScriptScanner::ScanEscape() {
   uc32 c = c0_;
   Advance();
 
   // Skip escaped newlines.
@@ skipping 316 matching lines @@
   { "break",  KEYWORD_PREFIX, Token::BREAK },
   { NULL,     C,              Token::ILLEGAL },
   { NULL,     D,              Token::ILLEGAL },
   { NULL,     E,              Token::ILLEGAL },
   { NULL,     F,              Token::ILLEGAL },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
   { NULL,     I,              Token::ILLEGAL },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
-  { "let",    KEYWORD_PREFIX, Token::FUTURE_RESERVED_WORD },
+  { "let",    KEYWORD_PREFIX, Token::FUTURE_STRICT_RESERVED_WORD },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
   { NULL,     N,              Token::ILLEGAL },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
   { NULL,     P,              Token::ILLEGAL },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
   { "return", KEYWORD_PREFIX, Token::RETURN },
   { NULL,     S,              Token::ILLEGAL },
   { NULL,     T,              Token::ILLEGAL },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
   { NULL,     V,              Token::ILLEGAL },
   { NULL,     W,              Token::ILLEGAL },
   { NULL,     UNMATCHABLE,    Token::ILLEGAL },
-  { "yield",  KEYWORD_PREFIX, Token::FUTURE_RESERVED_WORD }
+  { "yield",  KEYWORD_PREFIX, Token::FUTURE_STRICT_RESERVED_WORD }
 };
 
 
 void KeywordMatcher::Step(unibrow::uchar input) {
   switch (state_) {
     case INITIAL: {
       // matching the first character is the only state with significant fanout.
       // Match only lower-case letters in range 'b'..'y'.
       unsigned int offset = input - kFirstCharRangeMin;
       if (offset < kFirstCharRangeLength) {
@@ skipping 16 matching lines @@
         }
         return;
       }
       break;
     case KEYWORD_MATCHED:
       token_ = Token::IDENTIFIER;
       break;
     case C:
       if (MatchState(input, 'a', CA)) return;
       if (MatchKeywordStart(input, "class", 1,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_RESERVED_WORD)) return;
       if (MatchState(input, 'o', CO)) return;
       break;
     case CA:
       if (MatchKeywordStart(input, "case", 2, Token::CASE)) return;
       if (MatchKeywordStart(input, "catch", 2, Token::CATCH)) return;
       break;
     case CO:
       if (MatchState(input, 'n', CON)) return;
       break;
     case CON:
       if (MatchKeywordStart(input, "const", 3, Token::CONST)) return;
       if (MatchKeywordStart(input, "continue", 3, Token::CONTINUE)) return;
       break;
     case D:
       if (MatchState(input, 'e', DE)) return;
       if (MatchKeyword(input, 'o', KEYWORD_MATCHED, Token::DO)) return;
       break;
     case DE:
       if (MatchKeywordStart(input, "debugger", 2, Token::DEBUGGER)) return;
       if (MatchKeywordStart(input, "default", 2, Token::DEFAULT)) return;
       if (MatchKeywordStart(input, "delete", 2, Token::DELETE)) return;
       break;
     case E:
       if (MatchKeywordStart(input, "else", 1, Token::ELSE)) return;
       if (MatchKeywordStart(input, "enum", 1,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_RESERVED_WORD)) return;
       if (MatchState(input, 'x', EX)) return;
       break;
     case EX:
       if (MatchKeywordStart(input, "export", 2,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_RESERVED_WORD)) return;
       if (MatchKeywordStart(input, "extends", 2,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_RESERVED_WORD)) return;
       break;
     case F:
       if (MatchKeywordStart(input, "false", 1, Token::FALSE_LITERAL)) return;
       if (MatchKeywordStart(input, "finally", 1, Token::FINALLY)) return;
       if (MatchKeywordStart(input, "for", 1, Token::FOR)) return;
       if (MatchKeywordStart(input, "function", 1, Token::FUNCTION)) return;
       break;
     case I:
       if (MatchKeyword(input, 'f', KEYWORD_MATCHED, Token::IF)) return;
       if (MatchState(input, 'm', IM)) return;
       if (MatchKeyword(input, 'n', IN, Token::IN)) return;
       break;
     case IM:
       if (MatchState(input, 'p', IMP)) return;
       break;
     case IMP:
       if (MatchKeywordStart(input, "implements", 3,
-         Token::FUTURE_RESERVED_WORD )) return;
+                            Token::FUTURE_STRICT_RESERVED_WORD )) return;
       if (MatchKeywordStart(input, "import", 3,
-         Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_RESERVED_WORD)) return;
       break;
     case IN:
       token_ = Token::IDENTIFIER;
       if (MatchKeywordStart(input, "interface", 2,
-         Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_STRICT_RESERVED_WORD)) return;
       if (MatchKeywordStart(input, "instanceof", 2, Token::INSTANCEOF)) return;
       break;
     case N:
-      if (MatchKeywordStart(input, "native", 1, Token::NATIVE)) return;
       if (MatchKeywordStart(input, "new", 1, Token::NEW)) return;
       if (MatchKeywordStart(input, "null", 1, Token::NULL_LITERAL)) return;
       break;
     case P:
       if (MatchKeywordStart(input, "package", 1,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_STRICT_RESERVED_WORD)) return;
       if (MatchState(input, 'r', PR)) return;
       if (MatchKeywordStart(input, "public", 1,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_STRICT_RESERVED_WORD)) return;
       break;
     case PR:
       if (MatchKeywordStart(input, "private", 2,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_STRICT_RESERVED_WORD)) return;
       if (MatchKeywordStart(input, "protected", 2,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_STRICT_RESERVED_WORD)) return;
       break;
     case S:
       if (MatchKeywordStart(input, "static", 1,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_STRICT_RESERVED_WORD)) return;
       if (MatchKeywordStart(input, "super", 1,
-          Token::FUTURE_RESERVED_WORD)) return;
+                            Token::FUTURE_RESERVED_WORD)) return;
       if (MatchKeywordStart(input, "switch", 1,
-          Token::SWITCH)) return;
+                            Token::SWITCH)) return;
       break;
     case T:
       if (MatchState(input, 'h', TH)) return;
       if (MatchState(input, 'r', TR)) return;
       if (MatchKeywordStart(input, "typeof", 1, Token::TYPEOF)) return;
       break;
     case TH:
       if (MatchKeywordStart(input, "this", 2, Token::THIS)) return;
       if (MatchKeywordStart(input, "throw", 2, Token::THROW)) return;
       break;
@@ skipping 10 matching lines @@
       if (MatchKeywordStart(input, "with", 1, Token::WITH)) return;
       break;
     case UNMATCHABLE:
       break;
   }
   // On fallthrough, it's a failure.
   state_ = UNMATCHABLE;
 }
 
 } }  // namespace v8::internal