Change scanner buffers to not use utf-8.

src/scanner-base.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 17 matching lines @@
 // Features shared by parsing and pre-parsing scanners.
 
 #include "../include/v8stdint.h"
 #include "scanner-base.h"
 #include "char-predicates-inl.h"
 
 namespace v8 {
 namespace internal {
 
 // ----------------------------------------------------------------------------
-// LiteralCollector
-
-LiteralCollector::LiteralCollector()
-    : buffer_(kInitialCapacity), recording_(false) { }
-
-
-LiteralCollector::~LiteralCollector() {}
-
-
-void LiteralCollector::AddCharSlow(uc32 c) {
-  ASSERT(static_cast<unsigned>(c) > unibrow::Utf8::kMaxOneByteChar);
-  int length = unibrow::Utf8::Length(c);
-  Vector<char> block = buffer_.AddBlock(length, '\0');
-#ifdef DEBUG
-  int written_length = unibrow::Utf8::Encode(block.start(), c);
-  CHECK_EQ(length, written_length);
-#else
-  unibrow::Utf8::Encode(block.start(), c);
-#endif
-}
-
-// ----------------------------------------------------------------------------
 // Character predicates
 
 unibrow::Predicate<IdentifierStart, 128> ScannerConstants::kIsIdentifierStart;
 unibrow::Predicate<IdentifierPart, 128> ScannerConstants::kIsIdentifierPart;
 unibrow::Predicate<unibrow::WhiteSpace, 128> ScannerConstants::kIsWhiteSpace;
 unibrow::Predicate<unibrow::LineTerminator, 128>
   ScannerConstants::kIsLineTerminator;
 
 StaticResource<ScannerConstants::Utf8Decoder> ScannerConstants::utf8_decoder_;
 
@@ skipping 179 matching lines @@
     if (c0_ == '-') return SkipSingleLineComment();
     PushBack('-');  // undo Advance()
   }
   PushBack('!');  // undo Advance()
   ASSERT(c0_ == '!');
   return Token::LT;
 }
 
 
 void JavaScriptScanner::Scan() {
-  next_.literal_chars = Vector<const char>();
+  next_.literal_chars = NULL;
   Token::Value token;
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
 
     switch (c0_) {
       case ' ':
       case '\t':
         Advance();
         token = Token::WHITESPACE;
@@ skipping 284 matching lines @@
   // should be illegal, but they are commonly handled
   // as non-escaped characters by JS VMs.
   AddLiteralChar(c);
 }
 
 
 Token::Value JavaScriptScanner::ScanString() {
   uc32 quote = c0_;
   Advance();  // consume quote
 
-  LiteralScope literal(this, kLiteralString);
+  LiteralScope literal(this);
   while (c0_ != quote && c0_ >= 0
          && !ScannerConstants::kIsLineTerminator.get(c0_)) {
     uc32 c = c0_;
     Advance();
     if (c == '\\') {
       if (c0_ < 0) return Token::ILLEGAL;
       ScanEscape();
     } else {
       AddLiteralChar(c);
     }
@@ skipping 10 matching lines @@
   while (IsDecimalDigit(c0_))
     AddLiteralCharAdvance();
 }
 
 
 Token::Value JavaScriptScanner::ScanNumber(bool seen_period) {
   ASSERT(IsDecimalDigit(c0_));  // the first digit of the number or the fraction
 
   enum { DECIMAL, HEX, OCTAL } kind = DECIMAL;
 
-  LiteralScope literal(this, kLiteralNumber);
+  LiteralScope literal(this);
   if (seen_period) {
     // we have already seen a decimal point of the float
     AddLiteralChar('.');
     ScanDecimalDigits();  // we know we have at least one digit
 
   } else {
     // if the first character is '0' we must check for octals and hex
     if (c0_ == '0') {
       AddLiteralCharAdvance();
 
@@ skipping 68 matching lines @@
   uc32 c = ScanHexEscape('u', 4);
   // We do not allow a unicode escape sequence to start another
   // unicode escape sequence.
   if (c == '\\') return unibrow::Utf8::kBadChar;
   return c;
 }
 
 
 Token::Value JavaScriptScanner::ScanIdentifierOrKeyword() {
   ASSERT(ScannerConstants::kIsIdentifierStart.get(c0_));
-  LiteralScope literal(this, kLiteralIdentifier);
+  LiteralScope literal(this);
   KeywordMatcher keyword_match;
   // Scan identifier start character.
   if (c0_ == '\\') {
     uc32 c = ScanIdentifierUnicodeEscape();
     // Only allow legal identifier start characters.
     if (!ScannerConstants::kIsIdentifierStart.get(c)) return Token::ILLEGAL;
     AddLiteralChar(c);
     return ScanIdentifierSuffix(&literal);
   }
 
@@ skipping 45 matching lines @@
   bool in_character_class = false;
 
   // Previous token is either '/' or '/=', in the second case, the
   // pattern starts at =.
   next_.location.beg_pos = source_pos() - (seen_equal ? 2 : 1);
   next_.location.end_pos = source_pos() - (seen_equal ? 1 : 0);
 
   // Scan regular expression body: According to ECMA-262, 3rd, 7.8.5,
   // the scanner should pass uninterpreted bodies to the RegExp
   // constructor.
-  LiteralScope literal(this, kLiteralRegExp);
+  LiteralScope literal(this);
   if (seen_equal)
     AddLiteralChar('=');
 
   while (c0_ != '/' || in_character_class) {
     if (ScannerConstants::kIsLineTerminator.get(c0_) || c0_ < 0) return false;
     if (c0_ == '\\') {  // escaped character
       AddLiteralCharAdvance();
       if (ScannerConstants::kIsLineTerminator.get(c0_) || c0_ < 0) return false;
       AddLiteralCharAdvance();
     } else {  // unescaped character
       if (c0_ == '[') in_character_class = true;
       if (c0_ == ']') in_character_class = false;
       AddLiteralCharAdvance();
     }
   }
   Advance();  // consume '/'
 
   literal.Complete();
 
   return true;
 }
 
 
 bool JavaScriptScanner::ScanRegExpFlags() {
   // Scan regular expression flags.
-  LiteralScope literal(this, kLiteralRegExpFlags);
+  LiteralScope literal(this);
   while (ScannerConstants::kIsIdentifierPart.get(c0_)) {
     if (c0_ == '\\') {
       uc32 c = ScanIdentifierUnicodeEscape();
       if (c != static_cast<uc32>(unibrow::Utf8::kBadChar)) {
         // We allow any escaped character, unlike the restriction on
         // IdentifierPart when it is used to build an IdentifierName.
         AddLiteralChar(c);
         continue;
       }
     }
@@ skipping 129 matching lines @@
       if (MatchKeywordStart(input, "with", 1, Token::WITH)) return;
       break;
     case UNMATCHABLE:
       break;
   }
   // On fallthrough, it's a failure.
   state_ = UNMATCHABLE;
 }
 
 } }  // namespace v8::internal