Created collector class and used it to collect identifiers during scanning.

src/scanner.cc

 // Copyright 2006-2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 32 matching lines @@
 unibrow::Predicate<unibrow::LineTerminator, 128> Scanner::kIsLineTerminator;
 unibrow::Predicate<unibrow::WhiteSpace, 128> Scanner::kIsWhiteSpace;
 
 
 StaticResource<Scanner::Utf8Decoder> Scanner::utf8_decoder_;
 
 
 // ----------------------------------------------------------------------------
 // UTF8Buffer
 
-UTF8Buffer::UTF8Buffer() : data_(NULL), limit_(NULL) { }
+UTF8Buffer::UTF8Buffer() : buffer_(kInitialCapacity) { }
 
 
-UTF8Buffer::~UTF8Buffer() {
-  if (data_ != NULL) DeleteArray(data_);
+UTF8Buffer::~UTF8Buffer() {}
+
+
+void UTF8Buffer::AddCharSlow(uc32 c) {
+  ASSERT(static_cast<unsigned>(c) > unibrow::Utf8::kMaxOneByteChar);
+  int length = unibrow::Utf8::Length(c);
+  Vector<char> block = buffer_.AddBlock(length, '\0');
+#ifdef DEBUG
+  int written_length = unibrow::Utf8::Encode(block.start(), c);
+  CHECK_EQ(length, written_length);
+#else
+  unibrow::Utf8::Encode(block.start(), c);
+#endif
 }
 
 
-void UTF8Buffer::AddCharSlow(uc32 c) {
-  static const int kCapacityGrowthLimit = 1 * MB;
-  if (cursor_ > limit_) {
-    int old_capacity = Capacity();
-    int old_position = pos();
-    int new_capacity =
-        Min(old_capacity * 3, old_capacity + kCapacityGrowthLimit);
-    char* new_data = NewArray<char>(new_capacity);
-    memcpy(new_data, data_, old_position);
-    DeleteArray(data_);
-    data_ = new_data;
-    cursor_ = new_data + old_position;
-    limit_ = ComputeLimit(new_data, new_capacity);
-    ASSERT(Capacity() == new_capacity && pos() == old_position);
-  }
-  if (static_cast<unsigned>(c) <= unibrow::Utf8::kMaxOneByteChar) {
-    *cursor_++ = c;  // Common case: 7-bit ASCII.
-  } else {
-    cursor_ += unibrow::Utf8::Encode(cursor_, c);
-  }
-  ASSERT(pos() <= Capacity());
-}
-
-
 // ----------------------------------------------------------------------------
 // UTF16Buffer
 
 
 UTF16Buffer::UTF16Buffer()
     : pos_(0), end_(Scanner::kNoEndPosition) { }
 
 
 // CharacterStreamUTF16Buffer
 CharacterStreamUTF16Buffer::CharacterStreamUTF16Buffer()
@@ skipping 297 matching lines @@
                                    start_position,
                                    end_position);
     source_ = &char_stream_buffer_;
   }
 
   is_parsing_json_ = (language == JSON);
 
   // Set c0_ (one character ahead)
   ASSERT(kCharacterLookaheadBufferSize == 1);
   Advance();
-  // Initializer current_ to not refer to a literal buffer.
-  current_.literal_buffer = NULL;
+  // Initialise current_ to not refer to a literal.
+  current_.literal_chars = Vector<const char>();
 
   // Skip initial whitespace allowing HTML comment ends just like
   // after a newline and scan first token.
   has_line_terminator_before_next_ = true;
   SkipWhiteSpace();
   Scan();
 }
 
 
 Token::Value Scanner::Next() {
   // BUG 1215673: Find a thread safe way to set a stack limit in
   // pre-parse mode. Otherwise, we cannot safely pre-parse from other
   // threads.
   current_ = next_;
   // Check for stack-overflow before returning any tokens.
   StackLimitCheck check;
   if (check.HasOverflowed()) {
     stack_overflow_ = true;
     next_.token = Token::ILLEGAL;
   } else {
     Scan();
   }
   return current_.token;
 }
 
 
 void Scanner::StartLiteral() {
-  // Use the first buffer unless it's currently in use by the current_ token.
-  // In most cases we won't have two literals/identifiers in a row, so
-  // the second buffer won't be used very often and is unlikely to grow much.
-  UTF8Buffer* free_buffer =
-      (current_.literal_buffer != &literal_buffer_1_) ? &literal_buffer_1_
-                                                      : &literal_buffer_2_;
-  next_.literal_buffer = free_buffer;
-  free_buffer->Reset();
+  literal_buffer_.StartLiteral();
 }
 
 
 void Scanner::AddChar(uc32 c) {
-  next_.literal_buffer->AddChar(c);
+  literal_buffer_.AddChar(c);
+}
+
+void Scanner::TerminateLiteral() {
+  next_.literal_chars = literal_buffer_.EndLiteral();
 }
 
 
-void Scanner::TerminateLiteral() {
-  AddChar(0);
-}
-
-
 void Scanner::AddCharAdvance() {
   AddChar(c0_);
   Advance();
 }
 
 
 static inline bool IsByteOrderMark(uc32 c) {
   // The Unicode value U+FFFE is guaranteed never to be assigned as a
   // Unicode character; this implies that in a Unicode context the
   // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF
@@ skipping 106 matching lines @@
     PushBack('-');  // undo Advance()
   }
   PushBack('!');  // undo Advance()
   ASSERT(c0_ == '!');
   return Token::LT;
 }
 
 
 
 void Scanner::ScanJson() {
-  next_.literal_buffer = NULL;
+  next_.literal_chars = Vector<const char>();
   Token::Value token;
   has_line_terminator_before_next_ = false;
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
     switch (c0_) {
       case '\t':
       case '\r':
       case '\n':
       case ' ':
@@ skipping 165 matching lines @@
     Advance();
     text++;
   }
   if (kIsIdentifierPart.get(c0_)) return Token::ILLEGAL;
   TerminateLiteral();
   return token;
 }
 
 
 void Scanner::ScanJavaScript() {
-  next_.literal_buffer = NULL;
+  next_.literal_chars = Vector<const char>();
   Token::Value token;
   has_line_terminator_before_next_ = false;
   do {
     // Remember the position of the next token
     next_.location.beg_pos = source_pos();
 
     switch (c0_) {
       case ' ':
       case '\t':
         Advance();
@@ skipping 562 matching lines @@
     }
     AddCharAdvance();
   }
   TerminateLiteral();
 
   next_.location.end_pos = source_pos() - 1;
   return true;
 }
 
 } }  // namespace v8::internal