Created collector class and used it to collect identifiers during scanning.

src/scanner.h

 // Copyright 2006-2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 22 matching lines @@
 
 namespace v8 {
 namespace internal {
 
 
 class UTF8Buffer {
  public:
   UTF8Buffer();
   ~UTF8Buffer();
 
-  void AddChar(uc32 c) {
-    ASSERT_NOT_NULL(data_);
-    if (cursor_ <= limit_ &&
-        static_cast<unsigned>(c) <= unibrow::Utf8::kMaxOneByteChar) {
-      *cursor_++ = static_cast<char>(c);
+  inline void AddChar(uc32 c) {
+    if (static_cast<unsigned>(c) <= unibrow::Utf8::kMaxOneByteChar) {
+      buffer_.Add(static_cast<char>(c));
     } else {
       AddCharSlow(c);
     }
   }
 
-  void Reset() {
-    if (data_ == NULL) {
-      data_ = NewArray<char>(kInitialCapacity);
-      limit_ = ComputeLimit(data_, kInitialCapacity);
-    }
-    cursor_ = data_;
+  void StartLiteral() {
+    buffer_.StartSequence();
   }
 
-  int pos() const {
-    ASSERT_NOT_NULL(data_);
-    return static_cast<int>(cursor_ - data_);
+  Vector<const char> EndLiteral() {
+    buffer_.Add(kEndMarker);
+    Vector<char> sequence = buffer_.EndSequence();
+    return Vector<const char>(sequence.start(), sequence.length());
   }
 
-  char* data() const { return data_; }
-
+  // The end marker added after a parsed literal.
+  // Using zero allows the usage of strlen and similar functions on
+  // identifiers and numbers (but not strings, since they may contain zero
+  // bytes).
+  // TODO(lrn): Use '\xff' as end marker, since it cannot occur inside
+  // an utf-8 string. This requires changes in all places that uses
+  // str-functions on the literals, but allows a single pointer to represent
+  // the literal, even if it contains embedded zeros.
+  static const char kEndMarker = '\x00';
  private:
   static const int kInitialCapacity = 256;
-  char* data_;
-  char* cursor_;
-  char* limit_;
-
-  int Capacity() const {
-    ASSERT_NOT_NULL(data_);
-    return static_cast<int>(limit_ - data_) + unibrow::Utf8::kMaxEncodedSize;
-  }
-
-  static char* ComputeLimit(char* data, int capacity) {
-    return (data + capacity) - unibrow::Utf8::kMaxEncodedSize;
-  }
+  SequenceCollector<char> buffer_;
 
   void AddCharSlow(uc32 c);
 };
 
 
 // Interface through which the scanner reads characters from the input source.
 class UTF16Buffer {
  public:
   UTF16Buffer();
   virtual ~UTF16Buffer() {}
@@ skipping 215 matching lines @@
   Location location() const  { return current_.location; }
   Location peek_location() const  { return next_.location; }
 
   // Returns the literal string, if any, for the current token (the
   // token returned by Next()). The string is 0-terminated and in
   // UTF-8 format; they may contain 0-characters. Literal strings are
   // collected for identifiers, strings, and numbers.
   // These functions only give the correct result if the literal
   // was scanned between calls to StartLiteral() and TerminateLiteral().
   const char* literal_string() const {
-    return current_.literal_buffer->data();
+    return current_.literal_chars.start();
   }
+
   int literal_length() const {
-    // Excluding terminal '\0' added by TerminateLiteral().
-    return current_.literal_buffer->pos() - 1;
+    // Excluding terminal '\x00' added by TerminateLiteral().
+    return current_.literal_chars.length() - 1;
+  }
+
+  Vector<const char> literal() const {
+    return Vector<const char>(literal_string(), literal_length());
   }
 
   // Returns the literal string for the next token (the token that
   // would be returned if Next() were called).
   const char* next_literal_string() const {
-    return next_.literal_buffer->data();
+    return next_.literal_chars.start();
   }
+
+
   // Returns the length of the next token (that would be returned if
   // Next() were called).
   int next_literal_length() const {
-    return next_.literal_buffer->pos() - 1;
+    // Excluding terminal '\x00' added by TerminateLiteral().
+    return next_.literal_chars.length() - 1;
   }
 
   Vector<const char> next_literal() const {
-    return Vector<const char>(next_literal_string(),
-                              next_literal_length());
+    return Vector<const char>(next_literal_string(), next_literal_length());
   }
 
   // Scans the input as a regular expression pattern, previous
   // character(s) must be /(=). Returns true if a pattern is scanned.
   bool ScanRegExpPattern(bool seen_equal);
   // Returns true if regexp flags are scanned (always since flags can
   // be empty).
   bool ScanRegExpFlags();
 
   // Seek forward to the given position.  This operation does not
@@ skipping 16 matching lines @@
   static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace;
 
   static const int kCharacterLookaheadBufferSize = 1;
   static const int kNoEndPosition = 1;
 
  private:
   // The current and look-ahead token.
   struct TokenDesc {
     Token::Value token;
     Location location;
-    UTF8Buffer* literal_buffer;
+    Vector<const char> literal_chars;
   };
 
   void Init(Handle<String> source,
             unibrow::CharacterStream* stream,
             int start_position, int end_position,
             ParserLanguage language);
 
   // Literal buffer support
-  void StartLiteral();
-  void AddChar(uc32 ch);
-  void AddCharAdvance();
-  void TerminateLiteral();
+  inline void StartLiteral();
+  inline void AddChar(uc32 ch);
+  inline void AddCharAdvance();
+  inline void TerminateLiteral();
 
   // Low-level scanning support.
   void Advance() { c0_ = source_->Advance(); }
   void PushBack(uc32 ch) {
     source_->PushBack(ch);
     c0_ = ch;
   }
 
   bool SkipWhiteSpace() {
     if (is_parsing_json_) {
@@ skipping 83 matching lines @@
   ExternalStringUTF16Buffer<ExternalAsciiString, char> ascii_string_buffer_;
 
   // Source. Will point to one of the buffers declared above.
   UTF16Buffer* source_;
 
   // Used to convert the source string into a character stream when a stream
   // is not passed to the scanner.
   SafeStringInputBuffer safe_string_input_buffer_;
 
   // Buffer to hold literal values (identifiers, strings, numbers)
-  // using 0-terminated UTF-8 encoding.
-  UTF8Buffer literal_buffer_1_;
-  UTF8Buffer literal_buffer_2_;
+  // using '\x00'-terminated UTF-8 encoding. Handles allocation internally.
+  UTF8Buffer literal_buffer_;
 
   bool stack_overflow_;
   static StaticResource<Utf8Decoder> utf8_decoder_;
 
   // One Unicode character look-ahead; c0_ < 0 at the end of the input.
   uc32 c0_;
 };
 
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_