Changed interface to preparser to not require pushback support.

src/preparser-api.cc

 // Copyright 2010 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 23 matching lines @@
 #include "scanner-base.h"
 #include "preparse-data.h"
 #include "preparser.h"
 
 namespace v8 {
 namespace internal {
 
 // UTF16Buffer based on a v8::UnicodeInputStream.
 class InputStreamUTF16Buffer : public UC16CharacterStream {
  public:
+  /* The InputStreamUTF16Buffer maintains an internal buffer
+   * that is filled in chunks from the UC16CharacterStream.
+   * It also maintains unlimited pushback capability, but optimized
+   * for small pushbacks.
+   * The pushback_buffer_ pointer points to the limit of pushbacks
+   * in the current buffer. There is room for a few pushback'ed chars before
+   * the buffer containing the most recently read chunk. If this is overflowed,
+   * an external buffer is allocated/reused to hold further pushbacks, and

[Rico, 2010/12/08 09:46:25]

and but -> and

[Lasse Reichstein, 2010/12/08 10:05:36]

Done.

+   * but pushback_buffer_ and buffer_cursor_/buffer_end_ now points to the
+   * new buffer. When this buffer is read to the end again, the cursor is
+   * switched back to the internal buffer
+   */
   explicit InputStreamUTF16Buffer(v8::UnicodeInputStream* stream)
       : UC16CharacterStream(),
         stream_(stream),
-        pushback_active_(false) {
+        pushback_buffer_(buffer_),
+        pushback_buffer_end_cache_(NULL),
+        pushback_buffer_backing_(NULL),
+        pushback_buffer_backing_size_(0) {
     buffer_cursor_ = buffer_end_ = buffer_ + kPushBackSize;
   }
 
-  virtual ~InputStreamUTF16Buffer() { }
+  virtual ~InputStreamUTF16Buffer() {
+    if (pushback_buffer_backing_ != NULL) {
+      DeleteArray(pushback_buffer_backing_);
+    }
+  }
 
   virtual void PushBack(uc16 ch) {
     ASSERT(pos_ > 0);
-    if (buffer_cursor_ > buffer_) {
-      // While we can stay within the buffer, just do so.
-      *--buffer_cursor_ = ch;
-      pos_--;
-      return;
+    if (buffer_cursor_ <= pushback_buffer_) {
+      // No more room in the current buffer to do pushbacks.
+      if (pushback_buffer_end_cache_ == NULL) {
+        // We have overflowed the pushback space at the beginning of buffer_.
+        // Switch to using a separate allocated pushback buffer.
+        if (pushback_buffer_backing_ == NULL) {
+          // Allocate a buffer the first time we need it.
+          pushback_buffer_backing_ = NewArray<uc16>(kPushBackSize);
+          pushback_buffer_backing_size_ = kPushBackSize;
+        }
+        pushback_buffer_ = pushback_buffer_backing_;
+        pushback_buffer_end_cache_ = buffer_end_;
+        buffer_end_ = pushback_buffer_backing_ + pushback_buffer_backing_size_;
+        buffer_cursor_ = buffer_end_ - 1;
+      } else {
+        // Hit the bottom of the allocated pushback buffer.
+        // Double the buffer and continue.
+        uc16* new_buffer = NewArray<uc16>(pushback_buffer_backing_size_ * 2);
+        memcpy(new_buffer + pushback_buffer_backing_size_,
+               pushback_buffer_backing_,
+               pushback_buffer_backing_size_);
+        DeleteArray(pushback_buffer_backing_);
+        buffer_cursor_ = new_buffer + pushback_buffer_backing_size_;
+        pushback_buffer_backing_ = pushback_buffer_ = new_buffer;
+        buffer_end_ = pushback_buffer_backing_ + pushback_buffer_backing_size_;
+      }
     }
-    if (!pushback_active_) {
-      // Push back the entire buffer to the stream and let the
-      // stream handle pushbacks from now.
-      // We leave buffer_cursor_ == buffer_end_, so the next read
-      // will fill the buffer from the current position.
-      // This should happen exceedingly rarely.
-      while (buffer_end_ > buffer_) {
-        stream_->PushBack(*--buffer_end_);
-      }
-      buffer_cursor_ = buffer_end_;
-      pushback_active_ = true;
-    }
-    stream_->PushBack(ch);
+    pushback_buffer_[buffer_cursor_ - pushback_buffer_- 1] = ch;
     pos_--;
   }
 
  protected:
   virtual bool ReadBlock() {
+    if (pushback_buffer_end_cache_ != NULL) {
+      buffer_cursor_ = buffer_;
+      buffer_end_ = pushback_buffer_end_cache_;
+      pushback_buffer_end_cache_ = NULL;
+      return buffer_end_ > buffer_cursor_;
+    }
     // Copy the top of the buffer into the pushback area.
-    pushback_active_ = false;
     int32_t value;
     uc16* buffer_start = buffer_ + kPushBackSize;
     buffer_cursor_ = buffer_end_ = buffer_start;
     while ((value = stream_->Next()) >= 0) {
       if (value > static_cast<int32_t>(unibrow::Utf8::kMaxThreeByteChar)) {
         value = unibrow::Utf8::kBadChar;
       }
       // buffer_end_ is a const pointer, but buffer_ is writable.
       buffer_start[buffer_end_++ - buffer_start] = static_cast<uc16>(value);
       if (buffer_end_ == buffer_ + kPushBackSize + kBufferSize) break;
     }
     return buffer_end_ > buffer_start;
   }
 
   virtual unsigned SlowSeekForward(unsigned pos) {
     // Seeking in the input is not used by preparsing.
     // It's only used by the real parser based on preparser data.
     UNIMPLEMENTED();
     return 0;
   }
 
  private:
   static const unsigned kBufferSize = 512;
   static const unsigned kPushBackSize = 16;
   v8::UnicodeInputStream* const stream_;
   // Buffer holding first kPushBackSize characters of pushback buffer,
   // then kBufferSize chars of read-ahead.
   // The pushback buffer is only used if pushing back characters past
   // the start of a block.
-  uc16 buffer_[kBufferSize + kPushBackSize];
-  bool pushback_active_;
+  uc16 buffer_[kPushBackSize + kBufferSize];
+  // Limit of pushbacks before new allocation is necessary.
+  uc16* pushback_buffer_;
+  // Only if that pushback buffer at the start of buffer_ isn't sufficient
+  // is the following used.
+  const uc16* pushback_buffer_end_cache_;
+  uc16* pushback_buffer_backing_;
+  unsigned pushback_buffer_backing_size_;
 };
 
 
 class StandAloneJavaScriptScanner : public JavaScriptScanner {
  public:
   void Initialize(UC16CharacterStream* source) {
     source_ = source;
     literal_flags_ = kLiteralString | kLiteralIdentifier;
     Init();
     // Skip initial whitespace allowing HTML comment ends just like
     // after a newline and scan first token.
     has_line_terminator_before_next_ = true;
     SkipWhiteSpace();
     Scan();
   }
 };
 
 
 // Functions declared by allocation.h
 
 void FatalProcessOutOfMemory(const char* reason) {
   V8_Fatal(__FILE__, __LINE__, reason);
 }
 
 bool EnableSlowAsserts() { return true; }
 
-
 }  // namespace internal.
 
 
 UnicodeInputStream::~UnicodeInputStream() { }
 
 
 PreParserData Preparse(UnicodeInputStream* input, size_t max_stack) {
   internal::InputStreamUTF16Buffer buffer(input);
   uintptr_t stack_limit = reinterpret_cast<uintptr_t>(&buffer) - max_stack;
   internal::StandAloneJavaScriptScanner scanner;
@@ skipping 13 matching lines @@
   return PreParserData(size, data);
 }
 
 }  // namespace v8.
 
 
 // Used by ASSERT macros and other immediate exits.
 extern "C" void V8_Fatal(const char* file, int line, const char* format, ...) {
   exit(EXIT_FAILURE);
 }