Add script streaming API

src/scanner-character-streams.cc

Index: src/scanner-character-streams.cc
diff --git a/src/scanner-character-streams.cc b/src/scanner-character-streams.cc
index 9ec0ad1008316dfca01264c619d9b5c88655a231..d38b1d440a0f7f234f54d4ad3944c39ea77199af 100644
--- a/src/scanner-character-streams.cc
+++ b/src/scanner-character-streams.cc
@@ -6,12 +6,43 @@
 
 #include "src/scanner-character-streams.h"
 
+#include "include/v8.h"
 #include "src/handles.h"
 #include "src/unicode-inl.h"
 
 namespace v8 {
 namespace internal {
 
+namespace {
+
+unsigned CopyCharsHelper(
+    uint16_t* dest, unsigned length, const char* src, unsigned* src_pos,
+    unsigned src_length,
+    ScriptCompiler::ExternalSourceStream::Encoding encoding) {
+  if (encoding == ScriptCompiler::ExternalSourceStream::UTF8) {
+    return v8::internal::Utf8ToUtf16CharacterStream::CopyChars(
+        dest, length, reinterpret_cast<const uint8_t*>(src), src_pos,
+        src_length);
+  }
+
+  unsigned to_fill = length;
+  if (to_fill > src_length - *src_pos) to_fill = src_length - *src_pos;
+
+  if (encoding == ScriptCompiler::ExternalSourceStream::ONE_BYTE) {
+    v8::internal::CopyChars<uint8_t, uint16_t>(
+        dest, reinterpret_cast<const uint8_t*>(src + *src_pos), to_fill);
+  } else {
+    DCHECK(encoding == ScriptCompiler::ExternalSourceStream::TWO_BYTE);
+    v8::internal::CopyChars<uint16_t, uint16_t>(
+        dest, reinterpret_cast<const uint16_t*>(src + *src_pos), to_fill);
+  }
+  *src_pos += to_fill;
+  return to_fill;
+}
+
+}  // namespace
+
+
 // ----------------------------------------------------------------------------
 // BufferedUtf16CharacterStreams
 
@@ -145,6 +176,35 @@ Utf8ToUtf16CharacterStream::Utf8ToUtf16CharacterStream(const byte* data,
 Utf8ToUtf16CharacterStream::~Utf8ToUtf16CharacterStream() { }
 
 
+unsigned Utf8ToUtf16CharacterStream::CopyChars(uint16_t* dest, unsigned length,
+                                               const byte* src,
+                                               unsigned* src_pos,
+                                               unsigned src_length) {
+  static const unibrow::uchar kMaxUtf16Character = 0xffff;
+  unsigned i = 0;
+  // Because of the UTF-16 lead and trail surrogates, we stop filling the buffer
+  // one character early (in the normal case), because we need to have at least
+  // two free spaces in the buffer to be sure that the next character will fit.
+  while (i < length - 1) {
+    if (*src_pos == src_length) break;
+    unibrow::uchar c = src[*src_pos];
+    if (c <= unibrow::Utf8::kMaxOneByteChar) {
+      *src_pos = *src_pos + 1;
+    } else {
+      c = unibrow::Utf8::CalculateValue(src + *src_pos, src_length - *src_pos,
+                                        src_pos);
+    }
+    if (c > kMaxUtf16Character) {
+      dest[i++] = unibrow::Utf16::LeadSurrogate(c);
+      dest[i++] = unibrow::Utf16::TrailSurrogate(c);
+    } else {
+      dest[i++] = static_cast<uc16>(c);
+    }
+  }
+  return i;
+}
+
+
 unsigned Utf8ToUtf16CharacterStream::BufferSeekForward(unsigned delta) {
   unsigned old_pos = pos_;
   unsigned target_pos = pos_ + delta;
@@ -156,31 +216,14 @@ unsigned Utf8ToUtf16CharacterStream::BufferSeekForward(unsigned delta) {
 
 
 unsigned Utf8ToUtf16CharacterStream::FillBuffer(unsigned char_position) {
-  static const unibrow::uchar kMaxUtf16Character = 0xffff;
   SetRawPosition(char_position);
   if (raw_character_position_ != char_position) {
     // char_position was not a valid position in the stream (hit the end
     // while spooling to it).
     return 0u;
   }
-  unsigned i = 0;
-  while (i < kBufferSize - 1) {
-    if (raw_data_pos_ == raw_data_length_) break;
-    unibrow::uchar c = raw_data_[raw_data_pos_];
-    if (c <= unibrow::Utf8::kMaxOneByteChar) {
-      raw_data_pos_++;
-    } else {
-      c =  unibrow::Utf8::CalculateValue(raw_data_ + raw_data_pos_,
-                                         raw_data_length_ - raw_data_pos_,
-                                         &raw_data_pos_);
-    }
-    if (c > kMaxUtf16Character) {
-      buffer_[i++] = unibrow::Utf16::LeadSurrogate(c);
-      buffer_[i++] = unibrow::Utf16::TrailSurrogate(c);
-    } else {
-      buffer_[i++] = static_cast<uc16>(c);
-    }
-  }
+  unsigned i = CopyChars(buffer_, kBufferSize, raw_data_, &raw_data_pos_,
+                         raw_data_length_);
   raw_character_position_ = char_position + i;
   return i;
 }
@@ -276,6 +319,45 @@ void Utf8ToUtf16CharacterStream::SetRawPosition(unsigned target_position) {
 }
 
 
+unsigned ExternalStreamingStream::FillBuffer(unsigned position) {
+  // Ignore "position" which is the position in the decoded data. Instead,
+  // ExternalStreamingStream keeps track of the position in the raw data.
+  unsigned data_in_buffer = 0;
+  // Note that the UTF-8 decoder might not be able to fill the buffer
+  // completely; it will typically leave the last character empty (see
+  // Utf8ToUtf16CharacterStream::CopyChars).
+  while (data_in_buffer < kBufferSize - 1) {
+    if (current_data_ == NULL) {
+      // GetSomeData will wait until the embedder has enough data.
+      current_data_length_ = source_stream_->GetMoreData(&current_data_);
+      current_data_offset_ = 0;
+      // Did the data stream end?
+      if (current_data_length_ == 0) {
+        return data_in_buffer;
+      }
+    }
+    // Fill the buffer from current_data_. FIXME: this doesn't work if the data
+    // chunk ends in the middle of an UTF-8 character.
+    unsigned new_offset = 0;
+    unsigned new_chars_in_buffer = CopyCharsHelper(
+        buffer_ + data_in_buffer, kBufferSize - data_in_buffer,
+        current_data_ + current_data_offset_, &new_offset,
+        current_data_length_ - current_data_offset_, source_stream_->encoding);
+    data_in_buffer += new_chars_in_buffer;
+    current_data_offset_ += new_offset;
+    DCHECK(data_in_buffer <= kBufferSize);
+    // Did we use all the data?
+    if (current_data_offset_ == current_data_length_) {
+      delete[] current_data_;
+      current_data_ = NULL;
+      current_data_length_ = 0;
+      current_data_offset_ = 0;
+    }
+  }
+  return data_in_buffer;
+}
+
+
 // ----------------------------------------------------------------------------
 // ExternalTwoByteStringUtf16CharacterStream