Add script streaming API

include/v8.h

Index: include/v8.h
diff --git a/include/v8.h b/include/v8.h
index ef0bda63f43d82e36eed2269462be9990111dbe9..0ace2d08516e8ad397be795738d16b00a3162121 100644
--- a/include/v8.h
+++ b/include/v8.h
@@ -125,6 +125,10 @@ template<typename T> class ReturnValue;
 
 namespace internal {
 class Arguments;
+class BackgroundParsingTask;
+class CompilationInfo;
+class ExternalStreamingStream;
+class FinalizeParsingTask;
 class Heap;
 class HeapObject;
 class Isolate;
@@ -945,6 +949,38 @@ class ScriptOrigin {
 
 
 /**
+ * For streaming incomplete script data to V8. The embedder should implement a
+ * subclass of this class.
+ */
+class V8_EXPORT ExternalSourceStream {
+ public:
+  enum Encoding { ONE_BYTE, TWO_BYTE, UTF8, UNKNOWN };
+
+  ExternalSourceStream() : encoding(UNKNOWN) {}
+  virtual ~ExternalSourceStream() {}
+
+  /**
+   * V8 calls this to request the next chunk of data from the embedder. This
+   * function will be called on a background thread, so it's OK to block and
+   * wait for the data, if the embedder doesn't have data yet. Returns the
+   * length of the data returned. The caller takes ownership of the data.
+   */
+  virtual unsigned GetSomeData(const char** src, unsigned position) = 0;

[jochen (gone - plz use gerrit), 2014/08/13 14:24:45]

GetMoreData?

it's a bit odd to pass TWO_BYTE data around as char* btw

what's the meaning of position?

why is v8 creating the background thread, and not the embedder? maybe
StartStreamingScript should return some object that has a method
"StartOnThread()". the embedder then creates a thread and calls that method on
that thread.

[marja, 2014/08/14 11:29:21]

1) This API (the name GetSomeData and passing the position) was adopted from
SharedBuffer. But GetMoreData without position works too (the embedder can keep
track of where we are), so changed this to GetMoreData without position.

2) It's not odd to pass raw data as char*. Note that V8 is responsible of
decoding it, and at this point it's just a byte stream. Other places  pass raw
byte data (which is possibly UTF-16) as char* too, for example the threaded HTML
parser: BackgroundHTMLParser::appendRawBytesFromParserThread

3) Moved the threading to Blink. This now returns a ScriptStreamingTask which
has a method Run(). (StartOnThread() would be implying that it just launches
something and returns, whereas Run() can really block and wait for data.)

+
+ protected:
+  /**
+   * The embedder should call this as soon as it knows the encoding. It must be
+   * called before the first call to GetSomeData returns.
+   */
+  void SetEncoding(Encoding e) { encoding = e; }
+
+ private:
+  friend class internal::ExternalStreamingStream;
+  Encoding encoding;
+};
+
+
+/**
  * A compiled JavaScript script, not yet tied to a Context.
  */
 class V8_EXPORT UnboundScript {
@@ -1016,6 +1052,9 @@ class V8_EXPORT Script {
  */
 class V8_EXPORT ScriptCompiler {
  public:
+  class StreamedSource;
+  typedef void (*StreamingCompleteCallback)(StreamedSource*, void*);
+
   /**
    * Compilation data that the embedder can cache and pass back to speed up
    * future compilations. The data is produced if the CompilerOptions passed to
@@ -1087,6 +1126,41 @@ class V8_EXPORT ScriptCompiler {
     CachedData* cached_data;
   };
 
+  /**
+   * Source code which can be streamed into V8 in pieces. It will be parsed
+   * while streaming, and compiled after the streaming is complete.
+   * StreamedSource must be kept alive while the compilation is ongoing.
+   */
+  class StreamedSource {
+   public:
+    // Source takes ownership of CachedData.
+    V8_INLINE StreamedSource(ExternalSourceStream* source_stream);

[jochen (gone - plz use gerrit), 2014/08/13 14:24:45]

explicit

[marja, 2014/08/14 11:29:21]

Done.

+    ~StreamedSource();
+
+    // Ownership of the CachedData or its buffers is *not* transferred to the
+    // caller. The CachedData object is alive as long as the Source object is

[jochen (gone - plz use gerrit), 2014/08/13 14:24:45]

you mean StreamedSource object?

[marja, 2014/08/14 11:29:22]

Done.

+    // alive.
+    V8_INLINE const CachedData* GetCachedData() const;
+
+   private:
+    friend class ScriptCompiler;
+    friend class internal::BackgroundParsingTask;
+    friend class internal::FinalizeParsingTask;
+
+    // Prevent copying. Not implemented.
+    StreamedSource(const StreamedSource&);
+    StreamedSource& operator=(const StreamedSource&);
+
+    ExternalSourceStream* source_stream;
+
+    // Cached data generated during compilation (if the generate_cached_data
+    // flag is passed to ScriptCompiler::StartStreamingScript). Streamed scripts
+    // cannot utilize already existing cached data.
+    CachedData* cached_data;
+
+    internal::CompilationInfo* info;
+  };
+
   enum CompileOptions {
     kNoCompileOptions = 0,
     kProduceParserCache,
@@ -1116,6 +1190,28 @@ class V8_EXPORT ScriptCompiler {
       CompileOptions options = kNoCompileOptions);
 
   /**
+   * Starts streaming script data into V8. This API allows to start the
+   * streaming with as little data as possible, and the remaining data (for
+   * example, the ScriptOrigin) is passed to Compile (see below). The script is
+   * parsed while streaming. When all data has been streamed, the callback
+   * provided by the embedder is called, and the script can be compiled.
+   */
+  static bool StartStreamingScript(Isolate* isolate, StreamedSource* source,
+                                   StreamingCompleteCallback callback,
+                                   void* callback_data,
+                                   CompileOptions options = kNoCompileOptions);
+
+  /**
+   * Compiles a streamed script. This can only be called after the streaming has
+   * finished (the callback passed to StartStreamingScript has been called). V8
+   * doesn't construct the source string as it streams, so the embedder needs to
+   * pass the full source here.
+   */
+  static Local<Script> Compile(Isolate* isolate, StreamedSource* source,
+                               Handle<String> full_source_string,
+                               const ScriptOrigin& origin);
+
+  /**
    * Compiles the specified script (bound to current context).
    *
    * \param source Script source code.
@@ -6180,6 +6276,16 @@ const ScriptCompiler::CachedData* ScriptCompiler::Source::GetCachedData()
 }
 
 
+ScriptCompiler::StreamedSource::StreamedSource(ExternalSourceStream* stream)
+    : source_stream(stream), cached_data(NULL), info(NULL) {}
+
+
+const ScriptCompiler::CachedData*
+ScriptCompiler::StreamedSource::GetCachedData() const {
+  return cached_data;
+}
+
+
 Handle<Boolean> Boolean::New(Isolate* isolate, bool value) {
   return value ? True(isolate) : False(isolate);
 }