Add script streaming API

include/v8.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /** \mainpage V8 API Reference Guide
  *
  * V8 is Google's open source JavaScript engine.
  *
  * This set of documents provides reference material generated from the
  * V8 header file, include/v8.h.
@@ skipping 108 matching lines @@
 class Isolate;
 class DeclaredAccessorDescriptor;
 class ObjectOperationDescriptor;
 class RawOperationDescriptor;
 class CallHandlerHelper;
 class EscapableHandleScope;
 template<typename T> class ReturnValue;
 
 namespace internal {
 class Arguments;
+class BackgroundParsingTask;
+class CompilationInfo;
+class ExternalStreamingStream;
 class Heap;
 class HeapObject;
 class Isolate;
 class Object;
 template<typename T> class CustomArguments;
 class PropertyCallbackArguments;
 class FunctionCallbackArguments;
 class GlobalHandles;
 }
 
@@ skipping 942 matching lines @@
     Handle<Integer> resource_line_offset;
     Handle<Integer> resource_column_offset;
     Handle<Boolean> resource_is_shared_cross_origin;
 
     // Cached data from previous compilation (if a kConsume*Cache flag is
     // set), or hold newly generated cache data (kProduce*Cache flags) are
     // set when calling a compile method.
     CachedData* cached_data;
   };
 
+  /**
+   * For streaming incomplete script data to V8. The embedder should implement a
+   * subclass of this class.
+   */
+  class V8_EXPORT ExternalSourceStream {
+   public:
+    enum Encoding { ONE_BYTE, TWO_BYTE, UTF8, UNKNOWN };
+
+    ExternalSourceStream() : encoding(UNKNOWN) {}
+    virtual ~ExternalSourceStream() {}
+
+    /**
+     * V8 calls this to request the next chunk of data from the embedder. This
+     * function will be called on a background thread, so it's OK to block and
+     * wait for the data, if the embedder doesn't have data yet. Returns the
+     * length of the data returned. The caller takes ownership of the data.
+     */
+    virtual unsigned GetMoreData(const char** src) = 0;

[Sven Panne, 2014/09/01 09:07:46]

I think size_t is more appropriate than unsigned.

[marja, 2014/09/01 11:58:17]

Done.

+
+   protected:
+    /**
+     * The embedder should call this as soon as it knows the encoding. It must
+     * be called before the first call to GetMoreData returns.
+     */
+    void SetEncoding(Encoding e) { encoding = e; }

[Sven Panne, 2014/09/01 09:07:46]

What's the rationale behind this two-stage initialization of the encoding?
Naively, I would assume a single constructor with the encoding and "encoding"
being const. If there is a good reason for the current design, it should be
explained in a comment.

[marja, 2014/09/01 11:58:17]

The original purpose was to allow the embedder pass the encoding when the first
block of data has arrived (i.e., not require it until it's absolutely needed).
However, the current Blink side does *not* do encoding detection from data, so I
simplified this: encoding is now passed as ctor param.

If we really want to do encoding detection on the Blink side, Blink can either
delay constructing ExternalSourceStream, or this can change.

+
+   private:
+    friend class internal::ExternalStreamingStream;
+    Encoding encoding;
+  };
+
+
+  /**
+   * Source code which can be streamed into V8 in pieces. It will be parsed
+   * while streaming. It can be compiled after the streaming is complete.
+   * StreamedSource must be kept alive while the compilation is ongoing.
+   */
+  class StreamedSource {
+   public:
+    V8_INLINE explicit StreamedSource(ExternalSourceStream* source_stream);
+    ~StreamedSource();

[Sven Panne, 2014/09/01 09:07:46]

If we don't allow subclassing, we should mark this class as final. No clue
whether we want subclassing, though... :-)

[marja, 2014/09/01 11:58:17]

Added V8_FINAL (subclassing this won't make sense; it has no virtual funcs).

Btw, none of the other classes in the v8.h have V8_FINAL defined. (But the macro
is accessible, since this includes v8stdint.h which includes v8config.h.)

[marja, 2014/09/02 12:43:36]

Meanwhile, things seem to progress in bleeding edge. There's no V8_FINAL any
more; now V8 is using FINAL like chromium, and FINAL is not defined by any of
the include files of include/v8.h.

So I'm removing V8_FINAL; no other class in the API defined it anyway, so would
be a bit obscure to add it to this class only...

+
+    // Ownership of the CachedData or its buffers is *not* transferred to the
+    // caller. The CachedData object is alive as long as the StreamedSource
+    // object is alive.
+    V8_INLINE const CachedData* GetCachedData() const;
+
+   private:
+    friend class ScriptCompiler;
+    friend class internal::BackgroundParsingTask;
+
+    // Prevent copying. Not implemented.
+    StreamedSource(const StreamedSource&);
+    StreamedSource& operator=(const StreamedSource&);
+
+    ExternalSourceStream* source_stream;
+
+    // Cached data generated during compilation (if the generate_cached_data
+    // flag is passed to ScriptCompiler::StartStreamingScript). Streamed scripts
+    // cannot utilize already existing cached data.
+    CachedData* cached_data;
+
+    internal::CompilationInfo* info;
+  };
+
+  /**
+   * A streaming task which the embedder must run on a background thread to
+   * stream scripts into V8. Returned by ScriptCompiler::StartStreamingScript.
+   */
+  class ScriptStreamingTask {
+   public:
+    virtual ~ScriptStreamingTask() {}
+    virtual void Run() = 0;
+  };
+
   enum CompileOptions {
     kNoCompileOptions = 0,
     kProduceParserCache,
     kConsumeParserCache,
     kProduceCodeCache,
     kConsumeCodeCache,
 
     // Support the previous API for a transition period.
     kProduceDataToCache
   };
 
   /**
    * Compiles the specified script (context-independent).
    * Cached data as part of the source object can be optionally produced to be
    * consumed later to speed up compilation of identical source scripts.
    *
    * Note that when producing cached data, the source must point to NULL for
    * cached data. When consuming cached data, the cached data must have been
    * produced by the same version of V8.
    *
    * \param source Script source code.
    * \return Compiled script object (context independent; for running it must be
    *   bound to a context).
    */
   static Local<UnboundScript> CompileUnbound(
       Isolate* isolate, Source* source,
       CompileOptions options = kNoCompileOptions);
 
   /**
+   * Returns a task which streams script data into V8, or NULL if the script
+   * cannot be streamed. The user is responsible of running the task on a
+   * background thread and deleting it. When ran, the task starts parsing the
+   * script, and it will request data from the StreamedSource as needed. When
+   * ScriptStreamingTask::Run exits, all data has been streamed and the script
+   * can be compiled (see Compile below).
+   *
+   * This API allows to start the streaming with as little data as possible, and
+   * the remaining data (for example, the ScriptOrigin) is passed to Compile.
+   */
+  static ScriptStreamingTask* StartStreamingScript(

[Sven Panne, 2014/09/01 09:07:46]

It might be nice to move this function and the one below to Isolate, but this is
not crucial. Nevertheless, "static" functions are a code smell most of the time.

[marja, 2014/09/01 11:58:17]

The other compile functions are below, as static functions of ScriptCompiler, so
I think these make sense here, given the current state of the rest of the API.

I'm not convinced that putting everything to Isolate smells less. At least now
the functions are grouped nicely (functions related to compiling scripts are in
ScriptCompiler).

Ofc if the Compile funcs were to move away from ScriptCompiler, these funcs
would move with them at that point.

+      Isolate* isolate, StreamedSource* source,
+      CompileOptions options = kNoCompileOptions);
+
+  /**
+   * Compiles a streamed script (bound to current context).
+   *
+   * This can only be called after the streaming has finished (the background
+   * task has been run and completed). V8 doesn't construct the source string
+   * during streaming, so the embedder needs to pass the full source here.
+   */
+  static Local<Script> Compile(Isolate* isolate, StreamedSource* source,
+                               Handle<String> full_source_string,
+                               const ScriptOrigin& origin);
+
+  /**
    * Compiles the specified script (bound to current context).
    *
    * \param source Script source code.
    * \param pre_data Pre-parsing data, as obtained by ScriptData::PreCompile()
    *   using pre_data speeds compilation if it's done multiple times.
    *   Owned by caller, no references are kept when this function returns.
    * \return Compiled script object, bound to the context that was active
    *   when this function was called. When run it will always use this
    *   context.
    */
@@ skipping 5125 matching lines @@
   delete cached_data;
 }
 
 
 const ScriptCompiler::CachedData* ScriptCompiler::Source::GetCachedData()
     const {
   return cached_data;
 }
 
 
+ScriptCompiler::StreamedSource::StreamedSource(ExternalSourceStream* stream)
+    : source_stream(stream), cached_data(NULL), info(NULL) {}
+
+
+const ScriptCompiler::CachedData*
+ScriptCompiler::StreamedSource::GetCachedData() const {
+  return cached_data;
+}
+
+
 Handle<Boolean> Boolean::New(Isolate* isolate, bool value) {
   return value ? True(isolate) : False(isolate);
 }
 
 
 void Template::Set(Isolate* isolate, const char* name, v8::Handle<Data> value) {
   Set(v8::String::NewFromUtf8(isolate, name), value);
 }
 
 
@@ skipping 547 matching lines @@
  */
 
 
 }  // namespace v8
 
 
 #undef TYPE_CHECK
 
 
 #endif  // V8_H_