Add script streaming API

include/v8.h

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 /** \mainpage V8 API Reference Guide
  *
  * V8 is Google's open source JavaScript engine.
  *
  * This set of documents provides reference material generated from the
  * V8 header file, include/v8.h.
@@ skipping 107 matching lines @@
 class Isolate;
 class DeclaredAccessorDescriptor;
 class ObjectOperationDescriptor;
 class RawOperationDescriptor;
 class CallHandlerHelper;
 class EscapableHandleScope;
 template<typename T> class ReturnValue;
 
 namespace internal {
 class Arguments;
+class BackgroundParsingTask;
+class CompilationInfo;
+class ExternalStreamingStream;
+class FinalizeParsingTask;
 class Heap;
 class HeapObject;
 class Isolate;
 class Object;
 template<typename T> class CustomArguments;
 class PropertyCallbackArguments;
 class FunctionCallbackArguments;
 class GlobalHandles;
 }
 
@@ skipping 800 matching lines @@
  private:
   Handle<Value> resource_name_;
   Handle<Integer> resource_line_offset_;
   Handle<Integer> resource_column_offset_;
   Handle<Boolean> resource_is_shared_cross_origin_;
   Handle<Integer> script_id_;
 };
 
 
 /**
+ * For streaming incomplete script data to V8. The embedder should implement a
+ * subclass of this class.
+ */
+class V8_EXPORT ExternalSourceStream {
+ public:
+  enum Encoding { ONE_BYTE, TWO_BYTE, UTF8, UNKNOWN };
+
+  ExternalSourceStream() : encoding(UNKNOWN) {}
+  virtual ~ExternalSourceStream() {}
+
+  /**
+   * V8 calls this to request the next chunk of data from the embedder. This
+   * function will be called on a background thread, so it's OK to block and
+   * wait for the data, if the embedder doesn't have data yet. Returns the
+   * length of the data returned. The caller takes ownership of the data.
+   */
+  virtual unsigned GetSomeData(const char** src, unsigned position) = 0;

[jochen (gone - plz use gerrit), 2014/08/13 14:24:45]

GetMoreData?

it's a bit odd to pass TWO_BYTE data around as char* btw

what's the meaning of position?

why is v8 creating the background thread, and not the embedder? maybe
StartStreamingScript should return some object that has a method
"StartOnThread()". the embedder then creates a thread and calls that method on
that thread.

[marja, 2014/08/14 11:29:21]

1) This API (the name GetSomeData and passing the position) was adopted from
SharedBuffer. But GetMoreData without position works too (the embedder can keep
track of where we are), so changed this to GetMoreData without position.

2) It's not odd to pass raw data as char*. Note that V8 is responsible of
decoding it, and at this point it's just a byte stream. Other places  pass raw
byte data (which is possibly UTF-16) as char* too, for example the threaded HTML
parser: BackgroundHTMLParser::appendRawBytesFromParserThread

3) Moved the threading to Blink. This now returns a ScriptStreamingTask which
has a method Run(). (StartOnThread() would be implying that it just launches
something and returns, whereas Run() can really block and wait for data.)

+
+ protected:
+  /**
+   * The embedder should call this as soon as it knows the encoding. It must be
+   * called before the first call to GetSomeData returns.
+   */
+  void SetEncoding(Encoding e) { encoding = e; }
+
+ private:
+  friend class internal::ExternalStreamingStream;
+  Encoding encoding;
+};
+
+
+/**
  * A compiled JavaScript script, not yet tied to a Context.
  */
 class V8_EXPORT UnboundScript {
  public:
   /**
    * Binds the script to the currently entered context.
    */
   Local<Script> BindToCurrentContext();
 
   int GetId();
@@ skipping 51 matching lines @@
     return GetUnboundScript()->GetId();
   }
 };
 
 
 /**
  * For compiling scripts.
  */
 class V8_EXPORT ScriptCompiler {
  public:
+  class StreamedSource;
+  typedef void (*StreamingCompleteCallback)(StreamedSource*, void*);
+
   /**
    * Compilation data that the embedder can cache and pass back to speed up
    * future compilations. The data is produced if the CompilerOptions passed to
    * the compilation functions in ScriptCompiler contains produce_data_to_cache
    * = true. The data to cache can then can be retrieved from
    * UnboundScript.
    */
   struct V8_EXPORT CachedData {
     enum BufferPolicy {
       BufferNotOwned,
@@ skipping 51 matching lines @@
     Handle<Integer> resource_line_offset;
     Handle<Integer> resource_column_offset;
     Handle<Boolean> resource_is_shared_cross_origin;
 
     // Cached data from previous compilation (if a kConsume*Cache flag is
     // set), or hold newly generated cache data (kProduce*Cache flags) are
     // set when calling a compile method.
     CachedData* cached_data;
   };
 
+  /**
+   * Source code which can be streamed into V8 in pieces. It will be parsed
+   * while streaming, and compiled after the streaming is complete.
+   * StreamedSource must be kept alive while the compilation is ongoing.
+   */
+  class StreamedSource {
+   public:
+    // Source takes ownership of CachedData.
+    V8_INLINE StreamedSource(ExternalSourceStream* source_stream);

[jochen (gone - plz use gerrit), 2014/08/13 14:24:45]

explicit

[marja, 2014/08/14 11:29:21]

Done.

+    ~StreamedSource();
+
+    // Ownership of the CachedData or its buffers is *not* transferred to the
+    // caller. The CachedData object is alive as long as the Source object is

[jochen (gone - plz use gerrit), 2014/08/13 14:24:45]

you mean StreamedSource object?

[marja, 2014/08/14 11:29:22]

Done.

+    // alive.
+    V8_INLINE const CachedData* GetCachedData() const;
+
+   private:
+    friend class ScriptCompiler;
+    friend class internal::BackgroundParsingTask;
+    friend class internal::FinalizeParsingTask;
+
+    // Prevent copying. Not implemented.
+    StreamedSource(const StreamedSource&);
+    StreamedSource& operator=(const StreamedSource&);
+
+    ExternalSourceStream* source_stream;
+
+    // Cached data generated during compilation (if the generate_cached_data
+    // flag is passed to ScriptCompiler::StartStreamingScript). Streamed scripts
+    // cannot utilize already existing cached data.
+    CachedData* cached_data;
+
+    internal::CompilationInfo* info;
+  };
+
   enum CompileOptions {
     kNoCompileOptions = 0,
     kProduceParserCache,
     kConsumeParserCache,
     kProduceCodeCache,
     kConsumeCodeCache,
 
     // Support the previous API for a transition period.
     kProduceDataToCache
   };
 
   /**
    * Compiles the specified script (context-independent).
    * Cached data as part of the source object can be optionally produced to be
    * consumed later to speed up compilation of identical source scripts.
    *
    * Note that when producing cached data, the source must point to NULL for
    * cached data. When consuming cached data, the cached data must have been
    * produced by the same version of V8.
    *
    * \param source Script source code.
    * \return Compiled script object (context independent; for running it must be
    *   bound to a context).
    */
   static Local<UnboundScript> CompileUnbound(
       Isolate* isolate, Source* source,
       CompileOptions options = kNoCompileOptions);
 
   /**
+   * Starts streaming script data into V8. This API allows to start the
+   * streaming with as little data as possible, and the remaining data (for
+   * example, the ScriptOrigin) is passed to Compile (see below). The script is
+   * parsed while streaming. When all data has been streamed, the callback
+   * provided by the embedder is called, and the script can be compiled.
+   */
+  static bool StartStreamingScript(Isolate* isolate, StreamedSource* source,
+                                   StreamingCompleteCallback callback,
+                                   void* callback_data,
+                                   CompileOptions options = kNoCompileOptions);
+
+  /**
+   * Compiles a streamed script. This can only be called after the streaming has
+   * finished (the callback passed to StartStreamingScript has been called). V8
+   * doesn't construct the source string as it streams, so the embedder needs to
+   * pass the full source here.
+   */
+  static Local<Script> Compile(Isolate* isolate, StreamedSource* source,
+                               Handle<String> full_source_string,
+                               const ScriptOrigin& origin);
+
+  /**
    * Compiles the specified script (bound to current context).
    *
    * \param source Script source code.
    * \param pre_data Pre-parsing data, as obtained by ScriptData::PreCompile()
    *   using pre_data speeds compilation if it's done multiple times.
    *   Owned by caller, no references are kept when this function returns.
    * \return Compiled script object, bound to the context that was active
    *   when this function was called. When run it will always use this
    *   context.
    */
@@ skipping 5044 matching lines @@
   delete cached_data;
 }
 
 
 const ScriptCompiler::CachedData* ScriptCompiler::Source::GetCachedData()
     const {
   return cached_data;
 }
 
 
+ScriptCompiler::StreamedSource::StreamedSource(ExternalSourceStream* stream)
+    : source_stream(stream), cached_data(NULL), info(NULL) {}
+
+
+const ScriptCompiler::CachedData*
+ScriptCompiler::StreamedSource::GetCachedData() const {
+  return cached_data;
+}
+
+
 Handle<Boolean> Boolean::New(Isolate* isolate, bool value) {
   return value ? True(isolate) : False(isolate);
 }
 
 
 void Template::Set(Isolate* isolate, const char* name, v8::Handle<Data> value) {
   Set(v8::String::NewFromUtf8(isolate, name), value);
 }
 
 
@@ skipping 539 matching lines @@
  */
 
 
 }  // namespace v8
 
 
 #undef TYPE_CHECK
 
 
 #endif  // V8_H_