Sketch of on demand snapshot generation for d8

src/api.cc

Index: src/api.cc
diff --git a/src/api.cc b/src/api.cc
index cc0f70bf318c5f57fca3e49c5d5c9d7daaf9a3ff..5b97f8ff236f45e84d4a95120b9ec5da1554dc65 100644
--- a/src/api.cc
+++ b/src/api.cc
@@ -4,6 +4,15 @@
 
 #include "src/api.h"
 
+// Includes from mksnapshot.cc
+#include <errno.h>
+#include <stdio.h>
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+#include <bzlib.h>
+#endif
+#include <signal.h>
+// Includes from mksnapshot.cc
+
 #include <string.h>  // For memcpy, strlen.
 #ifdef V8_USE_ADDRESS_SANITIZER
 #include <sanitizer/asan_interface.h>
@@ -50,6 +59,19 @@
 #include "src/version.h"
 #include "src/vm-state-inl.h"
 
+// Includes from mksnapshot.cc
+#include "src/v8.h"
+
+#include "include/libplatform/libplatform.h"
+#include "src/assembler.h"
+#include "src/base/platform/platform.h"
+#include "src/bootstrapper.h"
+#include "src/flags.h"
+#include "src/list.h"
+#include "src/natives.h"
+#include "src/serialize.h"
+// Includes from mksnapshot.cc
+
 
 #define LOG_API(isolate, expr) LOG(isolate, ApiEntryCall(expr))
 
@@ -206,6 +228,523 @@ void V8::SetSnapshotDataBlob(StartupData* snapshot_blob) {
 }
 
 
+// Code needed to write the snapshot -- extracted from mksnapshot.cc
+class Compressor {
+ public:
+  virtual ~Compressor() {}
+  virtual bool Compress(i::Vector<i::byte> input) = 0;
+  virtual i::Vector<i::byte>* output() = 0;
+};
+
+class SnapshotWriter {
+ public:
+  explicit SnapshotWriter(const char* snapshot_file)
+      : fp_(GetFileDescriptorOrDie(snapshot_file))
+      , raw_file_(NULL)
+      , raw_context_file_(NULL)
+      , startup_blob_file_(NULL)
+      , compressor_(NULL) {
+  }
+
+  ~SnapshotWriter() {
+    fclose(fp_);
+    if (raw_file_) fclose(raw_file_);
+    if (raw_context_file_) fclose(raw_context_file_);
+    if (startup_blob_file_) fclose(startup_blob_file_);
+  }
+
+  void SetCompressor(Compressor* compressor) {
+    compressor_ = compressor;
+  }
+
+  void SetRawFiles(const char* raw_file, const char* raw_context_file) {
+    raw_file_ = GetFileDescriptorOrDie(raw_file);
+    raw_context_file_ = GetFileDescriptorOrDie(raw_context_file);
+  }
+
+  void SetStartupBlobFile(const char* startup_blob_file) {
+    if (startup_blob_file != NULL)
+      startup_blob_file_ = GetFileDescriptorOrDie(startup_blob_file);
+  }
+
+  void WriteSnapshot(const i::List<i::byte>& snapshot_data,
+                     const i::Serializer& serializer,
+                     const i::List<i::byte>& context_snapshot_data,
+                     const i::Serializer& context_serializer) const {
+    WriteSnapshotFile(snapshot_data, serializer,
+                      context_snapshot_data, context_serializer);
+    MaybeWriteStartupBlob(snapshot_data, serializer,
+                          context_snapshot_data, context_serializer);
+  }
+
+ private:
+  void MaybeWriteStartupBlob(const i::List<i::byte>& snapshot_data,
+                             const i::Serializer& serializer,
+                             const i::List<i::byte>& context_snapshot_data,
+                             const i::Serializer& context_serializer) const {
+    if (!startup_blob_file_) return;
+
+    i::SnapshotByteSink sink;
+
+    int spaces[] = {i::NEW_SPACE,           i::OLD_POINTER_SPACE,
+                    i::OLD_DATA_SPACE,      i::CODE_SPACE,
+                    i::MAP_SPACE,           i::CELL_SPACE,
+                    i::PROPERTY_CELL_SPACE, i::LO_SPACE};
+
+    i::byte* snapshot_bytes = snapshot_data.begin();
+    sink.PutBlob(snapshot_bytes, snapshot_data.length(), "snapshot");
+    for (size_t i = 0; i < arraysize(spaces); ++i) {
+      i::Vector<const uint32_t> chunks =
+          serializer.FinalAllocationChunks(spaces[i]);
+      // For the start-up snapshot, none of the reservations has more than
+      // one chunk (reservation for each space fits onto a single page).
+      CHECK_EQ(1, chunks.length());
+      sink.PutInt(chunks[0], "spaces");
+    }
+
+    i::byte* context_bytes = context_snapshot_data.begin();
+    sink.PutBlob(context_bytes, context_snapshot_data.length(), "context");
+    for (size_t i = 0; i < arraysize(spaces); ++i) {
+      i::Vector<const uint32_t> chunks =
+          context_serializer.FinalAllocationChunks(spaces[i]);
+      // For the context snapshot, none of the reservations has more than
+      // one chunk (reservation for each space fits onto a single page).
+      CHECK_EQ(1, chunks.length());
+      sink.PutInt(chunks[0], "spaces");
+    }
+
+    const i::List<i::byte>& startup_blob = sink.data();
+    size_t written = fwrite(startup_blob.begin(), 1, startup_blob.length(),
+                            startup_blob_file_);
+    if (written != static_cast<size_t>(startup_blob.length())) {
+      i::PrintF("Writing snapshot file failed.. Aborting.\n");
+      exit(1);
+    }
+  }
+
+  void WriteSnapshotFile(const i::List<i::byte>& snapshot_data,
+                         const i::Serializer& serializer,
+                         const i::List<i::byte>& context_snapshot_data,
+                         const i::Serializer& context_serializer) const {
+    WriteFilePrefix();
+    WriteData("", snapshot_data, raw_file_);
+    WriteData("context_", context_snapshot_data, raw_context_file_);
+    WriteMeta("context_", context_serializer);
+    WriteMeta("", serializer);
+    WriteFileSuffix();
+  }
+
+  void WriteFilePrefix() const {
+    fprintf(fp_, "// Autogenerated snapshot file. Do not edit.\n\n");
+    fprintf(fp_, "#include \"src/v8.h\"\n");
+    fprintf(fp_, "#include \"src/base/platform/platform.h\"\n\n");
+    fprintf(fp_, "#include \"src/snapshot.h\"\n\n");
+    fprintf(fp_, "namespace v8 {\n");
+    fprintf(fp_, "namespace internal {\n\n");
+  }
+
+  void WriteFileSuffix() const {
+    fprintf(fp_, "}  // namespace internal\n");
+    fprintf(fp_, "}  // namespace v8\n");
+  }
+
+  void WriteData(const char* prefix, const i::List<i::byte>& source_data,
+                 FILE* raw_file) const {
+    const i::List<i::byte>* data_to_be_written = NULL;
+    i::List<i::byte> compressed_data;
+    if (!compressor_) {
+      data_to_be_written = &source_data;
+    } else if (compressor_->Compress(source_data.ToVector())) {
+      compressed_data.AddAll(*compressor_->output());
+      data_to_be_written = &compressed_data;
+    } else {
+      i::PrintF("Compression failed. Aborting.\n");
+      exit(1);
+    }
+
+    DCHECK(data_to_be_written);
+    MaybeWriteRawFile(data_to_be_written, raw_file);
+    WriteData(prefix, source_data, data_to_be_written);
+  }
+
+  void MaybeWriteRawFile(const i::List<i::byte>* data, FILE* raw_file) const {
+    if (!data || !raw_file)
+      return;
+
+    // Sanity check, whether i::List iterators truly return pointers to an
+    // internal array.
+    DCHECK(data->end() - data->begin() == data->length());
+
+    size_t written = fwrite(data->begin(), 1, data->length(), raw_file);
+    if (written != (size_t)data->length()) {
+      i::PrintF("Writing raw file failed.. Aborting.\n");
+      exit(1);
+    }
+  }
+
+  void WriteData(const char* prefix, const i::List<i::byte>& source_data,
+                 const i::List<i::byte>* data_to_be_written) const {
+    fprintf(fp_, "const byte Snapshot::%sdata_[] = {\n", prefix);
+    WriteSnapshotData(data_to_be_written);
+    fprintf(fp_, "};\n");
+    fprintf(fp_, "const int Snapshot::%ssize_ = %d;\n", prefix,
+            data_to_be_written->length());
+
+    if (data_to_be_written == &source_data) {
+      fprintf(fp_, "const byte* Snapshot::%sraw_data_ = Snapshot::%sdata_;\n",
+              prefix, prefix);
+      fprintf(fp_, "const int Snapshot::%sraw_size_ = Snapshot::%ssize_;\n",
+              prefix, prefix);
+    } else {
+      fprintf(fp_, "const byte* Snapshot::%sraw_data_ = NULL;\n", prefix);
+      fprintf(fp_, "const int Snapshot::%sraw_size_ = %d;\n",
+              prefix, source_data.length());
+    }
+    fprintf(fp_, "\n");
+  }
+
+  void WriteMeta(const char* prefix, const i::Serializer& ser) const {
+    WriteSizeVar(ser, prefix, "new", i::NEW_SPACE);
+    WriteSizeVar(ser, prefix, "pointer", i::OLD_POINTER_SPACE);
+    WriteSizeVar(ser, prefix, "data", i::OLD_DATA_SPACE);
+    WriteSizeVar(ser, prefix, "code", i::CODE_SPACE);
+    WriteSizeVar(ser, prefix, "map", i::MAP_SPACE);
+    WriteSizeVar(ser, prefix, "cell", i::CELL_SPACE);
+    WriteSizeVar(ser, prefix, "property_cell", i::PROPERTY_CELL_SPACE);
+    WriteSizeVar(ser, prefix, "lo", i::LO_SPACE);
+    fprintf(fp_, "\n");
+  }
+
+  void WriteSizeVar(const i::Serializer& ser, const char* prefix,
+                    const char* name, int space) const {
+    i::Vector<const uint32_t> chunks = ser.FinalAllocationChunks(space);
+    // For the start-up snapshot, none of the reservations has more than
+    // one chunk (total reservation fits into a single page).
+    CHECK_EQ(1, chunks.length());
+    fprintf(fp_, "const int Snapshot::%s%s_space_used_ = %d;\n", prefix, name,
+            chunks[0]);
+  }
+
+  void WriteSnapshotData(const i::List<i::byte>* data) const {
+    for (int i = 0; i < data->length(); i++) {
+      if ((i & 0x1f) == 0x1f)
+        fprintf(fp_, "\n");
+      if (i > 0)
+        fprintf(fp_, ",");
+      fprintf(fp_, "%u", static_cast<unsigned char>(data->at(i)));
+    }
+    fprintf(fp_, "\n");
+  }
+
+  FILE* GetFileDescriptorOrDie(const char* filename) {
+    FILE* fp = base::OS::FOpen(filename, "wb");
+    if (fp == NULL) {
+      i::PrintF("Unable to open file \"%s\" for writing.\n", filename);
+      exit(1);
+    }
+    return fp;
+  }
+
+  FILE* fp_;
+  FILE* raw_file_;
+  FILE* raw_context_file_;
+  FILE* startup_blob_file_;
+  Compressor* compressor_;
+};
+
+
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+class BZip2Compressor : public Compressor {
+ public:
+  BZip2Compressor() : output_(NULL) {}
+  virtual ~BZip2Compressor() {
+    delete output_;
+  }
+  virtual bool Compress(i::Vector<char> input) {
+    delete output_;
+    output_ = new i::ScopedVector<char>((input.length() * 101) / 100 + 1000);
+    unsigned int output_length_ = output_->length();
+    int result = BZ2_bzBuffToBuffCompress(output_->start(), &output_length_,
+                                          input.start(), input.length(),
+                                          9, 1, 0);
+    if (result == BZ_OK) {
+      output_->Truncate(output_length_);
+      return true;
+    } else {
+      fprintf(stderr, "bzlib error code: %d\n", result);
+      return false;
+    }
+  }
+  virtual i::Vector<char>* output() { return output_; }
+
+ private:
+  i::ScopedVector<char>* output_;
+};
+
+
+class BZip2Decompressor : public StartupDataDecompressor {
+ public:
+  virtual ~BZip2Decompressor() { }
+
+ protected:
+  virtual int DecompressData(char* raw_data,
+                             int* raw_data_size,
+                             const char* compressed_data,
+                             int compressed_data_size) {
+    DCHECK_EQ(StartupData::kBZip2,
+              V8::GetCompressedStartupDataAlgorithm());
+    unsigned int decompressed_size = *raw_data_size;
+    int result =
+        BZ2_bzBuffToBuffDecompress(raw_data,
+                                   &decompressed_size,
+                                   const_cast<char*>(compressed_data),
+                                   compressed_data_size,
+                                   0, 1);
+    if (result == BZ_OK) {
+      *raw_data_size = decompressed_size;
+    }
+    return result;
+  }
+};
+#endif
+
+
+void DumpException(Handle<Message> message) {
+  String::Utf8Value message_string(message->Get());
+  String::Utf8Value message_line(message->GetSourceLine());
+  fprintf(stderr, "%s at line %d\n", *message_string, message->GetLineNumber());
+  fprintf(stderr, "%s\n", *message_line);
+  for (int i = 0; i <= message->GetEndColumn(); ++i) {
+    fprintf(stderr, "%c", i < message->GetStartColumn() ? ' ' : '^');
+  }
+  fprintf(stderr, "\n");
+}
+
+void SnapshotV8Heap(Isolate* isolate, const char* raw_file,
+    const char* raw_context_file, const char* startup_blob,
+    const char* extra_code, const char* out_file) {
+
+  Isolate::Scope isolate_scope(isolate);
+  i::Isolate* internal_isolate = reinterpret_cast<i::Isolate*>(isolate);
+
+  Persistent<Context> context;
+  {
+    HandleScope handle_scope(isolate);
+    context.Reset(isolate, Context::New(isolate));
+  }
+
+  if (context.IsEmpty()) {
+    fprintf(stderr,
+            "\nException thrown while compiling natives - see above.\n\n");
+    exit(1);
+  }
+
+  {
+  // Make sure all builtin scripts are cached.
+  { HandleScope scope(isolate);
+    for (int i = 0; i < i::Natives::GetBuiltinsCount(); i++) {
+      internal_isolate->bootstrapper()->NativesSourceLookup(i);
+    }
+  }
+  // If we don't do this then we end up with a stray root pointing at the
+  // context even after we have disposed of the context.
+  internal_isolate->heap()->CollectAllAvailableGarbage("mksnapshot");
+  i::Object* raw_context = *v8::Utils::OpenPersistent(context);
+  context.Reset();
+
+  // This results in a somewhat smaller snapshot, probably because it gets
+  // rid of some things that are cached between garbage collections.
+  i::SnapshotByteSink snapshot_sink;
+  i::StartupSerializer ser(internal_isolate, &snapshot_sink);
+  ser.SerializeStrongReferences();
+
+  i::SnapshotByteSink context_sink;
+  i::PartialSerializer context_ser(internal_isolate, &ser, &context_sink);
+  context_ser.Serialize(&raw_context);
+  ser.SerializeWeakReferences();
+
+  context_ser.FinalizeAllocation();
+  ser.FinalizeAllocation();
+
+  {
+    SnapshotWriter writer(out_file);
+    if (raw_file && raw_context_file)
+      writer.SetRawFiles(raw_file, raw_context_file);
+    if (startup_blob)
+      writer.SetStartupBlobFile(startup_blob);
+  #ifdef COMPRESS_STARTUP_DATA_BZ2
+      BZip2Compressor bzip2;
+      writer.SetCompressor(&bzip2);
+  #endif
+      writer.WriteSnapshot(snapshot_sink.data(), ser, context_sink.data(),
+                           context_ser);
+    }
+  }
+}
+
+
+// This is called from mksnapshot.cc to create the
+// snapshot during the build.
+void CreateV8SnapshotOnDemand(const char* raw_file,
+                              const char* raw_context_file,
+                              const char* startup_blob,
+                              const char* extra_code,
+                              const char* out_file) {
+  i::CpuFeatures::Probe(true);
+  V8::InitializeICU();
+  v8::Platform* platform = v8::platform::CreateDefaultPlatform();
+  v8::V8::InitializePlatform(platform);
+  v8::V8::Initialize();
+
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  BZip2Decompressor natives_decompressor;
+  int bz2_result = natives_decompressor.Decompress();
+  if (bz2_result != BZ_OK) {
+    fprintf(stderr, "bzip error code: %d\n", bz2_result);
+    exit(1);
+  }
+#endif
+
+  Isolate::CreateParams params;
+  params.enable_serializer = true;
+  Isolate* isolate = v8::Isolate::New(params);
+  { Isolate::Scope isolate_scope(isolate);
+    i::Isolate* internal_isolate = reinterpret_cast<i::Isolate*>(isolate);
+
+    Persistent<Context> context;
+    {
+      HandleScope handle_scope(isolate);
+      context.Reset(isolate, Context::New(isolate));
+    }
+
+    if (context.IsEmpty()) {
+      fprintf(stderr,
+              "\nException thrown while compiling natives - see above.\n\n");
+      exit(1);
+    }
+    if (extra_code != NULL) {
+      // Capture 100 frames if anything happens.
+      V8::SetCaptureStackTraceForUncaughtExceptions(true, 100);
+      HandleScope scope(isolate);
+      v8::Context::Scope cscope(v8::Local<v8::Context>::New(isolate, context));
+      const char* name = extra_code;
+      FILE* file = base::OS::FOpen(name, "rb");
+      if (file == NULL) {
+        fprintf(stderr, "Failed to open '%s': errno %d\n", name, errno);
+        exit(1);
+      }
+
+      fseek(file, 0, SEEK_END);
+      int size = ftell(file);
+      rewind(file);
+
+      char* chars = new char[size + 1];
+      chars[size] = '\0';
+      for (int i = 0; i < size;) {
+        int read = static_cast<int>(fread(&chars[i], 1, size - i, file));
+        if (read < 0) {
+          fprintf(stderr, "Failed to read '%s': errno %d\n", name, errno);
+          exit(1);
+        }
+        i += read;
+      }
+      fclose(file);
+      Local<String> source = String::NewFromUtf8(isolate, chars);
+      TryCatch try_catch;
+      Local<Script> script = Script::Compile(source);
+      if (try_catch.HasCaught()) {
+        fprintf(stderr, "Failure compiling '%s'\n", name);
+        DumpException(try_catch.Message());
+        exit(1);
+      }
+      script->Run();
+      if (try_catch.HasCaught()) {
+        fprintf(stderr, "Failure running '%s'\n", name);
+        DumpException(try_catch.Message());
+        exit(1);
+      }
+    }
+    // Make sure all builtin scripts are cached.
+    { HandleScope scope(isolate);
+      for (int i = 0; i < i::Natives::GetBuiltinsCount(); i++) {
+        internal_isolate->bootstrapper()->NativesSourceLookup(i);
+      }
+    }
+    // If we don't do this then we end up with a stray root pointing at the
+    // context even after we have disposed of the context.
+    internal_isolate->heap()->CollectAllAvailableGarbage("mksnapshot");
+    i::Object* raw_context = *v8::Utils::OpenPersistent(context);
+    context.Reset();
+
+    // This results in a somewhat smaller snapshot, probably because it gets
+    // rid of some things that are cached between garbage collections.
+    i::SnapshotByteSink snapshot_sink;
+    i::StartupSerializer ser(internal_isolate, &snapshot_sink);
+    ser.SerializeStrongReferences();
+
+    i::SnapshotByteSink context_sink;
+    i::PartialSerializer context_ser(internal_isolate, &ser, &context_sink);
+    context_ser.Serialize(&raw_context);
+    ser.SerializeWeakReferences();
+
+    context_ser.FinalizeAllocation();
+    ser.FinalizeAllocation();
+
+    {
+      SnapshotWriter writer(out_file);
+      if (raw_file && raw_context_file)
+        writer.SetRawFiles(raw_file, raw_context_file);
+      if (startup_blob)
+        writer.SetStartupBlobFile(startup_blob);
+  #ifdef COMPRESS_STARTUP_DATA_BZ2
+      BZip2Compressor bzip2;
+      writer.SetCompressor(&bzip2);
+  #endif
+      writer.WriteSnapshot(snapshot_sink.data(), ser, context_sink.data(),
+                           context_ser);
+    }
+  }
+
+  isolate->Dispose();
+  V8::Dispose();
+  V8::ShutdownPlatform();
+  delete platform;
+}
+
+
+void V8::SetFlags(bool harmony, int random_seed, bool probe) {
+  i::FLAG_harmony_shipping = harmony;
+  i::FLAG_random_seed = random_seed;
+  i::CpuFeatures::Probe(probe);
+}
+
+void V8::SetLogFilePerIsolate(bool logfile) {
+  i::FLAG_logfile_per_isolate = logfile;
+}
+
+void V8::CreateSnapshotFromHeap(Isolate* isolate, const char* raw_file,
+    const char* raw_context_file, const char* startup_blob,
+    const char* extra_code, const char* out_file) {
+#if defined(V8_ON_DEMAND_SNAPSHOT)
+  SnapshotV8Heap(isolate, raw_file, raw_context_file, startup_blob,
+      extra_code, out_file);
+#else
+  CHECK(false);
+#endif
+}
+
+void V8::CreateV8Snapshot(const char* raw_file, const char* raw_context_file,
+                          const char* startup_blob, const char* extra_code,
+                          const char* out_file) {
+#if defined(V8_ON_DEMAND_SNAPSHOT)
+  CreateV8SnapshotOnDemand(raw_file, raw_context_file, startup_blob,
+      extra_code, out_file);
+#else
+  CHECK(false);
+#endif
+}
+
+
 void V8::SetFlagsFromString(const char* str, int length) {
   i::FlagList::SetFlagsFromString(str, length);
 }