Add support for startup data (snapshot) compression.

src/mksnapshot.cc

 // Copyright 2006-2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
 //       with the distribution.
 //     * Neither the name of Google Inc. nor the names of its
 //       contributors may be used to endorse or promote products derived
 //       from this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+#include <bzlib.h>
+#endif
 #include <signal.h>
 #include <string>
 #include <map>
 
 #include "v8.h"
 
 #include "bootstrapper.h"
 #include "natives.h"
 #include "platform.h"
 #include "serialize.h"
@@ skipping 50 matching lines @@
 // We statically allocate a set of local counters to be used if we
 // don't want to store the stats in a memory-mapped file
 static CounterCollection local_counters;
 
 
 typedef std::map<std::string, int*> CounterMap;
 typedef std::map<std::string, int*>::iterator CounterMapIterator;
 static CounterMap counter_table_;
 
 
-class CppByteSink : public i::SnapshotByteSink {
+class Compressor {
  public:
-  explicit CppByteSink(const char* snapshot_file)
-      : bytes_written_(0),
-        partial_sink_(this) {
+  virtual ~Compressor() {}
+  virtual bool Compress(i::Vector<char> input) = 0;
+  virtual i::Vector<char>* output() = 0;
+};
+
+
+class PartialSnapshotSink : public i::SnapshotByteSink {
+ public:
+  PartialSnapshotSink() : data_(), decompressed_size_(-1) { }
+  virtual ~PartialSnapshotSink() { data_.Free(); }
+  virtual void Put(int byte, const char* description) {
+    data_.Add(byte);
+  }
+  virtual int Position() { return data_.length(); }
+  void Print(FILE* fp) {
+    int length = Position();
+    for (int j = 0; j < length; j++) {
+      if ((j & 0x1f) == 0x1f) {
+        fprintf(fp, "\n");
+      }
+      if (j != 0) {
+        fprintf(fp, ",");
+      }
+      fprintf(fp, "%d", at(j));
+    }
+  }
+  char at(int i) { return data_[i]; }
+  bool Compress(Compressor* compressor) {

[Søren Thygesen Gjesse, 2011/04/29 06:50:29]

ASSERT_EQ(-1, decompressed_size_)?

[mnaganov (inactive), 2011/04/29 12:07:58]

Done.

+    decompressed_size_ = data_.length();
+    if (!compressor->Compress(data_.ToVector())) return false;
+    data_.Clear();
+    data_.AddAll(*compressor->output());
+    return false;
+  }
+  int decompressed_size() { return decompressed_size_; }
+ private:
+  i::List<char> data_;
+  int decompressed_size_;
+};
+
+
+class CppByteSink : public PartialSnapshotSink {
+ public:
+  explicit CppByteSink(const char* snapshot_file) {
     fp_ = i::OS::FOpen(snapshot_file, "wb");
     if (fp_ == NULL) {
       i::PrintF("Unable to write to snapshot file \"%s\"\n", snapshot_file);
       exit(1);
     }
     fprintf(fp_, "// Autogenerated snapshot file. Do not edit.\n\n");
     fprintf(fp_, "#include \"v8.h\"\n");
     fprintf(fp_, "#include \"platform.h\"\n\n");
     fprintf(fp_, "#include \"snapshot.h\"\n\n");
     fprintf(fp_, "namespace v8 {\nnamespace internal {\n\n");
     fprintf(fp_, "const byte Snapshot::data_[] = {");
   }
 
   virtual ~CppByteSink() {
-    fprintf(fp_, "const int Snapshot::size_ = %d;\n\n", bytes_written_);
+    fprintf(fp_, "const int Snapshot::size_ = %d;\n", Position());
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+    fprintf(fp_, "const byte* Snapshot::decompressed_data_ = NULL;\n");
+    fprintf(fp_,
+            "const int Snapshot::decompressed_size_ = %d;\n\n",
+            decompressed_size());
+#else
+    fprintf(fp_,
+            "const byte* Snapshot::decompressed_data_ = Snapshot::data_;\n");
+    fprintf(fp_,
+            "const int Snapshot::decompressed_size_ = Snapshot::size_;\n\n");
+#endif
     fprintf(fp_, "} }  // namespace v8::internal\n");
     fclose(fp_);
   }
 
   void WriteSpaceUsed(
       int new_space_used,
       int pointer_space_used,
       int data_space_used,
       int code_space_used,
       int map_space_used,
       int cell_space_used,
       int large_space_used) {
-    fprintf(fp_, "};\n\n");
     fprintf(fp_, "const int Snapshot::new_space_used_ = %d;\n", new_space_used);
     fprintf(fp_,
             "const int Snapshot::pointer_space_used_ = %d;\n",
             pointer_space_used);
     fprintf(fp_,
             "const int Snapshot::data_space_used_ = %d;\n",
             data_space_used);
     fprintf(fp_,
             "const int Snapshot::code_space_used_ = %d;\n",
             code_space_used);
     fprintf(fp_, "const int Snapshot::map_space_used_ = %d;\n", map_space_used);
     fprintf(fp_,
             "const int Snapshot::cell_space_used_ = %d;\n",
             cell_space_used);
     fprintf(fp_,
             "const int Snapshot::large_space_used_ = %d;\n",
             large_space_used);
   }
 
   void WritePartialSnapshot() {
     int length = partial_sink_.Position();
     fprintf(fp_, "};\n\n");
     fprintf(fp_, "const int Snapshot::context_size_ = %d;\n",  length);
+    fprintf(fp_,
+            "const int Snapshot::context_decompressed_size_ = %d;\n",
+            partial_sink_.decompressed_size());
     fprintf(fp_, "const byte Snapshot::context_data_[] = {\n");
-    for (int j = 0; j < length; j++) {
-      if ((j & 0x1f) == 0x1f) {
-        fprintf(fp_, "\n");
-      }
-      char byte = partial_sink_.at(j);
-      if (j != 0) {
-        fprintf(fp_, ",");
-      }
-      fprintf(fp_, "%d", byte);
-    }
+    partial_sink_.Print(fp_);
+    fprintf(fp_, "};\n\n");
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+    fprintf(fp_, "const byte* Snapshot::context_decompressed_data_ = NULL;\n");
+#else
+    fprintf(fp_, "const byte* Snapshot::context_decompressed_data_ ="
+            " Snapshot::context_data_;\n");
+#endif
   }
 
-  virtual void Put(int byte, const char* description) {
-    if (bytes_written_ != 0) {
-      fprintf(fp_, ",");
-    }
-    fprintf(fp_, "%d", byte);
-    bytes_written_++;
-    if ((bytes_written_ & 0x1f) == 0) {
-      fprintf(fp_, "\n");
-    }
+  void WriteSnapshot() {
+    Print(fp_);
   }
 
-  virtual int Position() {
-    return bytes_written_;
-  }
-
-  i::SnapshotByteSink* partial_sink() { return &partial_sink_; }
-
-  class PartialSnapshotSink : public i::SnapshotByteSink {
-   public:
-    explicit PartialSnapshotSink(CppByteSink* parent)
-        : parent_(parent),
-          data_() { }
-    virtual ~PartialSnapshotSink() { data_.Free(); }
-    virtual void Put(int byte, const char* description) {
-      data_.Add(byte);
-    }
-    virtual int Position() { return data_.length(); }
-    char at(int i) { return data_[i]; }
-   private:
-    CppByteSink* parent_;
-    i::List<char> data_;
-  };
+  PartialSnapshotSink* partial_sink() { return &partial_sink_; }
 
  private:
   FILE* fp_;
-  int bytes_written_;
   PartialSnapshotSink partial_sink_;
 };
 
 
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+class BZip2Compressor : public Compressor {
+ public:
+  BZip2Compressor() : output_(NULL) {}
+  virtual ~BZip2Compressor() {
+    delete output_;
+  }
+  virtual bool Compress(i::Vector<char> input) {
+    output_ = new i::ScopedVector<char>((input.length() * 101) / 100 + 1000);
+    unsigned int output_length_;
+    int result = BZ2_bzBuffToBuffCompress(output_->start(), &output_length_,
+                                          input.start(), input.length(),
+                                          9, 1, 0);
+    if (result == BZ_OK) {
+      output_->Truncate(output_length_);
+      return true;
+    } else {
+      fprintf(stderr, "bzlib error code: %d\n", result);
+      return false;
+    }
+  }
+  virtual i::Vector<char>* output() { return output_; }
+
+ private:
+  i::ScopedVector<char>* output_;
+};
+#endif
+
+
 int main(int argc, char** argv) {
 #ifdef ENABLE_LOGGING_AND_PROFILING
   // By default, log code create information in the snapshot.
   i::FLAG_log_code = true;
 #endif
   // Print the usage if an error occurs when parsing the command line
   // flags or if the help flag is set.
   int result = i::FlagList::SetFlagsFromCommandLine(&argc, argv, true);
   if (result > 0 || argc != 2 || i::FLAG_help) {
     ::printf("Usage: %s [flag] ... outfile\n", argv[0]);
@@ skipping 18 matching lines @@
   // This results in a somewhat smaller snapshot, probably because it gets rid
   // of some things that are cached between garbage collections.
   i::StartupSerializer ser(&sink);
   ser.SerializeStrongReferences();
 
   i::PartialSerializer partial_ser(&ser, sink.partial_sink());
   partial_ser.Serialize(&raw_context);
 
   ser.SerializeWeakReferences();
 
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  BZip2Compressor compressor;
+  sink.Compress(&compressor);
+  sink.partial_sink()->Compress(&compressor);
+#endif
+  sink.WriteSnapshot();
   sink.WritePartialSnapshot();
 
   sink.WriteSpaceUsed(
       partial_ser.CurrentAllocationAddress(i::NEW_SPACE),
       partial_ser.CurrentAllocationAddress(i::OLD_POINTER_SPACE),
       partial_ser.CurrentAllocationAddress(i::OLD_DATA_SPACE),
       partial_ser.CurrentAllocationAddress(i::CODE_SPACE),
       partial_ser.CurrentAllocationAddress(i::MAP_SPACE),
       partial_ser.CurrentAllocationAddress(i::CELL_SPACE),
       partial_ser.CurrentAllocationAddress(i::LO_SPACE));
   return 0;
 }