[standalone] Automatically decompress gzip'd resources, including sources and script snapshots.

runtime/bin/gzip.cc

Index: runtime/bin/gzip.cc
diff --git a/runtime/bin/gzip.cc b/runtime/bin/gzip.cc
new file mode 100644
index 0000000000000000000000000000000000000000..e6eb9c631925904ee7088ae766a6abd3f286a43c
--- /dev/null
+++ b/runtime/bin/gzip.cc
@@ -0,0 +1,142 @@
+// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+#include "bin/gzip.h"
+
+#include "platform/assert.h"
+#include "platform/globals.h"
+#include "zlib/zlib.h"
+
+namespace dart {
+namespace bin {
+
+void Decompress(const uint8_t* input,
+                intptr_t input_len,
+                uint8_t** output,
+                intptr_t* output_length) {
+  ASSERT(input != NULL);
+  ASSERT(input_len > 0);
+  ASSERT(output != NULL);
+  ASSERT(output_length != NULL);
+
+  // Initialize output.
+  *output = NULL;
+  *output_length = 0;
+
+  const unsigned int kChunkSize = 256 * 1024;
+  uint8_t chunk_out[kChunkSize];
+  z_stream strm;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+  strm.avail_in = 0;
+  strm.next_in = 0;
+  int ret = inflateInit2(&strm, 32 + MAX_WBITS);
+  ASSERT(ret == Z_OK);
+
+  unsigned int input_cursor = 0;
+  unsigned int output_cursor = 0;
+  do {
+    // Setup input.
+    unsigned int size_in = input_len - input_cursor;
+    if (size_in > kChunkSize) {
+      size_in = kChunkSize;
+    }
+    strm.avail_in = size_in;
+    strm.next_in = const_cast<uint8_t*>(&input[input_cursor]);
+
+    // Inflate until we've exhausted the current input chunk.
+    do {
+      // Setup output.
+      strm.avail_out = kChunkSize;
+      strm.next_out = &chunk_out[0];
+      // Inflate.
+      ret = inflate(&strm, Z_SYNC_FLUSH);
+      // We either hit the end of the stream or made forward progress.
+      ASSERT((ret == Z_STREAM_END) || (ret == Z_OK));
+      // Grow output buffer size.
+      unsigned int size_out = kChunkSize - strm.avail_out;
+      *output_length += size_out;
+      *output = reinterpret_cast<uint8_t*>(realloc(*output, *output_length));

[zra, 2017/08/04 05:17:00]

This is probably fine when there are only a couple of chunks, but after that it
will be noticeable that this is n^2. 1MB of output is going to hit this 3 times,
and copy 2560KB bytes in addition to the 1MB of copies from chunk_out to
output[output_cursor]. I suspect that this is a good place to look for savings
in the decompression time.

[rmacnak, 2017/08/10 21:09:09]

Switched to exponential growth, but it doesn't have a noticeable impact on
decompression time. At least for dart2js's ~3MB script snapshot.

before
realloc 262144
realloc 524288
realloc 786432
realloc 786543
realloc 1048687
realloc 1310831
realloc 1455383
realloc 1717527
realloc 1979671
realloc 2128929
realloc 2391073
realloc 2653217
realloc 2861378
realloc 3123522
realloc 3385666
realloc 3507915

after
realloc 5208116

+      // Copy output.
+      memmove(&((*output)[output_cursor]), &chunk_out[0], size_out);
+      output_cursor += size_out;
+    } while (strm.avail_out == 0);
+
+    // We've processed size_in bytes.
+    input_cursor += size_in;
+
+    // We're finished decompressing when zlib tells us.
+  } while (ret != Z_STREAM_END);
+
+  inflateEnd(&strm);
+}
+
+void Compress(const uint8_t* input,

[zra, 2017/08/04 05:17:00]

Is this used anywhere?

[rmacnak, 2017/08/10 21:09:09]

Not anymore, removed.

+              intptr_t input_len,
+              uint8_t** output,
+              intptr_t* output_length) {
+  ASSERT(input != NULL);
+  ASSERT(input_len > 0);
+  ASSERT(output != NULL);
+  ASSERT(output_length != NULL);
+
+  // Initialize output.
+  *output = NULL;
+  *output_length = 0;
+
+  const unsigned int kChunkSize = 256 * 1024;
+  uint8_t chunk_out[kChunkSize];
+  z_stream strm;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+  strm.avail_in = 0;
+  strm.next_in = 0;
+
+  int ret = deflateInit(&strm, Z_DEFAULT_COMPRESSION);
+  ASSERT(ret == Z_OK);
+
+  unsigned int input_cursor = 0;
+  unsigned int output_cursor = 0;
+  do {
+    // Setup input.
+    unsigned int size_in = input_len - input_cursor;
+    bool end = true;
+    if (size_in > kChunkSize) {
+      size_in = kChunkSize;
+      end = false;
+    }
+    strm.avail_in = size_in;
+    strm.next_in = const_cast<uint8_t*>(&input[input_cursor]);
+
+    // Deflate until we've exhausted the current input chunk.
+    do {
+      // Setup output.
+      strm.avail_out = kChunkSize;
+      strm.next_out = &chunk_out[0];
+      // Deflate.
+      ret = deflate(&strm, end ? Z_FINISH : Z_NO_FLUSH);
+      // We either hit the end of the stream or made forward progress.
+      ASSERT((ret == Z_STREAM_END) || (ret == Z_BUF_ERROR) || (ret == Z_OK));
+      // Grow output buffer size.
+      unsigned int size_out = kChunkSize - strm.avail_out;
+      *output_length += size_out;
+      *output = reinterpret_cast<uint8_t*>(realloc(*output, *output_length));
+      // Copy output.
+      memmove(&((*output)[output_cursor]), &chunk_out[0], size_out);
+      output_cursor += size_out;
+    } while (strm.avail_out == 0);
+
+    // We've processed size_in bytes.
+    input_cursor += size_in;
+
+    // We're finished decompressing when zlib tells us.
+  } while (ret != Z_STREAM_END);
+
+  deflateEnd(&strm);
+}
+
+}  // namespace bin
+}  // namespace dart