[ON HOLD] Implement pull-based design for content decoding

net/filter/gzip_source_stream.cc

Index: net/filter/gzip_source_stream.cc
diff --git a/net/filter/gzip_source_stream.cc b/net/filter/gzip_source_stream.cc
new file mode 100644
index 0000000000000000000000000000000000000000..1a6c5db32b4c66222114f608bc3970894a3a0cc6
--- /dev/null
+++ b/net/filter/gzip_source_stream.cc
@@ -0,0 +1,237 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/filter/gzip_source_stream.h"
+
+#include "base/bind.h"
+#include "base/bit_cast.h"
+#include "base/logging.h"
+#include "third_party/zlib/zlib.h"
+
+namespace net {
+
+namespace {
+
+const char kDeflate[] = "DEFLATE";
+const char kGzip[] = "GZIP";
+const char kGzipFallback[] = "GZIP_FALLBACK";
+
+}  // namespace
+
+GzipSourceStream::~GzipSourceStream() {
+  if (zlib_stream_)
+    inflateEnd(zlib_stream_.get());
+}
+
+std::unique_ptr<GzipSourceStream> GzipSourceStream::Create(
+    std::unique_ptr<SourceStream> previous,
+    GzipSourceStreamMode mode) {
+  std::unique_ptr<GzipSourceStream> source(
+      new GzipSourceStream(std::move(previous), mode));
+
+  if (!source->Init()) {
+    return nullptr;
+  }

[mmenke, 2016/08/01 21:55:30]

nit:  Remove braces.

[xunjieli, 2016/08/02 13:44:32]

Done.

+  return source;
+}
+
+GzipSourceStream::GzipSourceStream(std::unique_ptr<SourceStream> previous,
+                                   GzipSourceStreamMode mode)
+    : FilterSourceStream(SourceStream::TYPE_GZIP, std::move(previous)),
+      mode_(mode),
+      zlib_eof_(false),
+      zlib_header_added_(false),
+      should_check_gzip_header_(true),
+      gzip_footer_bytes_left_(0) {}
+
+bool GzipSourceStream::Init() {
+  zlib_stream_.reset(new z_stream);
+  if (!zlib_stream_)
+    return false;
+  memset(zlib_stream_.get(), 0, sizeof(z_stream));
+
+  if (mode_ == GZIP_SOURCE_STREAM_GZIP ||
+      mode_ == GZIP_SOURCE_STREAM_GZIP_WITH_FALLBACK) {
+    if (inflateInit2(zlib_stream_.get(), -MAX_WBITS) != Z_OK)
+      return false;
+  } else {
+    should_check_gzip_header_ = false;
+    if (inflateInit(zlib_stream_.get()) != Z_OK)
+      return false;
+  }
+  return true;
+}
+
+std::string GzipSourceStream::GetTypeAsString() const {
+  switch (type()) {
+    case TYPE_GZIP:
+      return kGzip;
+    case TYPE_GZIP_FALLBACK:
+      return kGzipFallback;
+    case TYPE_DEFLATE:
+      return kDeflate;
+    default:
+      NOTREACHED();
+      return "";
+  }
+}
+
+int GzipSourceStream::FilterData(IOBuffer* output_buffer,
+                                 size_t output_buffer_size,
+                                 DrainableIOBuffer* input_buffer) {
+  // If this stream is not really gzipped as detected by
+  // ShouldFallbackToPlain, pretend the zlib stream already ended.
+  if (ShouldFallbackToPlain(input_buffer)) {
+    zlib_eof_ = true;
+    should_check_gzip_header_ = false;
+  }
+
+  // Require a valid gzip header when decompressing a gzip stream.
+  if (should_check_gzip_header_ && IsGzipHeaderInvalid(input_buffer))
+    return ERR_CONTENT_DECODING_FAILED;
+
+  size_t bytes_read =
+      Decompress(output_buffer, output_buffer_size, input_buffer);
+
+  // If there was already some data buffered internally in |buffer_|,
+  // or some output buffered internally in zlib, |Decompress| can succeed
+  // synchronously. If this happens, return right here.
+  if (bytes_read > 0)
+    return bytes_read;
+
+  // Since Decompress needs more input, it has consumed all existing input.
+  DCHECK_EQ(0, input_buffer->BytesRemaining());
+
+  return bytes_read;
+}
+
+int GzipSourceStream::Decompress(IOBuffer* output_buffer,
+                                 size_t output_buffer_size,
+                                 DrainableIOBuffer* input_buffer) {
+  DCHECK(output_buffer);
+  DCHECK_NE(0u, output_buffer_size);
+
+  if (input_buffer->BytesRemaining() == 0)
+    return 0;
+
+  // If the zlib stream has already ended, pass any further data through.
+  if (zlib_eof_)
+    return Passthrough(output_buffer->data(), output_buffer_size, input_buffer);
+  zlib_stream_.get()->next_in = bit_cast<Bytef*>(input_buffer->data());
+  zlib_stream_.get()->avail_in = input_buffer->BytesRemaining();
+  zlib_stream_.get()->next_out = bit_cast<Bytef*>(output_buffer->data());
+  zlib_stream_.get()->avail_out = output_buffer_size;
+
+  int ret = inflate(zlib_stream_.get(), Z_NO_FLUSH);
+
+  // Sometime misconfigured servers omit the zlib header, relying on clients
+  // to splice it back in.
+  if (ret < 0 && !zlib_header_added_) {
+    zlib_header_added_ = true;
+    if (!InsertZlibHeader())
+      return ERR_CONTENT_DECODING_FAILED;
+
+    zlib_stream_.get()->next_in = bit_cast<Bytef*>(input_buffer->data());
+    zlib_stream_.get()->avail_in = input_buffer->BytesRemaining();
+    zlib_stream_.get()->next_out = bit_cast<Bytef*>(output_buffer->data());
+    zlib_stream_.get()->avail_out = output_buffer_size;
+
+    ret = inflate(zlib_stream_.get(), Z_NO_FLUSH);
+    // TODO(xunjieli): add a histogram to see how often this happens. The
+    // original bug for this behavior was ancient and maybe it doesn't happen
+    // in the wild any more?
+  }
+
+  size_t bytes_used =
+      input_buffer->BytesRemaining() - zlib_stream_.get()->avail_in;
+  size_t bytes_out = output_buffer_size - zlib_stream_.get()->avail_out;
+
+  input_buffer->DidConsume(bytes_used);
+
+  if (ret != Z_STREAM_END && ret != Z_OK)
+    return ERR_CONTENT_DECODING_FAILED;
+
+  // The zlib stream can end before the input stream ends. If this happens,
+  // |Decompress| will pass any further data on untouched.
+  if (ret == Z_STREAM_END) {
+    zlib_eof_ = true;
+    return bytes_out + Passthrough(output_buffer->data() + bytes_out,
+                                   output_buffer_size - bytes_out,
+                                   input_buffer);
+  }
+  return bytes_out;
+}
+
+size_t GzipSourceStream::Passthrough(char* output_buffer,
+                                     size_t output_buffer_size,
+                                     DrainableIOBuffer* input_buffer) {
+  SkipGzipFooterIfNeeded(input_buffer);
+  size_t to_copy = input_buffer->BytesRemaining();
+  if (to_copy > output_buffer_size)
+    to_copy = output_buffer_size;
+  memcpy(output_buffer, input_buffer->data(), to_copy);
+  input_buffer->DidConsume(to_copy);
+  return to_copy;
+}
+
+bool GzipSourceStream::InsertZlibHeader() {
+  char dummy_header[] = {0x78, 0x01};
+  char dummy_output[4];
+
+  inflateReset(zlib_stream_.get());
+  zlib_stream_.get()->next_in = bit_cast<Bytef*>(&dummy_header[0]);
+  zlib_stream_.get()->avail_in = sizeof(dummy_header);
+  zlib_stream_.get()->next_out = bit_cast<Bytef*>(&dummy_output[0]);
+  zlib_stream_.get()->avail_out = sizeof(dummy_output);
+
+  int ret = inflate(zlib_stream_.get(), Z_NO_FLUSH);
+  return ret == Z_OK;
+}
+
+bool GzipSourceStream::IsGzipHeaderInvalid(DrainableIOBuffer* input_buffer) {
+  const size_t kGzipFooterBytes = 8;
+  const char* end = nullptr;
+  GZipHeader::Status status = gzip_header_.ReadMore(
+      input_buffer->data(), input_buffer->BytesRemaining(), &end);
+  if (status == GZipHeader::INCOMPLETE_HEADER) {
+    input_buffer->DidConsume(input_buffer->BytesRemaining());
+    return false;
+  }
+
+  should_check_gzip_header_ = false;
+  if (status == GZipHeader::COMPLETE_HEADER) {
+    // If there is a valid header, there should also be a valid footer.
+    gzip_footer_bytes_left_ = kGzipFooterBytes;
+    input_buffer->DidConsume(end - input_buffer->data());
+  }
+
+  return status == GZipHeader::INVALID_HEADER;
+}
+
+// Dumb heuristic. Gzip files always start with a two-byte magic value per RFC
+// 1952 2.3.1, so if the first byte isn't the first byte of the gzip magic, and
+// this filter is checking whether it should fallback, then fallback.
+bool GzipSourceStream::ShouldFallbackToPlain(DrainableIOBuffer* input_buffer) {
+  static const char kGzipFirstByte = 0x1f;
+  if (mode_ != GZIP_SOURCE_STREAM_GZIP_WITH_FALLBACK)
+    return false;
+  if (!should_check_gzip_header_)
+    return false;
+  if (input_buffer->BytesRemaining() == 0)
+    return false;
+  char d = input_buffer->data()[0];

[mmenke, 2016/08/01 21:55:30]

nit:  Prefer meaningful variable names.  first_byte?  Could even just remove the
variable.

[xunjieli, 2016/08/02 13:44:32]

Done.

+  return d != kGzipFirstByte;
+}
+
+void GzipSourceStream::SkipGzipFooterIfNeeded(DrainableIOBuffer* input_buffer) {
+  if (gzip_footer_bytes_left_ == 0)
+    return;
+  size_t to_read = gzip_footer_bytes_left_;
+  if (to_read > base::checked_cast<size_t>(input_buffer->BytesRemaining()))
+    to_read = input_buffer->BytesRemaining();
+  input_buffer->DidConsume(to_read);
+  gzip_footer_bytes_left_ -= to_read;
+}
+
+}  // namespace net