Add net::GzipSourceStream

net/filter/gzip_source_stream.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/filter/gzip_source_stream.h"
+
+#include "base/bind.h"
+#include "base/bit_cast.h"
+#include "base/logging.h"
+#include "net/base/io_buffer.h"
+#include "third_party/zlib/zlib.h"
+
+namespace net {
+
+namespace {
+
+const char kDeflate[] = "DEFLATE";
+const char kGzip[] = "GZIP";
+const char kGzipFallback[] = "GZIP_FALLBACK";
+
+}  // namespace
+
+GzipSourceStream::~GzipSourceStream() {
+  if (zlib_stream_)
+    inflateEnd(zlib_stream_.get());
+}
+
+std::unique_ptr<GzipSourceStream> GzipSourceStream::Create(
+    std::unique_ptr<SourceStream> previous,
+    GzipSourceStreamMode mode) {
+  std::unique_ptr<GzipSourceStream> source(
+      new GzipSourceStream(std::move(previous), mode));
+
+  if (!source->Init())
+    return nullptr;
+  return source;
+}
+
+GzipSourceStream::GzipSourceStream(std::unique_ptr<SourceStream> previous,
+                                   GzipSourceStreamMode mode)
+    : FilterSourceStream(SourceStream::TYPE_GZIP, std::move(previous)),
+      mode_(mode),
+      zlib_eof_(false),
+      zlib_header_added_(false),
+      should_check_gzip_header_(true),
+      gzip_footer_bytes_left_(0) {}
+
+bool GzipSourceStream::Init() {
+  zlib_stream_.reset(new z_stream);
+  if (!zlib_stream_)
+    return false;
+  memset(zlib_stream_.get(), 0, sizeof(z_stream));
+
+  if (mode_ == GZIP_SOURCE_STREAM_GZIP ||
+      mode_ == GZIP_SOURCE_STREAM_GZIP_WITH_FALLBACK) {
+    if (inflateInit2(zlib_stream_.get(), -MAX_WBITS) != Z_OK)
+      return false;
+  } else {
+    should_check_gzip_header_ = false;
+    if (inflateInit(zlib_stream_.get()) != Z_OK)
+      return false;
+  }
+  return true;
+}
+
+std::string GzipSourceStream::GetTypeAsString() const {
+  switch (type()) {
+    case TYPE_GZIP:
+      return kGzip;
+    case TYPE_GZIP_FALLBACK:
+      return kGzipFallback;
+    case TYPE_DEFLATE:
+      return kDeflate;
+    default:
+      NOTREACHED();
+      return "";
+  }
+}
+
+int GzipSourceStream::FilterData(IOBuffer* output_buffer,
+                                 int output_buffer_size,
+                                 DrainableIOBuffer* input_buffer) {
+  // If this stream is not really gzipped as detected by
+  // ShouldFallbackToPlain, pretend the zlib stream already ended.
+  if (ShouldFallbackToPlain(input_buffer)) {
+    zlib_eof_ = true;
+    should_check_gzip_header_ = false;
+  }
+
+  // Require a valid gzip header when decompressing a gzip stream.
+  if (should_check_gzip_header_ && IsGzipHeaderInvalid(input_buffer))
+    return ERR_CONTENT_DECODING_FAILED;
+
+  size_t bytes_read =
+      Decompress(output_buffer, output_buffer_size, input_buffer);
+
+  // If there was already some data buffered internally in |buffer_|,
+  // or some output buffered internally in zlib, |Decompress| can succeed
+  // synchronously. If this happens, return right here.

[Randy Smith (Not in Mondays), 2016/09/12 20:54:00]

Couldn't everything after line 97 here be replaced by something like:

DCHECK(bytes_read != 0 || 0 == input_buffer->BytesRemaining());
return bytes_read;

?

[xunjieli, 2016/09/14 16:44:01]

Done.

+  if (bytes_read > 0)
+    return bytes_read;
+
+  // Since Decompress needs more input, it has consumed all existing input.
+  DCHECK_EQ(0, input_buffer->BytesRemaining());

[Randy Smith (Not in Mondays), 2016/09/12 20:54:00]

This won't necessarily be true if Decompress() returns an error?

[xunjieli, 2016/09/14 16:44:01]

Done. Yes, we might not drain all bytes if there is an error. I have changed the
assertion according to your earlier suggestion which should take care of the
error case. Thanks for catching that.

+
+  return bytes_read;
+}
+
+int GzipSourceStream::Decompress(IOBuffer* output_buffer,
+                                 size_t output_buffer_size,
+                                 DrainableIOBuffer* input_buffer) {
+  DCHECK(output_buffer);
+  DCHECK_NE(0u, output_buffer_size);
+
+  if (input_buffer->BytesRemaining() == 0)
+    return 0;
+
+  // If the zlib stream has already ended, pass any further data through.
+  if (zlib_eof_)
+    return Passthrough(output_buffer->data(), output_buffer_size, input_buffer);
+  zlib_stream_.get()->next_in = bit_cast<Bytef*>(input_buffer->data());
+  zlib_stream_.get()->avail_in = input_buffer->BytesRemaining();
+  zlib_stream_.get()->next_out = bit_cast<Bytef*>(output_buffer->data());
+  zlib_stream_.get()->avail_out = output_buffer_size;
+
+  int ret = inflate(zlib_stream_.get(), Z_NO_FLUSH);
+
+  // Sometime misconfigured servers omit the zlib header, relying on clients
+  // to splice it back in.
+  if (ret < 0 && !zlib_header_added_) {
+    zlib_header_added_ = true;
+    if (!InsertZlibHeader())
+      return ERR_CONTENT_DECODING_FAILED;
+
+    zlib_stream_.get()->next_in = bit_cast<Bytef*>(input_buffer->data());
+    zlib_stream_.get()->avail_in = input_buffer->BytesRemaining();
+    zlib_stream_.get()->next_out = bit_cast<Bytef*>(output_buffer->data());
+    zlib_stream_.get()->avail_out = output_buffer_size;
+
+    ret = inflate(zlib_stream_.get(), Z_NO_FLUSH);
+    // TODO(xunjieli): add a histogram to see how often this happens. The
+    // original bug for this behavior was ancient and maybe it doesn't happen
+    // in the wild any more?
+  }
+
+  size_t bytes_used =
+      input_buffer->BytesRemaining() - zlib_stream_.get()->avail_in;
+  size_t bytes_out = output_buffer_size - zlib_stream_.get()->avail_out;
+
+  input_buffer->DidConsume(bytes_used);
+
+  if (ret != Z_STREAM_END && ret != Z_OK)
+    return ERR_CONTENT_DECODING_FAILED;
+
+  // The zlib stream can end before the input stream ends. If this happens,
+  // |Decompress| will pass any further data on untouched.
+  if (ret == Z_STREAM_END) {
+    zlib_eof_ = true;
+    return bytes_out + Passthrough(output_buffer->data() + bytes_out,
+                                   output_buffer_size - bytes_out,
+                                   input_buffer);
+  }
+  return bytes_out;
+}
+
+size_t GzipSourceStream::Passthrough(char* output_buffer,
+                                     size_t output_buffer_size,
+                                     DrainableIOBuffer* input_buffer) {
+  SkipGzipFooterIfNeeded(input_buffer);
+  size_t to_copy = input_buffer->BytesRemaining();
+  if (to_copy > output_buffer_size)
+    to_copy = output_buffer_size;
+  memcpy(output_buffer, input_buffer->data(), to_copy);
+  input_buffer->DidConsume(to_copy);
+  return to_copy;
+}
+
+bool GzipSourceStream::InsertZlibHeader() {
+  char dummy_header[] = {0x78, 0x01};
+  char dummy_output[4];
+
+  inflateReset(zlib_stream_.get());
+  zlib_stream_.get()->next_in = bit_cast<Bytef*>(&dummy_header[0]);
+  zlib_stream_.get()->avail_in = sizeof(dummy_header);
+  zlib_stream_.get()->next_out = bit_cast<Bytef*>(&dummy_output[0]);
+  zlib_stream_.get()->avail_out = sizeof(dummy_output);
+
+  int ret = inflate(zlib_stream_.get(), Z_NO_FLUSH);
+  return ret == Z_OK;
+}
+
+bool GzipSourceStream::IsGzipHeaderInvalid(DrainableIOBuffer* input_buffer) {
+  const size_t kGzipFooterBytes = 8;
+  const char* end = nullptr;
+  GZipHeader::Status status = gzip_header_.ReadMore(
+      input_buffer->data(), input_buffer->BytesRemaining(), &end);
+  if (status == GZipHeader::INCOMPLETE_HEADER) {
+    input_buffer->DidConsume(input_buffer->BytesRemaining());
+    return false;
+  }
+
+  should_check_gzip_header_ = false;
+  if (status == GZipHeader::COMPLETE_HEADER) {
+    // If there is a valid header, there should also be a valid footer.
+    gzip_footer_bytes_left_ = kGzipFooterBytes;
+    input_buffer->DidConsume(end - input_buffer->data());
+  }
+
+  return status == GZipHeader::INVALID_HEADER;
+}
+
+// Dumb heuristic. Gzip files always start with a two-byte magic value per RFC
+// 1952 2.3.1, so if the first byte isn't the first byte of the gzip magic, and
+// this filter is checking whether it should fallback, then fallback.
+bool GzipSourceStream::ShouldFallbackToPlain(DrainableIOBuffer* input_buffer) {

[Randy Smith (Not in Mondays), 2016/09/12 20:54:01]

Could we not pass a DrainableIOBuffer to this function?  That type implies the
function could actually drain data, which it won't.  char* or void*?

[Randy Smith (Not in Mondays), 2016/09/12 20:54:01]

I'm also concerned that the interface contract for this function says that it'll
return "false" if there isn't enough information in DrainableIOBuffer to decide
whether or not to fallback, but the code that calls it assumes that false always
means that we should not fall back.  We only need a single byte for the decision
we're making; maybe pass in that byte instead of a char* and require the calling
code to know that we need at least a byte?

[xunjieli, 2016/09/14 16:44:01]

Done.

[xunjieli, 2016/09/14 16:44:01]

Done.

+  static const char kGzipFirstByte = 0x1f;
+  if (mode_ != GZIP_SOURCE_STREAM_GZIP_WITH_FALLBACK)
+    return false;
+  if (!should_check_gzip_header_)
+    return false;
+  if (input_buffer->BytesRemaining() == 0)
+    return false;
+  return input_buffer->data()[0] != kGzipFirstByte;
+}
+
+void GzipSourceStream::SkipGzipFooterIfNeeded(DrainableIOBuffer* input_buffer) {
+  if (gzip_footer_bytes_left_ == 0)
+    return;
+  size_t to_read = gzip_footer_bytes_left_;
+  if (to_read > base::checked_cast<size_t>(input_buffer->BytesRemaining()))
+    to_read = input_buffer->BytesRemaining();
+  input_buffer->DidConsume(to_read);
+  gzip_footer_bytes_left_ -= to_read;
+}
+
+}  // namespace net