[Sync] Finch Experiment : Enable compression between sync client and server

sync/internal_api/http_bridge.cc

 // Copyright 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "sync/internal_api/public/http_bridge.h"
 
 #include "base/message_loop/message_loop.h"
+#include "base/metrics/field_trial.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/stringprintf.h"
 #include "net/base/load_flags.h"
 #include "net/base/net_errors.h"
 #include "net/http/http_cache.h"
 #include "net/http/http_network_layer.h"
 #include "net/http/http_request_headers.h"
 #include "net/http/http_response_headers.h"
 #include "net/url_request/static_http_user_agent_settings.h"
@@ skipping 10 matching lines @@
 // It's possible for an http request to be silently stalled. We set a time
 // limit for all http requests, beyond which the request is cancelled and
 // treated as a transient failure.
 const int kMaxHttpRequestTimeSeconds = 60 * 5;  // 5 minutes.
 
 // Helper method for logging timeouts via UMA.
 void LogTimeout(bool timed_out) {
   UMA_HISTOGRAM_BOOLEAN("Sync.URLFetchTimedOut", timed_out);
 }
 
+bool IsSyncHttpContentCompressionEnabled() {
+  const std::string group_name =
+      base::FieldTrialList::FindFullName("SyncHttpContentCompression");
+  return group_name == "Enabled";
+}
+
+// The difference in bytes between a zlib header and a gzip header.

[Nicolas Zea, 2015/07/30 19:59:44]

Add a comment that explains that makes it clear which of these are copied from
compression_utils.cc. To do that, it might be useful to move the
RecordXXHistogram methods here, and then say something like:
-----------
The rest of the code in the anon namespace is copied from
components/metrics/compression_utils.cc
-----------

[Gang Wu, 2015/07/31 00:39:45]

Done.

+const size_t kGzipZlibHeaderDifferenceBytes = 16;
+
+// Pass an integer greater than the following get a gzip header instead of a
+// zlib header when calling deflateInit2() and inflateInit2().
+const int kWindowBitsToGetGzipHeader = 16;
+
+// This describes the amount of memory zlib uses to compress data. It can go
+// from 1 to 9, with 8 being the default. For details, see:
+// http://www.zlib.net/manual.html (search for memLevel).
+const int kZlibMemoryLevel = 8;
+
+// This code is taken almost verbatim from third_party/zlib/compress.c. The only
+// difference is deflateInit2() is called which sets the window bits to be > 16.
+// That causes a gzip header to be emitted rather than a zlib header.
+int GzipCompressHelper(Bytef* dest,
+                       uLongf* dest_length,
+                       const Bytef* source,
+                       uLong source_length) {
+  z_stream stream;
+
+  stream.next_in = bit_cast<Bytef*>(source);
+  stream.avail_in = static_cast<uInt>(source_length);
+  stream.next_out = dest;
+  stream.avail_out = static_cast<uInt>(*dest_length);
+  if (static_cast<uLong>(stream.avail_out) != *dest_length)
+    return Z_BUF_ERROR;
+
+  stream.zalloc = static_cast<alloc_func>(0);
+  stream.zfree = static_cast<free_func>(0);
+  stream.opaque = static_cast<voidpf>(0);
+
+  gz_header gzip_header;
+  memset(&gzip_header, 0, sizeof(gzip_header));
+  int err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
+                         MAX_WBITS + kWindowBitsToGetGzipHeader,
+                         kZlibMemoryLevel, Z_DEFAULT_STRATEGY);
+  if (err != Z_OK)
+    return err;
+
+  err = deflateSetHeader(&stream, &gzip_header);
+  if (err != Z_OK)
+    return err;
+
+  err = deflate(&stream, Z_FINISH);
+  if (err != Z_STREAM_END) {
+    deflateEnd(&stream);
+    return err == Z_OK ? Z_BUF_ERROR : err;
+  }
+  *dest_length = stream.total_out;
+
+  err = deflateEnd(&stream);
+  return err;
+}
+
+// This code is taken almost verbatim from third_party/zlib/uncompr.c. The only
+// difference is inflateInit2() is called which sets the window bits to be > 16.
+// That causes a gzip header to be parsed rather than a zlib header.
+int GzipUncompressHelper(Bytef* dest,
+                         uLongf* dest_length,
+                         const Bytef* source,
+                         uLong source_length) {
+  z_stream stream;
+
+  stream.next_in = bit_cast<Bytef*>(source);
+  stream.avail_in = static_cast<uInt>(source_length);
+  if (static_cast<uLong>(stream.avail_in) != source_length)
+    return Z_BUF_ERROR;
+
+  stream.next_out = dest;
+  stream.avail_out = static_cast<uInt>(*dest_length);
+  if (static_cast<uLong>(stream.avail_out) != *dest_length)
+    return Z_BUF_ERROR;
+
+  stream.zalloc = static_cast<alloc_func>(0);
+  stream.zfree = static_cast<free_func>(0);
+
+  int err = inflateInit2(&stream, MAX_WBITS + kWindowBitsToGetGzipHeader);
+  if (err != Z_OK)
+    return err;
+
+  err = inflate(&stream, Z_FINISH);
+  if (err != Z_STREAM_END) {
+    inflateEnd(&stream);
+    if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
+      return Z_DATA_ERROR;
+    return err;
+  }
+  *dest_length = stream.total_out;
+
+  err = inflateEnd(&stream);
+  return err;
+}
+
+// Returns the uncompressed size from GZIP-compressed |compressed_data|.
+uint32 GetUncompressedSize(const std::string& compressed_data) {
+  // The uncompressed size is stored in the last 4 bytes of |input| in LE.
+  uint32 size;
+  if (compressed_data.length() < sizeof(size))
+    return 0;
+  memcpy(&size, &compressed_data[compressed_data.length() - sizeof(size)],
+         sizeof(size));
+  return base::ByteSwapToLE32(size);
+}
+
+void RecordSyncRequestContentLengthHistograms(int64 compressed_content_length,
+                                              int64 original_content_length) {
+  UMA_HISTOGRAM_COUNTS("Sync.RequestContentLengthCompressed",
+                       compressed_content_length);
+  UMA_HISTOGRAM_COUNTS("Sync.RequestContentLengthOriginal",
+                       original_content_length);
+}
+
+void RecordSyncResponseContentLengthHistograms(int64 compressed_content_length,
+                                               int64 original_content_length) {
+  UMA_HISTOGRAM_COUNTS("Sync.ResponseContentLengthCompressed",
+                       compressed_content_length);
+  UMA_HISTOGRAM_COUNTS("Sync.ResponseContentLengthOriginal",
+                       original_content_length);
+}
+
 }  // namespace
 
+bool GzipCompress(const std::string& input, std::string* output) {
+  const uLongf input_size = static_cast<uLongf>(input.size());
+  std::vector<Bytef> compressed_data(kGzipZlibHeaderDifferenceBytes +
+                                     compressBound(input_size));
+
+  uLongf compressed_size = static_cast<uLongf>(compressed_data.size());
+  if (GzipCompressHelper(&compressed_data.front(), &compressed_size,
+                         bit_cast<const Bytef*>(input.data()),
+                         input_size) != Z_OK) {
+    return false;
+  }
+
+  compressed_data.resize(compressed_size);
+  output->assign(compressed_data.begin(), compressed_data.end());
+  DCHECK_EQ(input_size, GetUncompressedSize(*output));
+  return true;
+}
+
+bool GzipUncompress(const std::string& input, std::string* output) {
+  std::string uncompressed_output;
+  uLongf uncompressed_size = static_cast<uLongf>(GetUncompressedSize(input));
+  uncompressed_output.resize(uncompressed_size);
+  if (GzipUncompressHelper(bit_cast<Bytef*>(uncompressed_output.data()),
+                           &uncompressed_size,
+                           bit_cast<const Bytef*>(input.data()),
+                           static_cast<uLongf>(input.length())) == Z_OK) {
+    output->swap(uncompressed_output);
+    return true;
+  }
+  return false;
+}
+
 HttpBridgeFactory::HttpBridgeFactory(
     const scoped_refptr<net::URLRequestContextGetter>& request_context_getter,
     const NetworkTimeUpdateCallback& network_time_update_callback,
     CancelationSignal* cancelation_signal)
     : request_context_getter_(request_context_getter),
       network_time_update_callback_(network_time_update_callback),
       cancelation_signal_(cancelation_signal) {
   // Registration should never fail.  This should happen on the UI thread during
   // init.  It would be impossible for a shutdown to have been requested at this
   // point.
@@ skipping 146 matching lines @@
 
   // Start the timer on the network thread (the same thread progress is made
   // on, and on which the url fetcher lives).
   DCHECK(!fetch_state_.http_request_timeout_timer.get());
   fetch_state_.http_request_timeout_timer.reset(new base::Timer(false, false));
   fetch_state_.http_request_timeout_timer->Start(
       FROM_HERE, base::TimeDelta::FromSeconds(kMaxHttpRequestTimeSeconds),
       base::Bind(&HttpBridge::OnURLFetchTimedOut, this));
 
   DCHECK(request_context_getter_.get());
+  fetch_state_.start_time = base::Time::Now();
   fetch_state_.url_poster =
       net::URLFetcher::Create(url_for_request_, net::URLFetcher::POST, this)
           .release();
   fetch_state_.url_poster->SetRequestContext(request_context_getter_.get());
-  fetch_state_.url_poster->SetUploadData(content_type_, request_content_);
   fetch_state_.url_poster->SetExtraRequestHeaders(extra_headers_);
+
+  int compressed_content_size = 0;

[Nicolas Zea, 2015/07/30 19:59:44]

You're using an int here, but the RequestXXXMethods take int64s, and std::string
actually returns a size_t. Which is correct?

[Gang Wu, 2015/07/31 00:39:45]

int64, since GetContentLength() return int64 it might return -1.
it might have problem from size_t -> int64, but since
HttpBridge::GetResponseContentLength did same thing(size_t -> int), and HTTP
will not send such a long content(will chunk it), so there is no risk to convert
from size_t to int64. 

+  if (IsSyncHttpContentCompressionEnabled()) {
+    std::string compressed_request_content;
+    GzipCompress(request_content_, &compressed_request_content);
+    compressed_content_size = compressed_request_content.size();
+    fetch_state_.url_poster->SetUploadData(content_type_,
+                                           compressed_request_content);
+    fetch_state_.url_poster->AddExtraRequestHeader("Content-Encoding: gzip");
+  } else {
+    fetch_state_.url_poster->SetUploadData(content_type_, request_content_);
+    fetch_state_.url_poster->AddExtraRequestHeader("Accept-Encoding: deflate");
+  }
+
+  RecordSyncRequestContentLengthHistograms(compressed_content_size,
+                                           request_content_.size());
+
   fetch_state_.url_poster->AddExtraRequestHeader(base::StringPrintf(
       "%s: %s", net::HttpRequestHeaders::kUserAgent, user_agent_.c_str()));
   fetch_state_.url_poster->SetLoadFlags(net::LOAD_BYPASS_CACHE |
                                         net::LOAD_DISABLE_CACHE |
                                         net::LOAD_DO_NOT_SAVE_COOKIES |
                                         net::LOAD_DO_NOT_SEND_COOKIES);
-  fetch_state_.start_time = base::Time::Now();
 
   fetch_state_.url_poster->Start();
 }
 
 int HttpBridge::GetResponseContentLength() const {
   DCHECK_EQ(base::MessageLoop::current(), created_on_loop_);
   base::AutoLock lock(fetch_state_lock_);
   DCHECK(fetch_state_.request_completed);
   return fetch_state_.response_content.size();
 }
@@ skipping 79 matching lines @@
   // Use a real (non-debug) log to facilitate troubleshooting in the wild.
   VLOG(2) << "HttpBridge::OnURLFetchComplete for: "
           << fetch_state_.url_poster->GetURL().spec();
   VLOG(1) << "HttpBridge received response code: "
           << fetch_state_.http_response_code;
 
   source->GetResponseAsString(&fetch_state_.response_content);
   fetch_state_.response_headers = source->GetResponseHeaders();
   UpdateNetworkTime();
 
+  int64 compressed_content_length = fetch_state_.response_content.size();

[Nicolas Zea, 2015/07/30 19:59:44]

Same comment above about type consistency/correctness.

[Gang Wu, 2015/07/31 00:39:45]

Done.

+  int64 original_content_length = compressed_content_length;
+  if (fetch_state_.response_headers &&
+      fetch_state_.response_headers->HasHeaderValue("content-encoding",
+                                                    "gzip")) {
+    compressed_content_length =
+        fetch_state_.response_headers->GetContentLength();
+  }
+  RecordSyncResponseContentLengthHistograms(compressed_content_length,
+                                            original_content_length);
+
   // End of the line for url_poster_. It lives only on the IO loop.
   // We defer deletion because we're inside a callback from a component of the
   // URLFetcher, so it seems most natural / "polite" to let the stack unwind.
   base::MessageLoop::current()->DeleteSoon(FROM_HERE, fetch_state_.url_poster);
   fetch_state_.url_poster = NULL;
 
   // Wake the blocked syncer thread in MakeSynchronousPost.
   // WARNING: DONT DO ANYTHING AFTER THIS CALL! |this| may be deleted!
   http_post_completed_.Signal();
 }
@@ skipping 64 matching lines @@
   int64 sane_time_ms = 0;
   if (base::StringToInt64(sane_time_str, &sane_time_ms)) {
     network_time_update_callback_.Run(
         base::Time::FromJsTime(sane_time_ms),
         base::TimeDelta::FromMilliseconds(1),
         fetch_state_.end_time - fetch_state_.start_time);
   }
 }
 
 }  // namespace syncer