HTTP retry support.

net/http/http_network_transaction.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/http/http_network_transaction.h"
 
 #include <set>
 #include <vector>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/compiler_specific.h"
 #include "base/format_macros.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/metrics/field_trial.h"
 #include "base/metrics/histogram.h"
 #include "base/metrics/stats_counters.h"
+#include "base/pickle.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
 #include "base/values.h"
 #include "build/build_config.h"
+#include "crypto/secure_hash.h"
+#include "crypto/sha2.h"
 #include "net/base/auth.h"
 #include "net/base/host_port_pair.h"
 #include "net/base/io_buffer.h"
 #include "net/base/load_flags.h"
 #include "net/base/load_timing_info.h"
 #include "net/base/net_errors.h"
 #include "net/base/net_util.h"
 #include "net/base/upload_data_stream.h"
 #include "net/http/http_auth.h"
 #include "net/http/http_auth_handler.h"
@@ skipping 33 matching lines @@
 #endif
 
 
 using base::Time;
 using base::TimeDelta;
 
 namespace net {
 
 namespace {
 
+const int kNumRetries = 3;
+
 void ProcessAlternateProtocol(
     HttpNetworkSession* session,
     const HttpResponseHeaders& headers,
     const HostPortPair& http_host_port_pair) {
   if (!headers.HasHeader(kAlternateProtocolHeader))
     return;
 
   std::vector<std::string> alternate_protocol_values;
   void* iter = NULL;
   std::string alternate_protocol_str;
@@ skipping 33 matching lines @@
   dict->SetInteger("net_error", net_error);
   dict->SetInteger("version_before", version_before);
   dict->SetInteger("version_after", version_after);
   return dict;
 }
 
 }  // namespace
 
 //-----------------------------------------------------------------------------
 
+// To verify that retry attempts will not cause errors we hash all received
+// content. When retrying we hash the content again and verify that the
+// previous hash matches once we have received the same amount of data.
+class HttpNetworkTransaction::HttpStreamHash {
+public:
+  HttpStreamHash()
+    :hash_(crypto::SecureHash::Create(crypto::SecureHash::SHA256)) {
+  }
+
+  // Add to hash.
+  void Update(const void* input, size_t len) {
+    hash_->Update(input, len);
+  }
+
+  // Finish hash once all content has been received.
+  void Finish() {
+    hash_->Finish(previous_hash_, sizeof(previous_hash_));
+    hash_.reset(crypto::SecureHash::Create(crypto::SecureHash::SHA256));
+  }
+
+  // Verify hash after serializing the state. Then deserialize so that we can
+  // keep hashing if we decide to continue fetching the content.
+  int VerifyHash() {
+    int result = OK;
+    uint8 hash[8];
+    Pickle pickle;
+    hash_->Serialize(&pickle);
+    hash_->Finish(hash, sizeof(hash));
+    DCHECK(sizeof(hash) == sizeof(previous_hash_));
+    if (memcmp(hash, previous_hash_, sizeof(previous_hash_)) != 0) {
+      result = ERR_RETRY_HASH_MISMATCH;
+      UMA_HISTOGRAM_COUNTS("Net.HttpRetry.VerifyHashFailure", 1);
+    }
+    else {
+      PickleIterator data_iterator(pickle);
+      hash_->Deserialize(&data_iterator);
+      UMA_HISTOGRAM_COUNTS("Net.HttpRetry.VerifyHashSuccess", 1);
+    }
+
+    return result;
+  }
+
+  void SetPreviousHash(const void* hash, size_t len) {
+    DCHECK(len == sizeof(previous_hash_));
+    memcpy(previous_hash_, hash, len);
+  }
+
+private:
+  // Hash of current attempt to retrieve the resource.
+  scoped_ptr<crypto::SecureHash> hash_;
+
+  // Hash of previous attempt to retrieve the resource.
+  uint8 previous_hash_[8];
+};
+
 HttpNetworkTransaction::HttpNetworkTransaction(RequestPriority priority,
                                                HttpNetworkSession* session)
     : pending_auth_target_(HttpAuth::AUTH_NONE),
       io_callback_(base::Bind(&HttpNetworkTransaction::OnIOComplete,
                               base::Unretained(this))),
       session_(session),
       request_(NULL),
       priority_(priority),
       headers_valid_(false),
       logged_response_time_(false),
       fallback_error_code_(ERR_SSL_INAPPROPRIATE_FALLBACK),
       request_headers_(),
       read_buf_len_(0),
       total_received_bytes_(0),
+      retry_attempt_(0),
+      offset_(0),
+      previous_content_length_(0),
+      received_body_length_(0),
+      stream_hash_(new HttpStreamHash()),
       next_state_(STATE_NONE),
       establishing_tunnel_(false),
       websocket_handshake_stream_base_create_helper_(NULL) {
   session->ssl_config_service()->GetSSLConfig(&server_ssl_config_);
   session->GetNextProtos(&server_ssl_config_.next_protos);
   proxy_ssl_config_ = server_ssl_config_;
 }
 
 HttpNetworkTransaction::~HttpNetworkTransaction() {
   if (stream_.get()) {
@@ skipping 838 matching lines @@
   // bizarre for SPDY. Assuming this logic is useful at all.
   // TODO(davidben): Bubble the error code up so we do not cache?
   if (result == ERR_CONNECTION_CLOSED && response_.headers.get())
     result = OK;
 
   if (result < 0)
     return HandleIOError(result);
 
   DCHECK(response_.headers.get());
 
+  if ((response_.headers->response_code() == 200 ||
+       response_.headers->response_code() == 206) && offset_) {
+    std::string etag, last_modified;
+    response_.headers->EnumerateHeader(NULL, "last-modified", &last_modified);
+    if (response_.headers->GetHttpVersion() >= HttpVersion(1, 1))
+       response_.headers->EnumerateHeader(NULL, "etag", &etag);
+
+    // Return an error if content was updated on a retry.
+    if ((previous_content_length_ != response_.headers->GetContentLength() &&
+         response_.headers->response_code() == 200) ||
+        previous_etag_ != etag || previous_last_modified_ != last_modified) {
+      return HandleIOError(ERR_RETRY_CONTENT_UPDATED);
+    }
+  }
+  if (response_.headers->response_code() != 200)
+    offset_ = 0;
+
   // On a 408 response from the server ("Request Timeout") on a stale socket,
   // retry the request.
   // Headers can be NULL because of http://crbug.com/384554.
   if (response_.headers.get() && response_.headers->response_code() == 408 &&
       stream_->IsConnectionReused()) {
     net_log_.AddEventWithNetErrorCode(
         NetLog::TYPE_HTTP_TRANSACTION_RESTART_AFTER_ERROR,
         response_.headers->response_code());
     // This will close the socket - it would be weird to try and reuse it, even
     // if the server doesn't actually close it.
@@ skipping 41 matching lines @@
 
   int rv = HandleAuthChallenge();
   if (rv != OK)
     return rv;
 
   if (is_https_request())
     stream_->GetSSLInfo(&response_.ssl_info);
 
   headers_valid_ = true;
 
+  if (retry_attempt_ && read_buf_.get())
+    next_state_ = STATE_READ_BODY;
+
   if (session_->huffman_aggregator()) {
     session_->huffman_aggregator()->AggregateTransactionCharacterCounts(
         *request_,
         request_headers_,
         proxy_info_.proxy_server(),
         *response_.headers);
   }
   return OK;
 }
 
 int HttpNetworkTransaction::DoReadBody() {
   DCHECK(read_buf_.get());
   DCHECK_GT(read_buf_len_, 0);
   DCHECK(stream_ != NULL);
 
   next_state_ = STATE_READ_BODY_COMPLETE;
   return stream_->ReadResponseBody(
       read_buf_.get(), read_buf_len_, io_callback_);
 }
 
 int HttpNetworkTransaction::DoReadBodyComplete(int result) {
   // We are done with the Read call.
   bool done = false;
   if (result <= 0) {
     DCHECK_NE(ERR_IO_PENDING, result);
+    stream_hash_->Finish();
+    if (result < 0)
+      return HandleIOError(result);
     done = true;
   }
 
   bool keep_alive = false;
   if (stream_->IsResponseBodyComplete()) {
     // Note: Just because IsResponseBodyComplete is true, we're not
     // necessarily "done".  We're only "done" when it is the last
     // read on this HttpNetworkTransaction, which will be signified
     // by a zero-length read.
     // TODO(mbelshe): The keepalive property is really a property of
     //    the stream.  No need to compute it here just to pass back
     //    to the stream's Close function.
     // TODO(rtenneti): CanFindEndOfResponse should return false if there are no
     // ResponseHeaders.
     if (stream_->CanFindEndOfResponse()) {
       HttpResponseHeaders* headers = GetResponseHeaders();
       if (headers)
         keep_alive = headers->IsKeepAlive();
     }
   }
 
+  if (result > 0) {
+    received_body_length_ += result;
+    if (offset_) {
+      if (offset_ <= result) {
+         stream_hash_->Update((uint8*)read_buf_->data(), offset_);
+        if (stream_hash_->VerifyHash() != OK)
+          return HandleIOError(ERR_RETRY_HASH_MISMATCH);
+        stream_hash_->Update((uint8*)read_buf_->data() + offset_,
+                             result - offset_);
+        memmove(read_buf_->data(), read_buf_->data() + offset_,
+                result - offset_);
+        result -= offset_;
+        offset_ = 0;
+      }
+      else
+      {
+        stream_hash_->Update((uint8*)read_buf_->data(), result);
+        offset_ -= result;
+        result = 0;
+      }
+
+      if (result == 0) {
+        next_state_ = STATE_READ_BODY;
+        return result;
+      }
+    }
+    else
+      stream_hash_->Update((uint8*)read_buf_->data(), result);
+  }
+
   // Clean up connection if we are done.
   if (done) {
     LogTransactionMetrics();
     stream_->Close(!keep_alive);
     // Note: we don't reset the stream here.  We've closed it, but we still
     // need it around so that callers can call methods such as
     // GetUploadProgress() and have them be meaningful.
     // TODO(mbelshe): This means we closed the stream here, and we close it
     // again in ~HttpNetworkTransaction.  Clean that up.
 
@@ skipping 100 matching lines @@
                              base::TimeDelta::FromMinutes(10), 100);
 
   if (!stream_->IsConnectionReused()) {
     UMA_HISTOGRAM_CUSTOM_TIMES(
         "Net.Transaction_Latency_Total_New_Connection",
         total_duration, base::TimeDelta::FromMilliseconds(1),
         base::TimeDelta::FromMinutes(10), 100);
   }
 }
 
+void HttpNetworkTransaction::LogTransactionRetryMetrics(int error) const {
+  UMA_HISTOGRAM_CUSTOM_COUNTS("Net.HttpRetry.Count", retry_attempt_, 1, 5, 5);
+  if (request_->load_flags & LOAD_MAIN_FRAME) {
+    UMA_HISTOGRAM_CUSTOM_ENUMERATION(
+    "Net.HttpRetry.Cause.MainFrame",
+    -error,
+    GetAllErrorCodesForUma());
+  } else {
+    UMA_HISTOGRAM_CUSTOM_ENUMERATION(
+    "Net.HttpRetry.Cause.Subresource",
+    -error,
+    GetAllErrorCodesForUma());
+  }
+}
+
 int HttpNetworkTransaction::HandleCertificateRequest(int error) {
   // There are two paths through which the server can request a certificate
   // from us.  The first is during the initial handshake, the second is
   // during SSL renegotiation.
   //
   // In both cases, we want to close the connection before proceeding.
   // We do this for two reasons:
   //   First, we don't want to keep the connection to the server hung for a
   //   long time while the user selects a certificate.
   //   Second, even if we did keep the connection open, NSS has a bug where
@@ skipping 174 matching lines @@
     // likely happen when trying to retrieve its IP address.
     // See http://crbug.com/105824 for more details.
     case ERR_SOCKET_NOT_CONNECTED:
     // If a socket is closed on its initial request, HttpStreamParser returns
     // ERR_EMPTY_RESPONSE. This may still be close/reuse race if the socket was
     // preconnected but failed to be used before the server timed it out.
     case ERR_EMPTY_RESPONSE:
       if (ShouldResendRequest()) {
         net_log_.AddEventWithNetErrorCode(
             NetLog::TYPE_HTTP_TRANSACTION_RESTART_AFTER_ERROR, error);
+        retry_attempt_++;
+        LogTransactionRetryMetrics(error);
         ResetConnectionAndRequestForResend();
         error = OK;
       }
       break;
     case ERR_SPDY_PING_FAILED:
     case ERR_SPDY_SERVER_REFUSED_STREAM:
     case ERR_QUIC_HANDSHAKE_FAILED:
       net_log_.AddEventWithNetErrorCode(
           NetLog::TYPE_HTTP_TRANSACTION_RESTART_AFTER_ERROR, error);
       ResetConnectionAndRequestForResend();
@@ skipping 21 matching lines @@
   request_headers_.Clear();
   response_ = HttpResponseInfo();
   establishing_tunnel_ = false;
 }
 
 HttpResponseHeaders* HttpNetworkTransaction::GetResponseHeaders() const {
   return response_.headers.get();
 }
 
 bool HttpNetworkTransaction::ShouldResendRequest() const {
-  bool connection_is_proven = stream_->IsConnectionReused();
-  bool has_received_headers = GetResponseHeaders() != NULL;
+  if (!stream_->IsConnectionReused() &&
+      !(request_->method == "GET" || request_->method == "HEAD"))

[rvargas (doing something else), 2014/07/26 00:30:33]

nit: requires {}

+    return false;
 
-  // NOTE: we resend a request only if we reused a keep-alive connection.
-  // This automatically prevents an infinite resend loop because we'll run
-  // out of the cached keep-alive connections eventually.
-  if (connection_is_proven && !has_received_headers)
-    return true;
-  return false;
+  if (retry_attempt_ > kNumRetries)
+    return false;
+
+  if (GetResponseHeaders() != NULL) {
+    // Do not retry automatically if resource is larger than 1 Mb until byte
+    // range support is enabled. This is temporary.
+    if (response_.headers->GetContentLength() > 1000000)
+      return false;
+
+    TimeDelta time_delta = base::TimeTicks::Now() - send_start_time_;
+    if (time_delta.InSeconds() > 30)
+      return false;
+
+    if (response_.headers->HasStrongValidators()) {

[rvargas (doing something else), 2014/07/26 00:30:33]

nit: no {}

+      return true;
+    }
+
+    if (response_.headers->HasHeaderValue("cache-control", "no-cache") ||
+        response_.headers->HasHeaderValue("cache-control", "no-store") ||
+        response_.headers->HasHeaderValue("cache-control", "must-revalidate") ||
+        response_.headers->HasHeaderValue("pragma", "no-cache")) {
+      return false;
+    }
+
+    if (response_.headers->GetMaxAgeValue(&time_delta))
+      return time_delta.InSeconds() > 60;
+
+    // If there is no Date header, then assume that the server response was
+    // generated at the time when we received the response, hence do not retry.
+    Time date_value;
+    if (!response_.headers->GetDateValue(&date_value))
+      return false;
+
+    Time expires_value;
+    if (response_.headers->GetExpiresValue(&expires_value)) {
+      time_delta = expires_value - date_value;
+      if (time_delta.InSeconds() > 60)
+        return true;
+    }
+    return false;
+  }
+  return true;
 }
 
 void HttpNetworkTransaction::ResetConnectionAndRequestForResend() {
+  // We need to clear request_headers_ because it contains the real request
+  // headers, but we may need to resend the CONNECT request first to recreate
+  // the SSL tunnel.
+  request_headers_.Clear();
+  headers_valid_ = false;
+
+  if (stream_) {
+    total_received_bytes_ += stream_->GetTotalReceivedBytes();
+    if (received_body_length_ > offset_) {
+      offset_ = received_body_length_;
+    }
+  }
+  received_body_length_ = 0;
+
+  HttpResponseHeaders* headers = GetResponseHeaders();
+  if (offset_ && headers) {
+    previous_content_length_ = headers->GetContentLength();
+    headers->EnumerateHeader(NULL, "last-modified",
+                             &previous_last_modified_);
+    if (headers->GetHttpVersion() >= HttpVersion(1, 1))
+      headers->EnumerateHeader(NULL, "etag", &previous_etag_);
+
+    // Disable until we have statistics that show that retrying does not
+    // result in too many ERR_RETRY_HASH_MISMATCH.
+    if (0 && previous_content_length_ &&
+        headers->HasHeaderValue("Accept-Ranges", "bytes")) {
+      // Try resending using a range request if supported.
+      request_headers_.SetHeader(HttpRequestHeaders::kRange,
+          "bytes=" + base::Uint64ToString(offset_) + "-");
+      if (!previous_etag_.empty())
+        request_headers_.SetHeader("If-Match", previous_etag_);
+      else if (!previous_last_modified_.empty()) {
+        request_headers_.SetHeader("If-Unmodified-Since",
+            previous_last_modified_);
+      }
+    }
+  }
+
+  response_ = HttpResponseInfo();
+  establishing_tunnel_ = false;
   if (stream_.get()) {
     stream_->Close(true);
     stream_.reset();
   }
-
-  // We need to clear request_headers_ because it contains the real request
-  // headers, but we may need to resend the CONNECT request first to recreate
-  // the SSL tunnel.
-  request_headers_.Clear();
   next_state_ = STATE_CREATE_STREAM;  // Resend the request.
 }
 
 bool HttpNetworkTransaction::ShouldApplyProxyAuth() const {
   return !is_https_request() &&
       (proxy_info_.is_https() || proxy_info_.is_http());
 }
 
 bool HttpNetworkTransaction::ShouldApplyServerAuth() const {
   return !(request_->load_flags & LOAD_DO_NOT_SEND_AUTH_DATA);
@@ skipping 88 matching lines @@
       description = base::StringPrintf("Unknown state 0x%08X (%u)", state,
                                        state);
       break;
   }
   return description;
 }
 
 #undef STATE_CASE
 
 }  // namespace net