HTTP retry support.

net/http/http_network_transaction.cc

Index: net/http/http_network_transaction.cc
diff --git a/net/http/http_network_transaction.cc b/net/http/http_network_transaction.cc
index f4df073b4bba5ad905631653d2cba46820c0daa7..1f05ea418c1b1ddf3e4112522472fd036333d5c7 100644
--- a/net/http/http_network_transaction.cc
+++ b/net/http/http_network_transaction.cc
@@ -15,6 +15,7 @@
 #include "base/metrics/field_trial.h"
 #include "base/metrics/histogram.h"
 #include "base/metrics/stats_counters.h"
+#include "base/pickle.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
@@ -22,6 +23,8 @@
 #include "base/time/time.h"
 #include "base/values.h"
 #include "build/build_config.h"
+#include "crypto/secure_hash.h"
+#include "crypto/sha2.h"
 #include "net/base/auth.h"
 #include "net/base/host_port_pair.h"
 #include "net/base/io_buffer.h"
@@ -75,6 +78,8 @@ namespace net {
 
 namespace {
 
+const int kNumRetries = 3;
+
 void ProcessAlternateProtocol(
     HttpNetworkSession* session,
     const HttpResponseHeaders& headers,
@@ -128,6 +133,61 @@ base::Value* NetLogSSLVersionFallbackCallback(
 
 //-----------------------------------------------------------------------------
 
+// To verify that retry attempts will not cause errors we hash all received
+// content. When retrying we hash the content again and verify that the
+// previous hash matches once we have received the same amount of data.
+class HttpNetworkTransaction::HttpStreamHash {
+public:
+  HttpStreamHash()
+    :hash_(crypto::SecureHash::Create(crypto::SecureHash::SHA256)) {
+  }
+
+  // Add to hash.
+  void Update(const void* input, size_t len) {
+    hash_->Update(input, len);
+  }
+
+  // Finish hash once all content has been received.
+  void Finish() {
+    hash_->Finish(previous_hash_, sizeof(previous_hash_));
+    hash_.reset(crypto::SecureHash::Create(crypto::SecureHash::SHA256));
+  }
+
+  // Verify hash after serializing the state. Then deserialize so that we can
+  // keep hashing if we decide to continue fetching the content.
+  int VerifyHash() {
+    int result = OK;
+    uint8 hash[8];
+    Pickle pickle;
+    hash_->Serialize(&pickle);
+    hash_->Finish(hash, sizeof(hash));
+    DCHECK(sizeof(hash) == sizeof(previous_hash_));
+    if (memcmp(hash, previous_hash_, sizeof(previous_hash_)) != 0) {
+      result = ERR_RETRY_HASH_MISMATCH;
+      UMA_HISTOGRAM_COUNTS("Net.HttpRetry.VerifyHashFailure", 1);
+    }
+    else {
+      PickleIterator data_iterator(pickle);
+      hash_->Deserialize(&data_iterator);
+      UMA_HISTOGRAM_COUNTS("Net.HttpRetry.VerifyHashSuccess", 1);
+    }
+
+    return result;
+  }
+
+  void SetPreviousHash(const void* hash, size_t len) {
+    DCHECK(len == sizeof(previous_hash_));
+    memcpy(previous_hash_, hash, len);
+  }
+
+private:
+  // Hash of current attempt to retrieve the resource.
+  scoped_ptr<crypto::SecureHash> hash_;
+
+  // Hash of previous attempt to retrieve the resource.
+  uint8 previous_hash_[8];
+};
+
 HttpNetworkTransaction::HttpNetworkTransaction(RequestPriority priority,
                                                HttpNetworkSession* session)
     : pending_auth_target_(HttpAuth::AUTH_NONE),
@@ -142,6 +202,11 @@ HttpNetworkTransaction::HttpNetworkTransaction(RequestPriority priority,
       request_headers_(),
       read_buf_len_(0),
       total_received_bytes_(0),
+      retry_attempt_(0),
+      offset_(0),
+      previous_content_length_(0),
+      received_body_length_(0),
+      stream_hash_(new HttpStreamHash()),
       next_state_(STATE_NONE),
       establishing_tunnel_(false),
       websocket_handshake_stream_base_create_helper_(NULL) {
@@ -1000,6 +1065,23 @@ int HttpNetworkTransaction::DoReadHeadersComplete(int result) {
 
   DCHECK(response_.headers.get());
 
+  if ((response_.headers->response_code() == 200 ||
+       response_.headers->response_code() == 206) && offset_) {
+    std::string etag, last_modified;
+    response_.headers->EnumerateHeader(NULL, "last-modified", &last_modified);
+    if (response_.headers->GetHttpVersion() >= HttpVersion(1, 1))
+       response_.headers->EnumerateHeader(NULL, "etag", &etag);
+
+    // Return an error if content was updated on a retry.
+    if ((previous_content_length_ != response_.headers->GetContentLength() &&
+         response_.headers->response_code() == 200) ||
+        previous_etag_ != etag || previous_last_modified_ != last_modified) {
+      return HandleIOError(ERR_RETRY_CONTENT_UPDATED);
+    }
+  }
+  if (response_.headers->response_code() != 200)
+    offset_ = 0;
+
   // On a 408 response from the server ("Request Timeout") on a stale socket,
   // retry the request.
   // Headers can be NULL because of http://crbug.com/384554.
@@ -1061,6 +1143,9 @@ int HttpNetworkTransaction::DoReadHeadersComplete(int result) {
 
   headers_valid_ = true;
 
+  if (retry_attempt_ && read_buf_.get())
+    next_state_ = STATE_READ_BODY;
+
   if (session_->huffman_aggregator()) {
     session_->huffman_aggregator()->AggregateTransactionCharacterCounts(
         *request_,
@@ -1086,6 +1171,9 @@ int HttpNetworkTransaction::DoReadBodyComplete(int result) {
   bool done = false;
   if (result <= 0) {
     DCHECK_NE(ERR_IO_PENDING, result);
+    stream_hash_->Finish();
+    if (result < 0)
+      return HandleIOError(result);
     done = true;
   }
 
@@ -1107,6 +1195,36 @@ int HttpNetworkTransaction::DoReadBodyComplete(int result) {
     }
   }
 
+  if (result > 0) {
+    received_body_length_ += result;
+    if (offset_) {
+      if (offset_ <= result) {
+         stream_hash_->Update((uint8*)read_buf_->data(), offset_);
+        if (stream_hash_->VerifyHash() != OK)
+          return HandleIOError(ERR_RETRY_HASH_MISMATCH);
+        stream_hash_->Update((uint8*)read_buf_->data() + offset_,
+                             result - offset_);
+        memmove(read_buf_->data(), read_buf_->data() + offset_,
+                result - offset_);
+        result -= offset_;
+        offset_ = 0;
+      }
+      else
+      {
+        stream_hash_->Update((uint8*)read_buf_->data(), result);
+        offset_ -= result;
+        result = 0;
+      }
+
+      if (result == 0) {
+        next_state_ = STATE_READ_BODY;
+        return result;
+      }
+    }
+    else
+      stream_hash_->Update((uint8*)read_buf_->data(), result);
+  }
+
   // Clean up connection if we are done.
   if (done) {
     LogTransactionMetrics();
@@ -1227,6 +1345,21 @@ void HttpNetworkTransaction::LogTransactionMetrics() const {
   }
 }
 
+void HttpNetworkTransaction::LogTransactionRetryMetrics(int error) const {
+  UMA_HISTOGRAM_CUSTOM_COUNTS("Net.HttpRetry.Count", retry_attempt_, 1, 5, 5);
+  if (request_->load_flags & LOAD_MAIN_FRAME) {
+    UMA_HISTOGRAM_CUSTOM_ENUMERATION(
+    "Net.HttpRetry.Cause.MainFrame",
+    -error,
+    GetAllErrorCodesForUma());
+  } else {
+    UMA_HISTOGRAM_CUSTOM_ENUMERATION(
+    "Net.HttpRetry.Cause.Subresource",
+    -error,
+    GetAllErrorCodesForUma());
+  }
+}
+
 int HttpNetworkTransaction::HandleCertificateRequest(int error) {
   // There are two paths through which the server can request a certificate
   // from us.  The first is during the initial handshake, the second is
@@ -1421,6 +1554,8 @@ int HttpNetworkTransaction::HandleIOError(int error) {
       if (ShouldResendRequest()) {
         net_log_.AddEventWithNetErrorCode(
             NetLog::TYPE_HTTP_TRANSACTION_RESTART_AFTER_ERROR, error);
+        retry_attempt_++;
+        LogTransactionRetryMetrics(error);
         ResetConnectionAndRequestForResend();
         error = OK;
       }
@@ -1462,27 +1597,99 @@ HttpResponseHeaders* HttpNetworkTransaction::GetResponseHeaders() const {
 }
 
 bool HttpNetworkTransaction::ShouldResendRequest() const {
-  bool connection_is_proven = stream_->IsConnectionReused();
-  bool has_received_headers = GetResponseHeaders() != NULL;
+  if (!stream_->IsConnectionReused() &&
+      !(request_->method == "GET" || request_->method == "HEAD"))

[rvargas (doing something else), 2014/07/26 00:30:33]

nit: requires {}

+    return false;
 
-  // NOTE: we resend a request only if we reused a keep-alive connection.
-  // This automatically prevents an infinite resend loop because we'll run
-  // out of the cached keep-alive connections eventually.
-  if (connection_is_proven && !has_received_headers)
-    return true;
-  return false;
-}
+  if (retry_attempt_ > kNumRetries)
+    return false;
 
-void HttpNetworkTransaction::ResetConnectionAndRequestForResend() {
-  if (stream_.get()) {
-    stream_->Close(true);
-    stream_.reset();
+  if (GetResponseHeaders() != NULL) {
+    // Do not retry automatically if resource is larger than 1 Mb until byte
+    // range support is enabled. This is temporary.
+    if (response_.headers->GetContentLength() > 1000000)
+      return false;
+
+    TimeDelta time_delta = base::TimeTicks::Now() - send_start_time_;
+    if (time_delta.InSeconds() > 30)
+      return false;
+
+    if (response_.headers->HasStrongValidators()) {

[rvargas (doing something else), 2014/07/26 00:30:33]

nit: no {}

+      return true;
+    }
+
+    if (response_.headers->HasHeaderValue("cache-control", "no-cache") ||
+        response_.headers->HasHeaderValue("cache-control", "no-store") ||
+        response_.headers->HasHeaderValue("cache-control", "must-revalidate") ||
+        response_.headers->HasHeaderValue("pragma", "no-cache")) {
+      return false;
+    }
+
+    if (response_.headers->GetMaxAgeValue(&time_delta))
+      return time_delta.InSeconds() > 60;
+
+    // If there is no Date header, then assume that the server response was
+    // generated at the time when we received the response, hence do not retry.
+    Time date_value;
+    if (!response_.headers->GetDateValue(&date_value))
+      return false;
+
+    Time expires_value;
+    if (response_.headers->GetExpiresValue(&expires_value)) {
+      time_delta = expires_value - date_value;
+      if (time_delta.InSeconds() > 60)
+        return true;
+    }
+    return false;
   }
+  return true;
+}
 
+void HttpNetworkTransaction::ResetConnectionAndRequestForResend() {
   // We need to clear request_headers_ because it contains the real request
   // headers, but we may need to resend the CONNECT request first to recreate
   // the SSL tunnel.
   request_headers_.Clear();
+  headers_valid_ = false;
+
+  if (stream_) {
+    total_received_bytes_ += stream_->GetTotalReceivedBytes();
+    if (received_body_length_ > offset_) {
+      offset_ = received_body_length_;
+    }
+  }
+  received_body_length_ = 0;
+
+  HttpResponseHeaders* headers = GetResponseHeaders();
+  if (offset_ && headers) {
+    previous_content_length_ = headers->GetContentLength();
+    headers->EnumerateHeader(NULL, "last-modified",
+                             &previous_last_modified_);
+    if (headers->GetHttpVersion() >= HttpVersion(1, 1))
+      headers->EnumerateHeader(NULL, "etag", &previous_etag_);
+
+    // Disable until we have statistics that show that retrying does not
+    // result in too many ERR_RETRY_HASH_MISMATCH.
+    if (0 && previous_content_length_ &&
+        headers->HasHeaderValue("Accept-Ranges", "bytes")) {
+      // Try resending using a range request if supported.
+      request_headers_.SetHeader(HttpRequestHeaders::kRange,
+          "bytes=" + base::Uint64ToString(offset_) + "-");
+      if (!previous_etag_.empty())
+        request_headers_.SetHeader("If-Match", previous_etag_);
+      else if (!previous_last_modified_.empty()) {
+        request_headers_.SetHeader("If-Unmodified-Since",
+            previous_last_modified_);
+      }
+    }
+  }
+
+  response_ = HttpResponseInfo();
+  establishing_tunnel_ = false;
+  if (stream_.get()) {
+    stream_->Close(true);
+    stream_.reset();
+  }
   next_state_ = STATE_CREATE_STREAM;  // Resend the request.
 }