stale-while-revalidate experimental implementation.

net/http/http_cache_transaction.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/http/http_cache_transaction.h"
 
 #include "build/build_config.h"
 
 #if defined(OS_POSIX)
 #include <unistd.h>
 #endif
 
 #include <algorithm>
 #include <string>
 
 #include "base/bind.h"
 #include "base/compiler_specific.h"
 #include "base/format_macros.h"
 #include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
 #include "base/metrics/field_trial.h"
 #include "base/metrics/histogram.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/rand_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_piece.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
+#include "base/values.h"
 #include "net/base/completion_callback.h"
 #include "net/base/io_buffer.h"
 #include "net/base/load_flags.h"
 #include "net/base/load_timing_info.h"
 #include "net/base/net_errors.h"
 #include "net/base/net_log.h"
 #include "net/base/upload_data_stream.h"
 #include "net/cert/cert_status_flags.h"
 #include "net/disk_cache/disk_cache.h"
 #include "net/http/disk_based_cert_cache.h"
 #include "net/http/http_network_session.h"
 #include "net/http/http_request_info.h"
 #include "net/http/http_response_headers.h"
 #include "net/http/http_transaction.h"
 #include "net/http/http_util.h"
 #include "net/http/partial_data.h"
 #include "net/ssl/ssl_cert_request_info.h"
 #include "net/ssl/ssl_config_service.h"
 
 using base::Time;
 using base::TimeDelta;
 using base::TimeTicks;
 
 namespace {
 
+typedef net::HttpResponseHeaders::ValidationType ValidationType;
+
 // TODO(ricea): Move this to HttpResponseHeaders once it is standardised.
 static const char kFreshnessHeader[] = "Resource-Freshness";
 
 // Stores data relevant to the statistics of writing and reading entire
 // certificate chains using DiskBasedCertCache. |num_pending_ops| is the number
 // of certificates in the chain that have pending operations in the
 // DiskBasedCertCache. |start_time| is the time that the read and write
 // commands began being issued to the DiskBasedCertCache.
 // TODO(brandonsalmon): Remove this when it is no longer necessary to
 // collect data.
@@ skipping 167 matching lines @@
 
 void RecordNoStoreHeaderHistogram(int load_flags,
                                   const net::HttpResponseInfo* response) {
   if (load_flags & net::LOAD_MAIN_FRAME) {
     UMA_HISTOGRAM_BOOLEAN(
         "Net.MainFrameNoStore",
         response->headers->HasHeaderValue("cache-control", "no-store"));
   }
 }
 
+base::Value* NetLogAsyncRevalidationInfoCallback(
+    const net::NetLog::Source& source,
+    const net::HttpRequestInfo* request,
+    net::NetLog::LogLevel log_level) {
+  base::DictionaryValue* dict = new base::DictionaryValue();
+  source.AddToEventParameters(dict);
+
+  dict->SetString("url", request->url.possibly_invalid_spec());
+  dict->SetString("method", request->method);
+  return dict;
+}
+
 }  // namespace
 
 namespace net {
 
 struct HeaderNameAndValue {
   const char* name;
   const char* value;
 };
 
 // If the request includes one of these request headers, then avoid caching
@@ skipping 1835 matching lines @@
 
   if (entry_->disk_entry->GetDataSize(kMetadataIndex))
     next_state_ = STATE_CACHE_READ_METADATA;
 
   return OK;
 }
 
 int HttpCache::Transaction::BeginCacheValidation() {
   DCHECK(mode_ == READ_WRITE);
 
-  bool skip_validation = !RequiresValidation();
+  ValidationType required_validation = RequiresValidation();
+
+  bool skip_validation = required_validation == ValidationType::VALIDATION_NONE;

[rvargas (doing something else), 2014/09/05 22:52:39]

nit: use () around ==

[Adam Rice, 2014/09/09 12:36:45]

Done.

+
+  if (required_validation == ValidationType::VALIDATION_ASYNCHRONOUS &&
+      !(request_->method == "GET" && (truncated_ || partial_)) && cache_ &&
+      cache_->use_stale_while_revalidate()) {
+    TriggerAsyncValidation();
+    skip_validation = true;
+  }
 
   if (request_->method == "HEAD" &&
       (truncated_ || response_.headers->response_code() == 206)) {
     DCHECK(!partial_);
     if (skip_validation)
       return SetupEntryForRead();
 
     // Bail out!
     next_state_ = STATE_SEND_REQUEST;
     mode_ = NONE;
     return OK;
   }
 
   if (truncated_) {
     // Truncated entries can cause partial gets, so we shouldn't record this
     // load in histograms.
     UpdateTransactionPattern(PATTERN_NOT_COVERED);
     skip_validation = !partial_->initial_validation();
   }
 
   if (partial_.get() && (is_sparse_ || truncated_) &&
       (!partial_->IsCurrentRangeCached() || invalid_range_)) {
     // Force revalidation for sparse or truncated entries. Note that we don't
     // want to ignore the regular validation logic just because a byte range was
     // part of the request.
     skip_validation = false;
   }
 
   if (skip_validation) {
+    // TODO(ricea): Is this pattern okay for asynchronous revalidations?
     UpdateTransactionPattern(PATTERN_ENTRY_USED);
     RecordOfflineStatus(effective_load_flags_, OFFLINE_STATUS_FRESH_CACHE);
     return SetupEntryForRead();
   } else {
     // Make the network request conditional, to see if we may reuse our cached
     // response.  If we cannot do so, then we just resort to a normal fetch.
     // Our mode remains READ_WRITE for a conditional request.  Even if the
     // conditionalization fails, we don't switch to WRITE mode until we
     // know we won't be falling back to using the cache entry in the
     // LOAD_FROM_CACHE_IF_OFFLINE case.
@@ skipping 119 matching lines @@
   DCHECK(network_trans_.get());
   DCHECK_EQ(STATE_NONE, next_state_);
 
   next_state_ = STATE_SEND_REQUEST_COMPLETE;
   int rv = network_trans_->RestartWithAuth(credentials, io_callback_);
   if (rv != ERR_IO_PENDING)
     return DoLoop(rv);
   return rv;
 }
 
-bool HttpCache::Transaction::RequiresValidation() {
+ValidationType HttpCache::Transaction::RequiresValidation() {
   // TODO(darin): need to do more work here:
   //  - make sure we have a matching request method
   //  - watch out for cached responses that depend on authentication
 
   // In playback mode, nothing requires validation.
   if (cache_->mode() == net::HttpCache::PLAYBACK)
-    return false;
+    return ValidationType::VALIDATION_NONE;
 
   if (response_.vary_data.is_valid() &&
       !response_.vary_data.MatchesRequest(*request_,
                                           *response_.headers.get())) {
     vary_mismatch_ = true;
-    return true;
+    return ValidationType::VALIDATION_SYNCHRONOUS;
   }
 
   if (effective_load_flags_ & LOAD_PREFERRING_CACHE)
-    return false;
+    return ValidationType::VALIDATION_NONE;

[rvargas (doing something else), 2014/09/05 22:52:40]

I think we should move the enum out of ResponseHeaders. It clearly has to be
used by other parts of the code when not interacting directly with response
headers and it feels weird that load flags can be used as they are and
validation type cannot. Besides, it is redundant to effectively have
'validation' twice in the name of each value.

(just to be clear, defining the enum in http_response_headers.h is fine)

[Adam Rice, 2014/09/09 12:36:45]

Done.

 
-  if (effective_load_flags_ & LOAD_VALIDATE_CACHE)
-    return true;
+  if (effective_load_flags_ & (LOAD_VALIDATE_CACHE | LOAD_ASYNC_REVALIDATION))

[rvargas (doing something else), 2014/09/05 22:52:39]

I just realized, can't we use LOAD_VALIDATE_CACHE instead of defining the new
flag? This one doesn't add headers to blast proxies, and it also makes sense to
ignore SWR for user initiated reloads.

[Adam Rice, 2014/09/09 12:36:45]

I tried that. It does add headers. See http_network_transaction.cc:880.

+    return ValidationType::VALIDATION_SYNCHRONOUS;
 
   if (request_->method == "PUT" || request_->method == "DELETE")
-    return true;
+    return ValidationType::VALIDATION_SYNCHRONOUS;
 
-  if (response_.headers->RequiresValidation(
-          response_.request_time, response_.response_time, Time::Now())) {
-    return true;
+  ValidationType validation_required_by_headers =
+      response_.headers->RequiresValidation(
+          response_.request_time, response_.response_time, Time::Now());
+
+  if (validation_required_by_headers ==
+      ValidationType::VALIDATION_ASYNCHRONOUS) {
+    // Asynchronous revalidation is only supported for GET and HEAD methods.
+    if (request_->method != "GET" && request_->method != "HEAD")
+      return ValidationType::VALIDATION_SYNCHRONOUS;
   }
 
-  return false;
+  return validation_required_by_headers;
 }
 
 bool HttpCache::Transaction::ConditionalizeRequest() {
   DCHECK(response_.headers.get());
 
   if (request_->method == "PUT" || request_->method == "DELETE")
     return false;
 
   // This only makes sense for cached 200 or 206 responses.
   if (response_.headers->response_code() != 200 &&
@@ skipping 31 matching lines @@
   bool use_if_range = partial_.get() && !partial_->IsCurrentRangeCached() &&
                       !invalid_range_;
 
   if (!use_if_range) {
     // stale-while-revalidate is not useful when we only have a partial response
     // cached, so don't set the header in that case.
     TimeDelta stale_while_revalidate;
     if (response_.headers->GetStaleWhileRevalidateValue(
             &stale_while_revalidate) &&
         stale_while_revalidate > TimeDelta()) {
-      TimeDelta max_age =
+      HttpResponseHeaders::Freshness freshness =
           response_.headers->GetFreshnessLifetime(response_.response_time);
       TimeDelta current_age = response_.headers->GetCurrentAge(
           response_.request_time, response_.response_time, Time::Now());
 
       custom_request_->extra_headers.SetHeader(
           kFreshnessHeader,
           base::StringPrintf("max-age=%" PRId64
                              ",stale-while-revalidate=%" PRId64 ",age=%" PRId64,
-                             max_age.InSeconds(),
+                             freshness.lifetime.InSeconds(),
                              stale_while_revalidate.InSeconds(),
                              current_age.InSeconds()));
     }
   }
 
   if (!etag_value.empty()) {
     if (use_if_range) {
       // We don't want to switch to WRITE mode if we don't have this block of a
       // byte-range request because we may have other parts cached.
       custom_request_->extra_headers.SetHeader(
@@ skipping 149 matching lines @@
 }
 
 void HttpCache::Transaction::FixHeadersForHead() {
   if (response_.headers->response_code() == 206) {
     response_.headers->RemoveHeader("Content-Length");
     response_.headers->RemoveHeader("Content-Range");
     response_.headers->ReplaceStatusLine("HTTP/1.1 200 OK");
   }
 }
 
+void HttpCache::Transaction::TriggerAsyncValidation() {
+  DCHECK(!request_->upload_data_stream);
+  BoundNetLog async_revalidation_net_log(
+      BoundNetLog::Make(net_log_.net_log(), NetLog::SOURCE_ASYNC_REVALIDATION));
+  net_log_.AddEvent(
+      NetLog::TYPE_HTTP_CACHE_VALIDATE_RESOURCE_ASYNC,
+      async_revalidation_net_log.source().ToEventParametersCallback());
+  async_revalidation_net_log.BeginEvent(
+      NetLog::TYPE_ASYNC_REVALIDATION,
+      base::Bind(
+          &NetLogAsyncRevalidationInfoCallback, net_log_.source(), request_));
+  // cache_ is a weak pointer.

[rvargas (doing something else), 2014/09/05 22:52:40]

nit: move comment inline with the argument

[Adam Rice, 2014/09/09 12:36:45]

Done.

+  base::MessageLoop::current()->PostTask(
+      FROM_HERE,
+      base::Bind(&HttpCache::PerformAsyncValidation,
+                 cache_,
+                 *request_,
+                 async_revalidation_net_log));
+}
+
 void HttpCache::Transaction::FailRangeRequest() {
   response_ = *new_response_;
   partial_->FixResponseHeaders(response_.headers.get(), false);
 }
 
 int HttpCache::Transaction::SetupEntryForRead() {
   if (network_trans_)
     ResetNetworkTransaction();
   if (partial_.get()) {
     if (truncated_ || is_sparse_ || !invalid_range_) {
@@ skipping 326 matching lines @@
     default:
       NOTREACHED();
   }
 }
 
 void HttpCache::Transaction::OnIOComplete(int result) {
   DoLoop(result);
 }
 
 }  // namespace net