stale-while-revalidate experimental implementation.

net/http/http_cache_transaction.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/http/http_cache_transaction.h"
 
 #include "build/build_config.h"
 
 #if defined(OS_POSIX)
 #include <unistd.h>
 #endif
 
 #include <algorithm>
 #include <string>
 
 #include "base/bind.h"
 #include "base/compiler_specific.h"
 #include "base/format_macros.h"
 #include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
 #include "base/metrics/field_trial.h"
 #include "base/metrics/histogram.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/rand_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_piece.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/time/time.h"
+#include "base/values.h"
 #include "net/base/completion_callback.h"
 #include "net/base/io_buffer.h"
 #include "net/base/load_flags.h"
 #include "net/base/load_timing_info.h"
 #include "net/base/net_errors.h"
 #include "net/base/net_log.h"
 #include "net/base/upload_data_stream.h"
 #include "net/cert/cert_status_flags.h"
 #include "net/disk_cache/disk_cache.h"
 #include "net/http/disk_based_cert_cache.h"
 #include "net/http/http_network_session.h"
 #include "net/http/http_request_info.h"
 #include "net/http/http_response_headers.h"
 #include "net/http/http_transaction.h"
 #include "net/http/http_util.h"
 #include "net/http/partial_data.h"
 #include "net/ssl/ssl_cert_request_info.h"
 #include "net/ssl/ssl_config_service.h"
 
 using base::Time;
 using base::TimeDelta;
 using base::TimeTicks;
 
 namespace {
 
+typedef net::HttpResponseHeaders::ValidationType ValidationType;
+const ValidationType VALIDATION_NONE = ValidationType::VALIDATION_NONE;

[rvargas (doing something else), 2014/08/29 23:36:07]

I think we should just live with HttpResponseHeaders::VALIDATION_NONE instead.
If that is too ugly, move the enum out of the class.

[Adam Rice, 2014/09/01 14:43:00]

Done.

+const ValidationType VALIDATION_SYNCHRONOUS =
+    ValidationType::VALIDATION_SYNCHRONOUS;
+const ValidationType VALIDATION_ASYNCHRONOUS =
+    ValidationType::VALIDATION_ASYNCHRONOUS;
+
 // TODO(ricea): Move this to HttpResponseHeaders once it is standardised.
 static const char kFreshnessHeader[] = "Resource-Freshness";
 
 // Stores data relevant to the statistics of writing and reading entire
 // certificate chains using DiskBasedCertCache. |num_pending_ops| is the number
 // of certificates in the chain that have pending operations in the
 // DiskBasedCertCache. |start_time| is the time that the read and write
 // commands began being issued to the DiskBasedCertCache.
 // TODO(brandonsalmon): Remove this when it is no longer necessary to
 // collect data.
@@ skipping 167 matching lines @@
 
 void RecordNoStoreHeaderHistogram(int load_flags,
                                   const net::HttpResponseInfo* response) {
   if (load_flags & net::LOAD_MAIN_FRAME) {
     UMA_HISTOGRAM_BOOLEAN(
         "Net.MainFrameNoStore",
         response->headers->HasHeaderValue("cache-control", "no-store"));
   }
 }
 
+base::Value* NetLogAsyncRevalidationInfoCallback(
+    const net::NetLog::Source& source,
+    const net::HttpRequestInfo* request,
+    net::NetLog::LogLevel log_level) {
+  base::DictionaryValue* dict = new base::DictionaryValue();
+  source.AddToEventParameters(dict);
+
+  dict->SetString("url", request->url.possibly_invalid_spec());
+  dict->SetString("method", request->method);
+  return dict;
+}
+
 }  // namespace
 
 namespace net {
 
 struct HeaderNameAndValue {
   const char* name;
   const char* value;
 };
 
 // If the request includes one of these request headers, then avoid caching
@@ skipping 154 matching lines @@
     return network_trans_->GetLoadState();
   if (entry_ || !request_)
     return LOAD_STATE_IDLE;
   return LOAD_STATE_WAITING_FOR_CACHE;
 }
 
 const BoundNetLog& HttpCache::Transaction::net_log() const {
   return net_log_;
 }
 
+void HttpCache::Transaction::ValidateInternalCache() {
+  DCHECK(!request_);  // Start() should not have been called.
+  effective_load_flags_ |= LOAD_VALIDATE_CACHE;
+}
+
 int HttpCache::Transaction::Start(const HttpRequestInfo* request,
                                   const CompletionCallback& callback,
                                   const BoundNetLog& net_log) {
   DCHECK(request);
   DCHECK(!callback.is_null());
 
   // Ensure that we only have one asynchronous call at a time.
   DCHECK(callback_.is_null());
   DCHECK(!reading_);
   DCHECK(!network_trans_.get());
@@ skipping 1506 matching lines @@
                                                     false /* is not leaf */,
                                                     shared_chain_data));
   }
   DCHECK_EQ(0, dist_from_root);
 }
 
 void HttpCache::Transaction::SetRequest(const BoundNetLog& net_log,
                                         const HttpRequestInfo* request) {
   net_log_ = net_log;
   request_ = request;
-  effective_load_flags_ = request_->load_flags;
+  // effective_load_flags_ may already have had LOAD_VALIDATE_CACHE set by the
+  // ValidateInternalCache() method.
+  effective_load_flags_ |= request_->load_flags;
 
   switch (cache_->mode()) {
     case NORMAL:
       break;
     case RECORD:
       // When in record mode, we want to NEVER load from the cache.
       // The reason for this is because we save the Set-Cookie headers
       // (intentionally).  If we read from the cache, we replay them
       // prematurely.
       effective_load_flags_ |= LOAD_BYPASS_CACHE;
@@ skipping 134 matching lines @@
 
   if (entry_->disk_entry->GetDataSize(kMetadataIndex))
     next_state_ = STATE_CACHE_READ_METADATA;
 
   return OK;
 }
 
 int HttpCache::Transaction::BeginCacheValidation() {
   DCHECK(mode_ == READ_WRITE);
 
-  bool skip_validation = !RequiresValidation();
+  ValidationType required_validation = RequiredValidationType();
+
+  bool skip_validation = required_validation == VALIDATION_NONE;
+
+  if (required_validation == VALIDATION_ASYNCHRONOUS && cache_ &&
+      cache_->use_stale_while_revalidate()) {
+    TriggerAsyncValidation();
+    skip_validation = true;
+  }
 
   if (request_->method == "HEAD" &&
       (truncated_ || response_.headers->response_code() == 206)) {
     DCHECK(!partial_);
     if (skip_validation)
       return SetupEntryForRead();
 
     // Bail out!
     next_state_ = STATE_SEND_REQUEST;
     mode_ = NONE;
     return OK;
   }
 
   if (truncated_) {
     // Truncated entries can cause partial gets, so we shouldn't record this
     // load in histograms.
     UpdateTransactionPattern(PATTERN_NOT_COVERED);
     skip_validation = !partial_->initial_validation();
   }
 
   if (partial_.get() && (is_sparse_ || truncated_) &&
       (!partial_->IsCurrentRangeCached() || invalid_range_)) {
     // Force revalidation for sparse or truncated entries. Note that we don't
     // want to ignore the regular validation logic just because a byte range was
     // part of the request.
     skip_validation = false;
   }
 
   if (skip_validation) {
+    // TODO(ricea): Is this pattern okay for asynchronous revalidations?
     UpdateTransactionPattern(PATTERN_ENTRY_USED);
     RecordOfflineStatus(effective_load_flags_, OFFLINE_STATUS_FRESH_CACHE);
     return SetupEntryForRead();
   } else {
     // Make the network request conditional, to see if we may reuse our cached
     // response.  If we cannot do so, then we just resort to a normal fetch.
     // Our mode remains READ_WRITE for a conditional request.  Even if the
     // conditionalization fails, we don't switch to WRITE mode until we
     // know we won't be falling back to using the cache entry in the
     // LOAD_FROM_CACHE_IF_OFFLINE case.
@@ skipping 119 matching lines @@
   DCHECK(network_trans_.get());
   DCHECK_EQ(STATE_NONE, next_state_);
 
   next_state_ = STATE_SEND_REQUEST_COMPLETE;
   int rv = network_trans_->RestartWithAuth(credentials, io_callback_);
   if (rv != ERR_IO_PENDING)
     return DoLoop(rv);
   return rv;
 }
 
-bool HttpCache::Transaction::RequiresValidation() {
+ValidationType HttpCache::Transaction::RequiredValidationType() {
   // TODO(darin): need to do more work here:
   //  - make sure we have a matching request method
   //  - watch out for cached responses that depend on authentication
 
   // In playback mode, nothing requires validation.
   if (cache_->mode() == net::HttpCache::PLAYBACK)
-    return false;
+    return VALIDATION_NONE;
 
   if (response_.vary_data.is_valid() &&
       !response_.vary_data.MatchesRequest(*request_,
                                           *response_.headers.get())) {
     vary_mismatch_ = true;
-    return true;
+    return VALIDATION_SYNCHRONOUS;
   }
 
   if (effective_load_flags_ & LOAD_PREFERRING_CACHE)
-    return false;
+    return VALIDATION_NONE;
 
   if (effective_load_flags_ & LOAD_VALIDATE_CACHE)
-    return true;
+    return VALIDATION_SYNCHRONOUS;
 
   if (request_->method == "PUT" || request_->method == "DELETE")
-    return true;
+    return VALIDATION_SYNCHRONOUS;
 
-  if (response_.headers->RequiresValidation(
-          response_.request_time, response_.response_time, Time::Now())) {
-    return true;
+  ValidationType validation_required_by_headers =
+      response_.headers->RequiresValidation(

[rvargas (doing something else), 2014/08/29 23:36:06]

If you keep the name for ResponseHeaders::RequiresValidation (which is fine),
you should also keep the name for Transaction::RequiresValidation (conceptually
they do the same thing)

[Adam Rice, 2014/09/01 14:43:00]

Done.

+          response_.request_time, response_.response_time, Time::Now());
+
+  if (validation_required_by_headers == VALIDATION_ASYNCHRONOUS) {
+    // Asynchronous revalidation is only supported for GET and HEAD methods.
+    if (request_->method != "GET" && request_->method != "HEAD")
+      return VALIDATION_SYNCHRONOUS;
+
+    // Asynchronous revalidation of GETs can only be used for completely cached
+    // resources.
+    if (request_->method == "GET" && (truncated_ || partial_))

[rvargas (doing something else), 2014/08/29 23:36:06]

This part belong on the caller side.

[Adam Rice, 2014/09/01 14:43:00]

Done.

+      return VALIDATION_SYNCHRONOUS;
   }
 
-  return false;
+  return validation_required_by_headers;
 }
 
 bool HttpCache::Transaction::ConditionalizeRequest() {
   DCHECK(response_.headers.get());
 
   if (request_->method == "PUT" || request_->method == "DELETE")
     return false;
 
   // This only makes sense for cached 200 or 206 responses.
   if (response_.headers->response_code() != 200 &&
@@ skipping 28 matching lines @@
   }
   DCHECK(custom_request_.get());
 
   bool use_if_range = partial_.get() && !partial_->IsCurrentRangeCached() &&
                       !invalid_range_;
 
   if (!use_if_range) {
     // stale-while-revalidate is not useful when we only have a partial response
     // cached, so don't set the header in that case.
     TimeDelta stale_while_revalidate;
+    TimeDelta max_age;
     if (response_.headers->GetStaleWhileRevalidateValue(
             &stale_while_revalidate) &&
+        response_.headers->GetFreshnessLifetime(response_.response_time,

[rvargas (doing something else), 2014/08/29 23:36:07]

This pattern reads very weird: returning a value that looks time related and the
actual time through an out argument, especially considering the name of the
method.

[Adam Rice, 2014/09/01 14:43:00]

I have attempted to improve it. PTAL.

+                                                &max_age) !=
+            HttpResponseHeaders::NEVER_FRESH &&
         stale_while_revalidate > TimeDelta()) {
-      TimeDelta max_age =
-          response_.headers->GetFreshnessLifetime(response_.response_time);
       TimeDelta current_age = response_.headers->GetCurrentAge(
           response_.request_time, response_.response_time, Time::Now());
 
       custom_request_->extra_headers.SetHeader(
           kFreshnessHeader,
           base::StringPrintf("max-age=%" PRId64
                              ",stale-while-revalidate=%" PRId64 ",age=%" PRId64,
                              max_age.InSeconds(),
                              stale_while_revalidate.InSeconds(),
                              current_age.InSeconds()));
@@ skipping 157 matching lines @@
 }
 
 void HttpCache::Transaction::FixHeadersForHead() {
   if (response_.headers->response_code() == 206) {
     response_.headers->RemoveHeader("Content-Length");
     response_.headers->RemoveHeader("Content-Range");
     response_.headers->ReplaceStatusLine("HTTP/1.1 200 OK");
   }
 }
 
+void HttpCache::Transaction::TriggerAsyncValidation() {
+  DCHECK(!request_->upload_data_stream);
+  BoundNetLog async_revalidation_net_log(
+      BoundNetLog::Make(net_log_.net_log(), NetLog::SOURCE_ASYNC_REVALIDATION));
+  net_log_.AddEvent(
+      NetLog::TYPE_HTTP_CACHE_VALIDATE_RESOURCE_ASYNC,
+      async_revalidation_net_log.source().ToEventParametersCallback());
+  async_revalidation_net_log.BeginEvent(
+      NetLog::TYPE_ASYNC_REVALIDATION,
+      base::Bind(
+          &NetLogAsyncRevalidationInfoCallback, net_log_.source(), request_));
+  base::MessageLoop::current()->PostTask(
+      FROM_HERE,
+      base::Bind(&HttpCache::PerformAsyncValidation,
+                 cache_,

[rvargas (doing something else), 2014/08/29 23:36:06]

Is the intention here to use a weak pointer? (I bet it is).

[Adam Rice, 2014/09/01 14:43:00]

Yes. I have added a comment.

+                 *request_,
+                 async_revalidation_net_log));
+}
+
 void HttpCache::Transaction::FailRangeRequest() {
   response_ = *new_response_;
   partial_->FixResponseHeaders(response_.headers.get(), false);
 }
 
 int HttpCache::Transaction::SetupEntryForRead() {
   if (network_trans_)
     ResetNetworkTransaction();
   if (partial_.get()) {
     if (truncated_ || is_sparse_ || !invalid_range_) {
@@ skipping 326 matching lines @@
     default:
       NOTREACHED();
   }
 }
 
 void HttpCache::Transaction::OnIOComplete(int result) {
   DoLoop(result);
 }
 
 }  // namespace net