content::ResourceDispatcherHostImpl changes for stale-while-revalidate

content/browser/loader/async_revalidation_manager_unittest.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/loader/async_revalidation_manager.h"
+
+#include <queue>
+#include <utility>
+
+#include "base/bind.h"
+#include "base/callback.h"
+#include "base/macros.h"
+#include "base/memory/shared_memory_handle.h"
+#include "base/pickle.h"
+#include "base/run_loop.h"
+#include "content/browser/child_process_security_policy_impl.h"
+#include "content/browser/loader/resource_dispatcher_host_impl.h"
+#include "content/browser/loader/resource_message_filter.h"
+#include "content/common/child_process_host_impl.h"
+#include "content/common/resource_messages.h"
+#include "content/public/browser/resource_context.h"
+#include "content/public/common/appcache_info.h"
+#include "content/public/common/process_type.h"
+#include "content/public/common/resource_type.h"
+#include "content/public/test/test_browser_context.h"
+#include "content/public/test/test_browser_thread_bundle.h"
+#include "ipc/ipc_param_traits.h"
+#include "net/base/load_flags.h"
+#include "net/base/network_delegate.h"
+#include "net/http/http_util.h"
+#include "net/url_request/url_request.h"
+#include "net/url_request/url_request_job.h"
+#include "net/url_request/url_request_job_factory.h"
+#include "net/url_request/url_request_test_job.h"
+#include "net/url_request/url_request_test_util.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "ui/base/page_transition_types.h"
+#include "url/gurl.h"
+#include "url/url_constants.h"
+
+namespace content {
+
+namespace {
+
+// This class is a variation on URLRequestTestJob that
+// returns ERR_IO_PENDING before every read, not just the first one.
+class URLRequestTestDelayedCompletionJob : public net::URLRequestTestJob {
+ public:
+  URLRequestTestDelayedCompletionJob(net::URLRequest* request,
+                                     net::NetworkDelegate* network_delegate,
+                                     const std::string& response_headers,
+                                     const std::string& response_data)
+      : net::URLRequestTestJob(request,
+                               network_delegate,
+                               response_headers,
+                               response_data,
+                               false) {}
+
+ private:
+  ~URLRequestTestDelayedCompletionJob() override {}
+
+  bool NextReadAsync() override { return true; }
+
+  DISALLOW_COPY_AND_ASSIGN(URLRequestTestDelayedCompletionJob);
+};
+
+class TestURLRequestJobFactory : public net::URLRequestJobFactory {
+ public:
+  using URLRequestJobCreateCallback =
+      base::Callback<net::URLRequestJob*(net::URLRequest*,
+                                         net::NetworkDelegate*)>;

[davidben, 2015/12/07 23:56:04]

Since you only ever create two job types and one of their create functions
depends on the other anyway, why not instead have your
TestURLRequestJobFactory::MaybeCreate... method do the string check and have
your tests either include or not include "redirect" in the URL as-needed? That
would save you all this callback machinery and a somewhat odd
RedirectAndRevalidate...::Create method.

[Adam Rice, 2015/12/08 18:05:35]

Okay, done. That's saved quite a bit of code.

+
+  TestURLRequestJobFactory() = default;
+
+  // Sets the contents of the response. |headers| should have "\n" as line
+  // breaks and end in "\n\n".
+  void SetResponse(const std::string& headers, const std::string& data) {
+    response_headers_ = headers;
+    response_data_ = data;
+  }
+
+  void SetCustomURLRequestJobCreateCallback(
+      const URLRequestJobCreateCallback& callback) {
+    custom_url_request_job_create_callback_ = callback;
+  }
+
+  net::URLRequestJob* MaybeCreateJobWithProtocolHandler(
+      const std::string& scheme,
+      net::URLRequest* request,
+      net::NetworkDelegate* network_delegate) const override {
+    if (!custom_url_request_job_create_callback_.is_null()) {
+      return custom_url_request_job_create_callback_.Run(request,
+                                                         network_delegate);
+    }
+
+    return new URLRequestTestDelayedCompletionJob(
+        request, network_delegate, response_headers_, response_data_);
+  }
+
+  net::URLRequestJob* MaybeInterceptRedirect(
+      net::URLRequest* request,
+      net::NetworkDelegate* network_delegate,
+      const GURL& location) const override {
+    return nullptr;
+  }
+
+  net::URLRequestJob* MaybeInterceptResponse(
+      net::URLRequest* request,
+      net::NetworkDelegate* network_delegate) const override {
+    return nullptr;
+  }
+
+  bool IsHandledProtocol(const std::string& scheme) const override {
+    // If non-standard schemes need to be tested in future it will be
+    // necessary to call ChildProcessSecurityPolicyImpl::
+    // RegisterWebSafeScheme() for them.
+    return scheme == url::kHttpScheme || scheme == url::kHttpsScheme;
+  }
+
+  bool IsHandledURL(const GURL& url) const override {
+    return IsHandledProtocol(url.scheme());
+  }
+
+  bool IsSafeRedirectTarget(const GURL& location) const override {
+    return false;
+  }
+
+ private:
+  std::string response_headers_;
+  std::string response_data_;
+  URLRequestJobCreateCallback custom_url_request_job_create_callback_;
+
+  DISALLOW_COPY_AND_ASSIGN(TestURLRequestJobFactory);
+};
+
+// On Windows, ResourceMsg_SetDataBuffer supplies a HANDLE which is not
+// automatically released.
+//
+// See ResourceDispatcher::ReleaseResourcesInDataMessage.
+//
+// TODO(ricea): Maybe share this implementation with
+// resource_dispatcher_host_unittest.cc
+void ReleaseHandlesInMessage(const IPC::Message& message) {
+  if (message.type() == ResourceMsg_SetDataBuffer::ID) {
+    base::PickleIterator iter(message);
+    int request_id;
+    CHECK(iter.ReadInt(&request_id));
+    base::SharedMemoryHandle shm_handle;
+    if (IPC::ParamTraits<base::SharedMemoryHandle>::Read(&message, &iter,
+                                                         &shm_handle)) {
+      if (base::SharedMemory::IsHandleValid(shm_handle))
+        base::SharedMemory::CloseHandle(shm_handle);
+    }
+  }
+}
+
+// This filter just deletes any messages that are sent through it.
+class BlackholeFilter : public ResourceMessageFilter {
+ public:
+  explicit BlackholeFilter(ResourceContext* resource_context)
+      : ResourceMessageFilter(
+            ChildProcessHostImpl::GenerateChildProcessUniqueId(),
+            PROCESS_TYPE_RENDERER,
+            nullptr,
+            nullptr,
+            nullptr,
+            nullptr,
+            nullptr,
+            base::Bind(&BlackholeFilter::GetContexts, base::Unretained(this))),
+        resource_context_(resource_context) {
+    ChildProcessSecurityPolicyImpl::GetInstance()->Add(child_id());
+  }
+
+  bool Send(IPC::Message* msg) override {
+    scoped_ptr<IPC::Message> take_ownership(msg);
+    ReleaseHandlesInMessage(*msg);
+    return true;
+  }
+
+ private:
+  ~BlackholeFilter() override {
+    ChildProcessSecurityPolicyImpl::GetInstance()->Remove(child_id());
+  }
+
+  void GetContexts(ResourceType resource_type,
+                   int origin_pid,
+                   ResourceContext** resource_context,
+                   net::URLRequestContext** request_context) {
+    *resource_context = resource_context_;
+    *request_context = resource_context_->GetRequestContext();
+  }
+
+  ResourceContext* resource_context_;
+
+  DISALLOW_COPY_AND_ASSIGN(BlackholeFilter);
+};
+
+ResourceHostMsg_Request CreateResourceRequest(const char* method,
+                                              ResourceType type,
+                                              const GURL& url) {
+  ResourceHostMsg_Request request;
+  request.method = std::string(method);
+  request.url = url;
+  request.first_party_for_cookies = url;  // bypass third-party cookie blocking

[davidben, 2015/12/07 23:56:04]

Nit: Capitalize and end with period.

[Adam Rice, 2015/12/08 18:05:35]

Done.

+  request.referrer_policy = blink::WebReferrerPolicyDefault;
+  request.load_flags = 0;
+  request.origin_pid = 0;
+  request.resource_type = type;
+  request.request_context = 0;
+  request.appcache_host_id = kAppCacheNoHostId;
+  request.download_to_file = false;
+  request.should_reset_appcache = false;
+  request.is_main_frame = true;
+  request.parent_is_main_frame = false;
+  request.parent_render_frame_id = -1;
+  request.transition_type = ui::PAGE_TRANSITION_LINK;
+  request.allow_download = true;
+  return request;
+}
+
+class AsyncRevalidationManagerTest : public ::testing::Test {
+ protected:
+  AsyncRevalidationManagerTest(
+      scoped_ptr<net::TestNetworkDelegate> network_delegate)
+      : thread_bundle_(content::TestBrowserThreadBundle::IO_MAINLOOP),
+        network_delegate_(std::move(network_delegate)) {
+    browser_context_.reset(new TestBrowserContext());
+    BrowserContext::EnsureResourceContextInitialized(browser_context_.get());
+    base::RunLoop().RunUntilIdle();
+    ResourceContext* resource_context = browser_context_->GetResourceContext();
+    filter_ = new BlackholeFilter(resource_context);
+    net::URLRequestContext* request_context =
+        resource_context->GetRequestContext();
+    job_factory_.reset(new TestURLRequestJobFactory);
+    request_context->set_job_factory(job_factory_.get());
+    request_context->set_network_delegate(network_delegate_.get());
+    host_.EnableStaleWhileRevalidateForTesting();
+  }
+
+  AsyncRevalidationManagerTest()
+      : AsyncRevalidationManagerTest(
+            make_scoped_ptr(new net::TestNetworkDelegate)) {}
+
+  void TearDown() override {
+    host_.CancelRequestsForProcess(filter_->child_id());
+    host_.Shutdown();
+    host_.CancelRequestsForContext(browser_context_->GetResourceContext());
+    browser_context_.reset();
+    base::RunLoop().RunUntilIdle();
+  }
+
+  void SetResponse(const std::string& headers, const std::string& data) {
+    job_factory_->SetResponse(headers, data);
+  }
+
+  void SetCustomURLRequestJobCreateCallback(
+      const TestURLRequestJobFactory::URLRequestJobCreateCallback& callback) {
+    job_factory_->SetCustomURLRequestJobCreateCallback(callback);
+  }
+
+  // Creates a request using the current test object as the filter and
+  // SubResource as the resource type.
+  void MakeTestRequest(int render_view_id, int request_id, const GURL& url) {
+    ResourceHostMsg_Request request =
+        CreateResourceRequest("GET", RESOURCE_TYPE_SUB_RESOURCE, url);
+    ResourceHostMsg_RequestResource msg(render_view_id, request_id, request);
+    host_.OnMessageReceived(msg, filter_.get());
+    base::RunLoop().RunUntilIdle();
+  }
+
+  void EnsureSchemeIsAllowed(const std::string& scheme) {
+    ChildProcessSecurityPolicyImpl* policy =
+        ChildProcessSecurityPolicyImpl::GetInstance();
+    if (!policy->IsWebSafeScheme(scheme))
+      policy->RegisterWebSafeScheme(scheme);
+  }
+
+  content::TestBrowserThreadBundle thread_bundle_;
+  scoped_ptr<TestBrowserContext> browser_context_;
+  scoped_ptr<TestURLRequestJobFactory> job_factory_;
+  scoped_refptr<BlackholeFilter> filter_;
+  scoped_ptr<net::TestNetworkDelegate> network_delegate_;
+  ResourceDispatcherHostImpl host_;
+};
+
+TEST_F(AsyncRevalidationManagerTest, SupportsAsyncRevalidation) {
+  SetResponse(net::URLRequestTestJob::test_headers(), "delay complete");
+  MakeTestRequest(0, 1, GURL("http://example.com/baz"));
+
+  net::URLRequest* url_request(
+      host_.GetURLRequest(GlobalRequestID(filter_->child_id(), 1)));
+  ASSERT_TRUE(url_request);
+
+  EXPECT_TRUE(url_request->load_flags() & net::LOAD_SUPPORT_ASYNC_REVALIDATION);
+}
+
+TEST_F(AsyncRevalidationManagerTest, AsyncRevalidationNotSupportedForPOST) {
+  SetResponse(net::URLRequestTestJob::test_headers(), "delay complete");
+  // Create POST request.
+  ResourceHostMsg_Request request = CreateResourceRequest(
+      "POST", RESOURCE_TYPE_SUB_RESOURCE, GURL("http://example.com/baz.php"));
+  ResourceHostMsg_RequestResource msg(0, 1, request);
+  host_.OnMessageReceived(msg, filter_.get());
+  base::RunLoop().RunUntilIdle();
+
+  net::URLRequest* url_request(
+      host_.GetURLRequest(GlobalRequestID(filter_->child_id(), 1)));
+  ASSERT_TRUE(url_request);
+
+  EXPECT_FALSE(url_request->load_flags() &
+               net::LOAD_SUPPORT_ASYNC_REVALIDATION);
+}
+
+TEST_F(AsyncRevalidationManagerTest,
+       AsyncRevalidationNotSupportedAfterRedirect) {
+  static const char kRedirectHeaders[] =
+      "HTTP/1.1 302 MOVED\n"
+      "Location: http://example.com/var\n"
+      "\n";
+  SetResponse(kRedirectHeaders, "");
+
+  MakeTestRequest(0, 1, GURL("http://example.com/baz"));
+
+  net::URLRequest* url_request(
+      host_.GetURLRequest(GlobalRequestID(filter_->child_id(), 1)));
+  ASSERT_TRUE(url_request);
+
+  EXPECT_FALSE(url_request->load_flags() &
+               net::LOAD_SUPPORT_ASYNC_REVALIDATION);
+}
+
+// A URLRequestJob implementation which sets the |async_revalidation_required|
+// flag on the HttpResponseInfo object to true if the request has the
+// LOAD_SUPPORT_ASYNC_REVALIDATION flag.
+class AsyncRevalidationRequiredURLRequestTestJob
+    : public net::URLRequestTestJob {
+ public:
+  // The Create() method is useful for wrapping the construction of the object
+  // in a Callback.
+  static net::URLRequestJob* Create(net::URLRequest* request,
+                                    net::NetworkDelegate* network_delegate) {
+    return new AsyncRevalidationRequiredURLRequestTestJob(request,
+                                                          network_delegate);
+  }
+
+  void GetResponseInfo(net::HttpResponseInfo* info) override {
+    URLRequestTestJob::GetResponseInfo(info);
+    if (request()->load_flags() & net::LOAD_SUPPORT_ASYNC_REVALIDATION)
+      info->async_revalidation_required = true;
+  }
+
+ private:
+  AsyncRevalidationRequiredURLRequestTestJob(
+      net::URLRequest* request,
+      net::NetworkDelegate* network_delegate)
+      : URLRequestTestJob(request,
+                          network_delegate,
+                          net::URLRequestTestJob::test_headers(),
+                          std::string(),
+                          false) {}
+
+  ~AsyncRevalidationRequiredURLRequestTestJob() override {}
+
+  DISALLOW_COPY_AND_ASSIGN(AsyncRevalidationRequiredURLRequestTestJob);
+};
+
+// A URLRequestJob implementation which serves a redirect and sets the
+// |async_revalidation_required| flag on the HttpResponseInfo object to true if
+// the request has the LOAD_SUPPORT_ASYNC_REVALIDATION flag.
+class RedirectAndRevalidateURLRequestTestJob : public net::URLRequestTestJob {
+ public:
+  // This Create() method returns a redirecting job if the URL contains the
+  // string "redirect", otherwise a AsyncRevalidationRequiredURLRequestTestJob.
+  static net::URLRequestJob* Create(net::URLRequest* request,
+                                    net::NetworkDelegate* network_delegate) {
+    if (request->url().spec().find("redirect") != std::string::npos) {
+      return new RedirectAndRevalidateURLRequestTestJob(request,
+                                                        network_delegate);
+    }
+    return AsyncRevalidationRequiredURLRequestTestJob::Create(request,
+                                                              network_delegate);
+  }
+
+  void GetResponseInfo(net::HttpResponseInfo* info) override {
+    URLRequestTestJob::GetResponseInfo(info);
+    if (request()->load_flags() & net::LOAD_SUPPORT_ASYNC_REVALIDATION)
+      info->async_revalidation_required = true;
+  }
+
+ private:
+  RedirectAndRevalidateURLRequestTestJob(net::URLRequest* request,
+                                         net::NetworkDelegate* network_delegate)
+      : URLRequestTestJob(request,
+                          network_delegate,
+                          std::string(CreateRedirectHeaders()),
+                          std::string(),
+                          false) {}
+
+  ~RedirectAndRevalidateURLRequestTestJob() override {}
+
+  static std::string CreateRedirectHeaders() {
+    static const char kRedirectHeaders[] =
+        "HTTP/1.1 302 MOVED\n"
+        "Location: http://example.com/var\n"
+        "\n";
+    return std::string(kRedirectHeaders, arraysize(kRedirectHeaders));
+  }
+
+  DISALLOW_COPY_AND_ASSIGN(RedirectAndRevalidateURLRequestTestJob);
+};
+
+// A NetworkDelegate that records the URLRequests as they are created.
+class URLRequestRecordingNetworkDelegate : public net::TestNetworkDelegate {
+ public:
+  URLRequestRecordingNetworkDelegate() : requests_() {}
+
+  net::URLRequest* NextRequest() {
+    if (requests_.empty())
+      return nullptr;
+    net::URLRequest* request = requests_.front();
+    requests_.pop();
+    return request;
+  }
+
+  bool IsEmpty() const { return requests_.empty(); }
+
+  int OnBeforeURLRequest(net::URLRequest* request,
+                         const net::CompletionCallback& callback,
+                         GURL* new_url) override {
+    requests_.push(request);
+    return TestNetworkDelegate::OnBeforeURLRequest(request, callback, new_url);
+  }

[davidben, 2015/12/07 23:56:04]

Might be worth also listening for destruction and doing some kind of appropriate
DCHECK. Maybe something like... after any URLRequest is destroyed, you're not
allowed to call NextRequest or IsEmpty anymore? Since after that it gets pretty
screwy.

[Adam Rice, 2015/12/08 18:05:36]

I made it so that it removes dangling pointers when the URLRequest is destroyed,
but you can still check that the URLRequest actually existed.

+
+ private:
+  std::queue<net::URLRequest*> requests_;
+
+  DISALLOW_COPY_AND_ASSIGN(URLRequestRecordingNetworkDelegate);
+};
+
+class AsyncRevalidationManagerRecordingTest
+    : public AsyncRevalidationManagerTest {
+ public:
+  AsyncRevalidationManagerRecordingTest()
+      : AsyncRevalidationManagerTest(
+            make_scoped_ptr(new URLRequestRecordingNetworkDelegate)) {
+    // Use the AsyncRevalidationRequiredURLRequestTestJob.
+    SetCustomURLRequestJobCreateCallback(
+        base::Bind(&AsyncRevalidationRequiredURLRequestTestJob::Create));
+  }
+
+  URLRequestRecordingNetworkDelegate* recording_network_delegate() const {
+    return static_cast<URLRequestRecordingNetworkDelegate*>(
+        network_delegate_.get());
+  }
+
+  net::URLRequest* NextRequest() {
+    return recording_network_delegate()->NextRequest();
+  }
+
+  bool IsEmpty() const { return recording_network_delegate()->IsEmpty(); }
+};
+
+// Verify that an async revalidation is actually created when needed.
+TEST_F(AsyncRevalidationManagerRecordingTest, Issued) {
+  // Create the original request.
+  MakeTestRequest(0, 1, GURL("http://example.com/baz"));
+
+  net::URLRequest* initial_request = NextRequest();
+  ASSERT_TRUE(initial_request);
+  EXPECT_TRUE(initial_request->load_flags() &
+              net::LOAD_SUPPORT_ASYNC_REVALIDATION);
+
+  net::URLRequest* async_request = NextRequest();
+  ASSERT_TRUE(async_request);
+}
+
+// Verify the the URL of the async revalidation matches the original request.
+TEST_F(AsyncRevalidationManagerRecordingTest, URLMatches) {
+  // Create the original request.
+  MakeTestRequest(0, 1, GURL("http://example.com/special-baz"));
+
+  // Discard the original request.
+  NextRequest();
+
+  net::URLRequest* async_request = NextRequest();
+  ASSERT_TRUE(async_request);
+  EXPECT_EQ(GURL("http://example.com/special-baz"), async_request->url());
+}
+
+TEST_F(AsyncRevalidationManagerRecordingTest,
+       AsyncRevalidationsDoNotSupportAsyncRevalidation) {
+  // Create the original request.
+  MakeTestRequest(0, 1, GURL("http://example.com/baz"));
+
+  // Discard the original request.
+  NextRequest();
+
+  // Get the async revalidation request.
+  net::URLRequest* async_request = NextRequest();
+  ASSERT_TRUE(async_request);
+  EXPECT_FALSE(async_request->load_flags() &
+               net::LOAD_SUPPORT_ASYNC_REVALIDATION);
+}
+
+TEST_F(AsyncRevalidationManagerRecordingTest, AsyncRevalidationsNotDuplicated) {
+  // Create the original request.
+  MakeTestRequest(0, 1, GURL("http://example.com/baz"));
+
+  // Discard the original request.
+  NextRequest();
+
+  // Get the async revalidation request.
+  net::URLRequest* async_request = NextRequest();
+  EXPECT_TRUE(async_request);
+
+  // Start a second request to the same URL.
+  MakeTestRequest(0, 2, GURL("http://example.com/baz"));
+
+  // Discard the second request.
+  NextRequest();
+
+  // There should not be another async revalidation request.
+  EXPECT_TRUE(IsEmpty());
+}
+
+// Async revalidation to different URLs should not be treated as duplicates.
+TEST_F(AsyncRevalidationManagerRecordingTest,
+       AsyncRevalidationsToSeparateURLsAreSeparate) {
+  // Create two requests to two URLs.
+  MakeTestRequest(0, 1, GURL("http://example.com/baz"));
+  MakeTestRequest(0, 2, GURL("http://example.com/far"));
+
+  net::URLRequest* initial_request = NextRequest();
+  ASSERT_TRUE(initial_request);
+  net::URLRequest* initial_async_revalidation = NextRequest();
+  ASSERT_TRUE(initial_async_revalidation);
+  net::URLRequest* second_request = NextRequest();
+  ASSERT_TRUE(second_request);
+  net::URLRequest* second_async_revalidation = NextRequest();
+  ASSERT_TRUE(second_async_revalidation);
+
+  EXPECT_EQ("http://example.com/baz", initial_request->url().spec());
+  EXPECT_EQ("http://example.com/baz", initial_async_revalidation->url().spec());
+  EXPECT_EQ("http://example.com/far", second_request->url().spec());
+  EXPECT_EQ("http://example.com/far", second_async_revalidation->url().spec());
+}
+
+// A stale-while-revalidate applicable redirect response should not result in an
+// async revalidation.
+TEST_F(AsyncRevalidationManagerRecordingTest, InitialRedirectLegRevalidated) {
+  // Use the appropriate URLRequestJob for the test.
+  SetCustomURLRequestJobCreateCallback(
+      base::Bind(&RedirectAndRevalidateURLRequestTestJob::Create));
+  MakeTestRequest(0, 1, GURL("http://example.com/redirect"));
+
+  net::URLRequest* initial_request = NextRequest();
+  EXPECT_TRUE(initial_request);
+
+  // There should be an async revalidation request.
+  ASSERT_TRUE(NextRequest());
+}
+
+// Nothing after the first redirect leg has stale-while-revalidate applied.
+// TODO(ricea): s-w-r should work with redirects. Change this test when it does.
+TEST_F(AsyncRevalidationManagerRecordingTest, NoSWRAfterFirstRedirectLeg) {
+  SetCustomURLRequestJobCreateCallback(
+      base::Bind(&RedirectAndRevalidateURLRequestTestJob::Create));
+  MakeTestRequest(0, 1, GURL("http://example.com/redirect"));
+
+  net::URLRequest* initial_request = NextRequest();
+  EXPECT_TRUE(initial_request);
+
+  EXPECT_FALSE(initial_request->load_flags() &
+               net::LOAD_SUPPORT_ASYNC_REVALIDATION);
+
+  // An async revalidation happens for the redirect.
+  EXPECT_TRUE(NextRequest());
+
+  // But no others.
+  EXPECT_TRUE(IsEmpty());
+}
+
+}  // namespace
+
+}  // namespace content