predictors: browser test for client-side redirect.

chrome/browser/predictors/resource_prefetch_predictor_browsertest.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <stddef.h>
 
 #include <set>
 
 #include "base/command_line.h"
 #include "base/memory/ptr_util.h"
@@ skipping 15 matching lines @@
 #include "net/test/embedded_test_server/http_response.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace predictors {
 
 namespace {
 
 const char kFooHost[] = "foo.com";
 const char kBarHost[] = "bar.com";
+const char kBazHost[] = "baz.com";
 
 const char kImageMime[] = "image/png";
 const char kStyleMime[] = "text/css";
 const char kJavascriptMime[] = "application/javascript";
 
 // Paths to resources handled by a custom request handler. They return empty
 // responses with controllable response headers.
 const char kImagePath[] = "/handled-by-test/image.png";
 const char kImagePath2[] = "/handled-by-test/image2.png";
 const char kStylePath[] = "/handled-by-test/style.css";
 const char kStylePath2[] = "/handled-by-test/style2.css";
 const char kScriptPath[] = "/handled-by-test/script.js";
 const char kScriptPath2[] = "/handled-by-test/script2.js";
 const char kFontPath[] = "/handled-by-test/font.ttf";
 const char kRedirectPath[] = "/handled-by-test/redirect.html";
 const char kRedirectPath2[] = "/handled-by-test/redirect2.html";
 const char kRedirectPath3[] = "/handled-by-test/redirect3.html";
 
 // These are loaded from a file by the test server.
 const char kHtmlSubresourcesPath[] = "/predictors/html_subresources.html";
 const char kHtmlDocumentWritePath[] = "/predictors/document_write.html";
 const char kScriptDocumentWritePath[] = "/predictors/document_write.js";
 const char kHtmlAppendChildPath[] = "/predictors/append_child.html";
 const char kScriptAppendChildPath[] = "/predictors/append_child.js";
 const char kHtmlInnerHtmlPath[] = "/predictors/inner_html.html";
 const char kScriptInnerHtmlPath[] = "/predictors/inner_html.js";
 const char kHtmlXHRPath[] = "/predictors/xhr.html";
 const char kScriptXHRPath[] = "/predictors/xhr.js";
 const char kHtmlIframePath[] = "/predictors/html_iframe.html";
+const char kHtmlJavascriptRedirectPath[] =
+    "/predictors/javascript_redirect.html";
 
 struct ResourceSummary {
   ResourceSummary()
       : version(0),
         is_no_store(false),
         is_external(false),
         is_observable(true),
         is_prohibited(false) {}
 
   ResourcePrefetchPredictor::URLRequestSummary request;
   std::string content;
   // Allows to update HTTP ETag.
   size_t version;
   // True iff "Cache-control: no-store" header is present.
   bool is_no_store;
   // True iff a request for this resource must be ignored by the custom handler.
   bool is_external;
   // True iff the LearningObserver must observe this resource.
   bool is_observable;
   // A request with |is_prohibited| set to true makes the test that originates
   // the request fail.
   bool is_prohibited;
 };
 
 struct RedirectEdge {
   // This response code should be returned by previous url in the chain.
   net::HttpStatusCode code;
   GURL url;
+  bool is_client_side;
 };
 
 // Helper class to track and allow waiting for ResourcePrefetchPredictor
 // initialization. WARNING: OnPredictorInitialized event will not be fired if
 // ResourcePrefetchPredictor is initialized before the observer creation.
 class InitializationObserver : public TestObserver {
  public:
   explicit InitializationObserver(ResourcePrefetchPredictor* predictor)
       : TestObserver(predictor) {}
 
@@ skipping 77 matching lines @@
     ModifySubresourceForComparison(&subresource, match_navigation_id);
 
   EXPECT_THAT(actual_subresources,
               testing::UnorderedElementsAreArray(expected_subresources));
 }
 
 std::string CreateVersionedETag(size_t version, const std::string& path) {
   return base::StringPrintf("'%zu%s'", version, path.c_str());
 }
 
+GURL GetRequestURL(const net::test_server::HttpRequest& request) {

[alexilin, 2017/01/05 17:11:46]

Extracted from HandleResourceRequest

+  GURL resource_url = request.GetURL();
+  // Retrieve the host that was used in the request because
+  // resource_url contains a resolved host (e.g. 127.0.0.1).
+  if (request.headers.find("Host") != request.headers.end()) {
+    auto host_port_pair =
+        net::HostPortPair::FromString(request.headers.at("Host"));
+    GURL::Replacements replace_host;
+    replace_host.SetHostStr(host_port_pair.host());
+    resource_url = resource_url.ReplaceComponents(replace_host);
+  } else {
+    ADD_FAILURE() << "Host header was not found in a HttpRequest to url: "
+                  << resource_url.spec();
+  }
+
+  return resource_url;
+}
+
 }  // namespace
 
 // Helper class to track and allow waiting for a single OnNavigationLearned
 // event. The information provided by this event is also used to verify that
 // ResourcePrefetchPredictor works as expected.
 class LearningObserver : public TestObserver {
  public:
   using PageRequestSummary = ResourcePrefetchPredictor::PageRequestSummary;
 
   LearningObserver(ResourcePrefetchPredictor* predictor,
@@ skipping 97 matching lines @@
     NavigateToURLAndCheckSubresources(main_frame_url);
     ClearCache();
     // It is needed to have at least two resource hits to trigger prefetch.
     NavigateToURLAndCheckSubresources(main_frame_url);
     ClearCache();
     // Prefetch all needed resources and change expectations so that all
     // cacheable resources should be served from cache next navigation.
     PrefetchURL(main_frame_url);
     // To be sure that the browser send no requests to the server after
     // prefetching.
+    NavigateToURLAndCheckSubresourcesAllCached(main_frame_url);
+  }
+
+  void NavigateToURLAndCheckSubresourcesAllCached(const GURL& navigation_url) {
     for (auto& kv : resources_) {
       if (kv.second.is_observable)
         kv.second.is_prohibited = true;
     }
-    NavigateToURLAndCheckSubresources(main_frame_url);
+    NavigateToURLAndCheckSubresources(navigation_url);
     for (auto& kv : resources_) {
       if (kv.second.is_observable)
         kv.second.is_prohibited = false;
     }
   }
 
   void NavigateToURLAndCheckSubresources(
-      const GURL& main_frame_url,
+      const GURL& navigation_url,
       WindowOpenDisposition disposition = WindowOpenDisposition::CURRENT_TAB) {
-    GURL endpoint_url = GetRedirectEndpoint(main_frame_url);
+    GURL initial_url = GetLastClientSideRedirectEndpoint(navigation_url);
+    GURL main_frame_url = GetRedirectEndpoint(navigation_url);
     std::vector<URLRequestSummary> url_request_summaries;
     for (const auto& kv : resources_) {
       if (kv.second.is_observable) {
         url_request_summaries.push_back(
-            GetURLRequestSummaryForResource(endpoint_url, kv.second));
+            GetURLRequestSummaryForResource(main_frame_url, kv.second));
       }
     }
 
     bool match_navigation_id =
         disposition == WindowOpenDisposition::CURRENT_TAB;
 
     LearningObserver observer(
-        predictor_, UpdateAndGetVisitCount(main_frame_url),
-        CreatePageRequestSummary(endpoint_url.spec(), main_frame_url.spec(),
+        predictor_, UpdateAndGetVisitCount(initial_url),
+        CreatePageRequestSummary(main_frame_url.spec(), initial_url.spec(),
                                  url_request_summaries),
         match_navigation_id);
     ui_test_utils::NavigateToURLWithDisposition(
-        browser(), main_frame_url, disposition,
+        browser(), navigation_url, disposition,
         ui_test_utils::BROWSER_TEST_NONE);
     observer.Wait();
+
     for (auto& kv : resources_) {
       if (kv.second.is_observable)
         kv.second.request.was_cached = true;
     }
     for (const auto& nav : observer.current_navigation_ids())
       navigation_id_history_.insert(nav);
   }
 
   void PrefetchURL(const GURL& main_frame_url) {
     PrefetchingObserver observer(predictor_, main_frame_url);
     predictor_->StartPrefetching(main_frame_url, PrefetchOrigin::EXTERNAL);
     observer.Wait();
     for (auto& kv : resources_) {
       if (kv.second.is_observable)
         kv.second.request.was_cached = true;
     }
   }
 
+  void TryToPrefetchURL(const GURL& main_frame_url) {
+    predictor_->StartPrefetching(main_frame_url, PrefetchOrigin::EXTERNAL);

[Benoit L, 2017/01/09 14:31:47]

Isn't this function only used in one place?
If so, please keep the call inline.

[alexilin, 2017/01/09 15:08:15]

Well, the problem is that |predictor_| is private. I didn't want to make it
public.

[Benoit L, 2017/01/09 15:21:03]

Acknowledged.

+  }
+
   ResourceSummary* AddResource(const GURL& resource_url,
                                content::ResourceType resource_type,
                                net::RequestPriority priority) {
     auto pair_and_whether_inserted =
         resources_.insert(std::make_pair(resource_url, ResourceSummary()));
     EXPECT_TRUE(pair_and_whether_inserted.second) << resource_url
                                                   << " was inserted twice";
     ResourceSummary* resource = &pair_and_whether_inserted.first->second;
     resource->request.resource_url = resource_url;
     resource->request.resource_type = resource_type;
@@ skipping 113 matching lines @@
     while (true) {
       std::map<GURL, RedirectEdge>::const_iterator it =
           redirects_.find(current);
       if (it == redirects_.end())
         break;
       current = it->second.url;
     }
     return current;
   }
 
+  GURL GetLastClientSideRedirectEndpoint(const GURL& initial_url) const {
+    GURL last_client_side_redirect_url = initial_url;
+    GURL current = initial_url;
+    while (true) {
+      std::map<GURL, RedirectEdge>::const_iterator it =
+          redirects_.find(current);
+      if (it == redirects_.end())
+        break;
+      current = it->second.url;
+      if (it->second.is_client_side)
+        last_client_side_redirect_url = current;
+    }
+    return last_client_side_redirect_url;
+  }
+
   void MonitorResourceRequest(
       const net::test_server::HttpRequest& request) const {
     std::map<GURL, ResourceSummary>::const_iterator resource_it =
         resources_.find(request.GetURL());
     if (resource_it == resources_.end())
       return;
 
     const ResourceSummary& summary = resource_it->second;
     EXPECT_FALSE(summary.is_prohibited) << request.GetURL() << "\n"
                                         << request.all_headers;
   }
 
   // The custom handler for resource requests from the browser to an
   // EmbeddedTestServer. Runs on the EmbeddedTestServer IO thread.
   // Finds the data to serve requests in |resources_| map keyed by a request
   // URL.
   // Uses also the following headers from the |request|:
   //   - "Host" to retrieve the host that actually was issued by the browser.
   //   - "If-None-Match" to determine if the resource is still valid. If the
   //      ETag values match, the handler responds with a HTTP 304 status.
   std::unique_ptr<net::test_server::HttpResponse> HandleResourceRequest(
       const net::test_server::HttpRequest& request) const {
-    GURL resource_url = request.GetURL();
-    // Retrieve the host that was used in the request because
-    // resource_url contains a resolved host (e.g. 127.0.0.1).
-    if (request.headers.find("Host") != request.headers.end()) {
-      auto host_port_pair =
-          net::HostPortPair::FromString(request.headers.at("Host"));
-      GURL::Replacements replace_host;
-      replace_host.SetHostStr(host_port_pair.host());
-      resource_url = resource_url.ReplaceComponents(replace_host);
-    } else {
-      ADD_FAILURE() << "Host header was not found in a HttpRequest to url: "
-                    << resource_url.spec();
-    }
-
     std::map<GURL, ResourceSummary>::const_iterator resource_it =
-        resources_.find(resource_url);
+        resources_.find(GetRequestURL(request));
     if (resource_it == resources_.end())
       return nullptr;
 
     const ResourceSummary& summary = resource_it->second;
     if (summary.is_external)
       return nullptr;
 
     auto http_response =
         base::MakeUnique<net::test_server::BasicHttpResponse>();
 
@@ skipping 12 matching lines @@
     if (summary.is_no_store)
       http_response->AddCustomHeader("Cache-Control", "no-store");
     if (summary.request.has_validators) {
       http_response->AddCustomHeader(
           "ETag", CreateVersionedETag(summary.version, request.relative_url));
     }
     if (summary.request.always_revalidate)
       http_response->AddCustomHeader("Cache-Control", "no-cache");
     else
       http_response->AddCustomHeader("Cache-Control", "max-age=2147483648");
+
     return std::move(http_response);
   }
 
   // The custom handler for redirect requests from the browser to an
   // EmbeddedTestServer. Running on the EmbeddedTestServer IO thread.
   // Finds the data to serve requests in |redirects_| map keyed by a request
   // URL.
   std::unique_ptr<net::test_server::HttpResponse> HandleRedirectRequest(
       const net::test_server::HttpRequest& request) const {
     std::map<GURL, RedirectEdge>::const_iterator redirect_it =
-        redirects_.find(request.GetURL());
-    if (redirect_it == redirects_.end())
+        redirects_.find(GetRequestURL(request));
+    if (redirect_it == redirects_.end() || redirect_it->second.is_client_side)
       return nullptr;
 
     auto http_response =
         base::MakeUnique<net::test_server::BasicHttpResponse>();
     http_response->set_code(redirect_it->second.code);
     http_response->AddCustomHeader("Location", redirect_it->second.url.spec());
     return std::move(http_response);
   }
 
   size_t UpdateAndGetVisitCount(const GURL& main_frame_url) {
@@ skipping 14 matching lines @@
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
               net::HIGHEST);
   TestLearningAndPrefetching(GetURL(kHtmlSubresourcesPath));
 }
 
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest, Redirect) {
-  AddRedirectChain(GetURL(kRedirectPath), {{net::HTTP_MOVED_PERMANENTLY,
-                                            GetURL(kHtmlSubresourcesPath)}});
+  GURL initial_url = embedded_test_server()->GetURL(kFooHost, kRedirectPath);
+  AddRedirectChain(initial_url, {{net::HTTP_MOVED_PERMANENTLY,
+                                  GetURL(kHtmlSubresourcesPath)}});
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
               net::HIGHEST);
-  TestLearningAndPrefetching(GetURL(kRedirectPath));
+  TestLearningAndPrefetching(initial_url);
 }
 
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest, RedirectChain) {
-  AddRedirectChain(GetURL(kRedirectPath),
-                   {{net::HTTP_FOUND, GetURL(kRedirectPath2)},
-                    {net::HTTP_MOVED_PERMANENTLY, GetURL(kRedirectPath3)},
+  GURL initial_url = embedded_test_server()->GetURL(kFooHost, kRedirectPath);
+  AddRedirectChain(initial_url,
+                   {{net::HTTP_FOUND,
+                     embedded_test_server()->GetURL(kBarHost, kRedirectPath2)},
+                    {net::HTTP_MOVED_PERMANENTLY,
+                     embedded_test_server()->GetURL(kBazHost, kRedirectPath3)},
                     {net::HTTP_FOUND, GetURL(kHtmlSubresourcesPath)}});
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
               net::HIGHEST);
-  TestLearningAndPrefetching(GetURL(kRedirectPath));
+  TestLearningAndPrefetching(initial_url);
 }
 
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
                        HttpToHttpsRedirect) {
   EnableHttpsServer();
-  AddRedirectChain(GetURL(kRedirectPath),
+  GURL initial_url = embedded_test_server()->GetURL(kFooHost, kRedirectPath);
+  AddRedirectChain(initial_url,
                    {{net::HTTP_MOVED_PERMANENTLY,
                      https_server()->GetURL(kHtmlSubresourcesPath)}});
   AddResource(https_server()->GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE,
               net::LOWEST);
   AddResource(https_server()->GetURL(kStylePath),
               content::RESOURCE_TYPE_STYLESHEET, net::HIGHEST);
   AddResource(https_server()->GetURL(kScriptPath),
               content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(https_server()->GetURL(kFontPath),
               content::RESOURCE_TYPE_FONT_RESOURCE, net::HIGHEST);
-  TestLearningAndPrefetching(GetURL(kRedirectPath));
+  TestLearningAndPrefetching(initial_url);
 }
 
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
                        JavascriptDocumentWrite) {
   auto externalScript =
       AddExternalResource(GetURL(kScriptDocumentWritePath),
                           content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   externalScript->request.mime_type = kJavascriptMime;
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
@@ skipping 131 matching lines @@
       AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT,
                   net::MEDIUM),
       AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
                   net::HIGHEST),
   };
   for (auto& resource : resources)
     resource->request.always_revalidate = true;
   TestLearningAndPrefetching(GetURL(kHtmlSubresourcesPath));
 }
 
+// Client-side redirects currently aren't tracked by ResourcePrefetchPredictor.
+// Client-side redirect initiates a new navigation to the redirect destination

[Benoit L, 2017/01/09 14:31:47]

nit: A client-side redirect... so that the OnLoad event...

[alexilin, 2017/01/09 15:08:16]

Done.

+// URL and aborts the current navigation so that OnLoad event is not fired.
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
+                       JavascriptRedirect) {

[Benoit L, 2017/01/09 14:31:47]

nit: JavascriptRedirectsAreNotHandled?

[alexilin, 2017/01/09 15:08:15]

Done.

+  std::string redirect_path_with_query =
+      std::string(kHtmlJavascriptRedirectPath) + "?url=" +
+      GetURL(kHtmlSubresourcesPath).spec();
+  GURL initial_url =
+      embedded_test_server()->GetURL(kBarHost, redirect_path_with_query);
+  AddRedirectChain(initial_url, {{net::HTTP_TEMPORARY_REDIRECT,
+                                  GetURL(kHtmlSubresourcesPath), true}});
+  AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
+  AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
+              net::HIGHEST);
+  AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
+              net::HIGHEST);
+
+  // Two navigations will occur. LearningObserver will get event only for the

[Benoit L, 2017/01/09 14:31:47]

nit: events

[alexilin, 2017/01/09 15:08:15]

Done.

+  // second navigation because the first one will be aborted.
+  NavigateToURLAndCheckSubresources(initial_url);
+  ClearCache();
+  // It is needed to have at least two resource hits to trigger prefetch.
+  NavigateToURLAndCheckSubresources(initial_url);
+  ClearCache();
+  // Prefetching of |initial_url| has no effect because there is no entry in
+  // the predictor database corresponding the client-side redirect.
+  TryToPrefetchURL(initial_url);
+  NavigateToURLAndCheckSubresources(initial_url);
+  ClearCache();
+  // But the predictor database contains all subresources for the endpoint url
+  // so this prefetch works.
+  PrefetchURL(GetURL(kHtmlSubresourcesPath));
+  NavigateToURLAndCheckSubresourcesAllCached(GetURL(kHtmlSubresourcesPath));
+}
+
 }  // namespace predictors