predictors: Make speculative_prefetch_predictor work with PlzNavigate

chrome/browser/predictors/resource_prefetch_predictor_browsertest.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/command_line.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor_test_util.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/ui/browser.h"
+#include "chrome/browser/ui/browser_commands.h"
 #include "chrome/browser/ui/tabs/tab_strip_model.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/test/base/in_process_browser_test.h"
 #include "chrome/test/base/ui_test_utils.h"
-#include "content/public/browser/render_frame_host.h"
-#include "content/public/browser/render_process_host.h"
+#include "net/dns/mock_host_resolver.h"
 #include "net/test/embedded_test_server/http_request.h"
 #include "net/test/embedded_test_server/http_response.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace predictors {
 
 namespace {
 
 const char kImageMime[] = "image/png";
@@ skipping 57 matching lines @@
 
  private:
   base::RunLoop run_loop_;
 
   DISALLOW_COPY_AND_ASSIGN(InitializationObserver);
 };
 
 using PageRequestSummary = ResourcePrefetchPredictor::PageRequestSummary;
 using URLRequestSummary = ResourcePrefetchPredictor::URLRequestSummary;
 
+struct ResourceComparisonParameters {

[ahemery, 2016/12/19 14:53:42]

Just improving modularity in case we need to add other matchers

+  bool match_navigation_id;
+
+  ResourceComparisonParameters() : match_navigation_id(true) {}
+};
+
 void RemoveDuplicateSubresources(std::vector<URLRequestSummary>* subresources) {
   std::stable_sort(subresources->begin(), subresources->end(),
                    [](const URLRequestSummary& x, const URLRequestSummary& y) {
                      return x.resource_url < y.resource_url;
                    });
   subresources->erase(
       std::unique(subresources->begin(), subresources->end(),
                   [](const URLRequestSummary& x, const URLRequestSummary& y) {
                     return x.resource_url == y.resource_url;
                   }),
       subresources->end());
 }
 
 // Fill a NavigationID with "empty" data that does not trigger
 // the is_valid DCHECK(). Allows comparing.
 void SetValidNavigationID(NavigationID* navigation_id) {
-  navigation_id->render_process_id = 0;
-  navigation_id->render_frame_id = 0;
+  navigation_id->session_id.set_id(0);
+  navigation_id->frame_tree_node_id = -1;
   navigation_id->main_frame_url = GURL("http://127.0.0.1");
 }
 
-void ModifySubresourceForComparison(URLRequestSummary* subresource,
-                                    bool match_navigation_id) {
-  if (!match_navigation_id)
+void ModifySubresourceForComparison(
+    URLRequestSummary* subresource,
+    const ResourceComparisonParameters& params) {
+  if (!params.match_navigation_id)
     SetValidNavigationID(&subresource->navigation_id);
   if (subresource->resource_type == content::RESOURCE_TYPE_IMAGE &&
       subresource->priority == net::LOWEST) {
     // Fuzzy comparison for images because an image priority can be
     // boosted during layout via
     // ResourceFetcher::updateAllImageResourcePriorities().
     subresource->priority = net::MEDIUM;
   }
 }
 
 // Does a custom comparison of subresources of URLRequestSummary
 // and fail the test if the expectation is not met.
 void CompareSubresources(std::vector<URLRequestSummary> actual_subresources,
                          std::vector<URLRequestSummary> expected_subresources,
-                         bool match_navigation_id) {
+                         const ResourceComparisonParameters& params) {
   // Duplicate resources can be observed in a single navigation but
   // ResourcePrefetchPredictor only cares about the first occurrence of each.
   RemoveDuplicateSubresources(&actual_subresources);
 
   for (auto& subresource : actual_subresources)
-    ModifySubresourceForComparison(&subresource, match_navigation_id);
+    ModifySubresourceForComparison(&subresource, params);
   for (auto& subresource : expected_subresources)
-    ModifySubresourceForComparison(&subresource, match_navigation_id);
+    ModifySubresourceForComparison(&subresource, params);
 
   EXPECT_THAT(actual_subresources,
               testing::UnorderedElementsAreArray(expected_subresources));
 }
 
 }  // namespace
 
 // Helper class to track and allow waiting for ResourcePrefetchPredictor events.
 // These events are also used to verify that ResourcePrefetchPredictor works as
 // expected.
 class ResourcePrefetchPredictorTestObserver : public TestObserver {
  public:
   using PageRequestSummary = ResourcePrefetchPredictor::PageRequestSummary;
 
   explicit ResourcePrefetchPredictorTestObserver(
       ResourcePrefetchPredictor* predictor,
       const size_t expected_url_visit_count,
       const PageRequestSummary& expected_summary,
-      bool match_navigation_id)
+      const ResourceComparisonParameters& resource_comparison_params)
       : TestObserver(predictor),
         url_visit_count_(expected_url_visit_count),
         summary_(expected_summary),
-        match_navigation_id_(match_navigation_id) {}
+        resource_comparison_params_(resource_comparison_params) {}
 
   // TestObserver:
   void OnNavigationLearned(size_t url_visit_count,
                            const PageRequestSummary& summary) override {
     EXPECT_EQ(url_visit_count, url_visit_count_);
     EXPECT_EQ(summary.main_frame_url, summary_.main_frame_url);
     EXPECT_EQ(summary.initial_url, summary_.initial_url);
+    for (const auto& resource : summary.subresource_requests)

[ahemery, 2016/12/19 14:53:42]

Tracking what NavigationID's have been used. Required to verify that tab_id has
expected behavior.

+      ++current_navigation_ids_[resource.navigation_id];
     CompareSubresources(summary.subresource_requests,
-                        summary_.subresource_requests, match_navigation_id_);
+                        summary_.subresource_requests,
+                        resource_comparison_params_);
     run_loop_.Quit();
   }
 
   void Wait() { run_loop_.Run(); }
 
+  std::map<NavigationID, int>* GetNavigationIds() {
+    return &current_navigation_ids_;
+  }
+
  private:
   base::RunLoop run_loop_;
   size_t url_visit_count_;
   PageRequestSummary summary_;
-  bool match_navigation_id_;
+  ResourceComparisonParameters resource_comparison_params_;
+  std::map<NavigationID, int> current_navigation_ids_;
 
   DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorTestObserver);
 };
 
 class ResourcePrefetchPredictorBrowserTest : public InProcessBrowserTest {
  protected:
   using URLRequestSummary = ResourcePrefetchPredictor::URLRequestSummary;
 
   void SetUpCommandLine(base::CommandLine* command_line) override {
     command_line->AppendSwitchASCII(
         switches::kSpeculativeResourcePrefetching,
         switches::kSpeculativeResourcePrefetchingEnabled);
   }
 
   void SetUpOnMainThread() override {
+    SetCurrentHostName("127.0.0.1");

[ahemery, 2016/12/19 14:53:42]

Resolving everything to localhost to be able to use various hostnames

+    host_resolver()->AddRule("*", "127.0.0.1");
+    embedded_test_server()->AddDefaultHandlers(
+        base::FilePath(FILE_PATH_LITERAL("chrome/test/data")));
     embedded_test_server()->RegisterRequestHandler(
         base::Bind(&ResourcePrefetchPredictorBrowserTest::HandleRedirectRequest,
                    base::Unretained(this)));
     embedded_test_server()->RegisterRequestHandler(
         base::Bind(&ResourcePrefetchPredictorBrowserTest::HandleResourceRequest,
                    base::Unretained(this)));
     ASSERT_TRUE(embedded_test_server()->Start());
     predictor_ =
         ResourcePrefetchPredictorFactory::GetForProfile(browser()->profile());
     ASSERT_TRUE(predictor_);
     EnsurePredictorInitialized();
   }
 
   void NavigateToURLAndCheckSubresources(
       const GURL& main_frame_url,
       WindowOpenDisposition disposition = WindowOpenDisposition::CURRENT_TAB) {
     GURL endpoint_url = GetRedirectEndpoint(main_frame_url);
     std::vector<URLRequestSummary> url_request_summaries;
     for (const auto& kv : resources_) {
       if (kv.second.is_no_store || !kv.second.should_be_recorded)
         continue;
       url_request_summaries.push_back(
           GetURLRequestSummaryForResource(endpoint_url, kv.second));
     }
+
+    ResourceComparisonParameters resource_comparison_params;
+    if (disposition != WindowOpenDisposition::CURRENT_TAB)
+      resource_comparison_params.match_navigation_id = false;
+
     ResourcePrefetchPredictorTestObserver observer(
         predictor_, UpdateAndGetVisitCount(main_frame_url),
         CreatePageRequestSummary(endpoint_url.spec(), main_frame_url.spec(),
                                  url_request_summaries),
-        true);  // Matching navigation id by default
+        resource_comparison_params);
     ui_test_utils::NavigateToURLWithDisposition(
         browser(), main_frame_url, disposition,
         ui_test_utils::BROWSER_TEST_NONE);
     observer.Wait();
+    for (auto& kv : resources_) {

[ahemery, 2016/12/19 14:53:42]

Copy of Alex modifications in https://codereview.chromium.org/2553083002/ to
facilitate merging. Using his way to disable caching.

+      if (!kv.second.is_no_store && kv.second.should_be_recorded)
+        kv.second.request.was_cached = true;
+    }
+    for (const auto& nav : *observer.GetNavigationIds())

[ahemery, 2016/12/19 14:53:41]

Aggregating every observer navigationID map into the browsertest object

+      ++navigation_id_history_[nav.first];
   }
 
   ResourceSummary* AddResource(const GURL& resource_url,
                                content::ResourceType resource_type,
                                net::RequestPriority priority) {
     auto pair_and_whether_inserted =
         resources_.insert(std::make_pair(resource_url, ResourceSummary()));
     EXPECT_TRUE(pair_and_whether_inserted.second) << resource_url
                                                   << " was inserted twice";
     ResourceSummary* resource = &pair_and_whether_inserted.first->second;
@@ skipping 25 matching lines @@
                         const std::vector<RedirectEdge>& redirect_chain) {
     ASSERT_FALSE(redirect_chain.empty());
     GURL current = initial_url;
     for (const auto& edge : redirect_chain) {
       auto result = redirects_.insert(std::make_pair(current, edge));
       EXPECT_TRUE(result.second) << current << " already has a redirect.";
       current = edge.url;
     }
   }
 
+  void ClearResources() { resources_.clear(); }

[ahemery, 2016/12/19 14:53:42]

Used for clearing resources before hostname change

+
+  void ClearCache() {

[ahemery, 2016/12/19 14:53:42]

Copy of Alex modifications. See line 255 in this file

+    chrome::ClearCache(browser());
+    for (auto& kv : resources_)
+      kv.second.request.was_cached = false;
+  }
+
   // Shortcut for convenience.
   GURL GetURL(const std::string& path) const {
-    return embedded_test_server()->GetURL(path);
+    return embedded_test_server()->GetURL(current_host_name_, path);
   }
 
   void EnableHttpsServer() {
     ASSERT_FALSE(https_server_);
     https_server_ = base::MakeUnique<net::EmbeddedTestServer>(
         net::EmbeddedTestServer::TYPE_HTTPS);
     https_server()->AddDefaultHandlers(
         base::FilePath(FILE_PATH_LITERAL("chrome/test/data")));
     https_server()->RegisterRequestHandler(
         base::Bind(&ResourcePrefetchPredictorBrowserTest::HandleRedirectRequest,
                    base::Unretained(this)));
     https_server()->RegisterRequestHandler(
         base::Bind(&ResourcePrefetchPredictorBrowserTest::HandleResourceRequest,
                    base::Unretained(this)));
     ASSERT_TRUE(https_server()->Start());
   }
 
   // Returns the embedded test server working over HTTPS. Must be enabled by
   // calling EnableHttpsServer() before use.
   const net::EmbeddedTestServer* https_server() const {
     return https_server_.get();
   }
 
   net::EmbeddedTestServer* https_server() { return https_server_.get(); }
 
+  void SetCurrentHostName(const std::string& new_host) {

[ahemery, 2016/12/19 14:53:41]

Will be used to resolve every URL using the getURL method

+    current_host_name_ = new_host;
+  }
+
+  unsigned int GetHistoryCount() { return navigation_id_history_.size(); }

[ahemery, 2016/12/19 14:53:42]

Returns the number of navigations that have been used in this test so far

+
  private:
   // ResourcePrefetchPredictor needs to be initialized before the navigation
   // happens otherwise this navigation will be ignored by predictor.
   void EnsurePredictorInitialized() {
     if (predictor_->initialization_state_ ==
         ResourcePrefetchPredictor::INITIALIZED) {
       return;
     }
 
     InitializationObserver observer(predictor_);
     if (predictor_->initialization_state_ ==
         ResourcePrefetchPredictor::NOT_INITIALIZED) {
       predictor_->StartInitialization();
     }
     observer.Wait();
   }
 
   URLRequestSummary GetURLRequestSummaryForResource(
       const GURL& main_frame_url,
       const ResourceSummary& resource_summary) const {
     URLRequestSummary summary(resource_summary.request);
     content::WebContents* web_contents =
         browser()->tab_strip_model()->GetActiveWebContents();
-    int process_id = web_contents->GetRenderProcessHost()->GetID();
-    int frame_id = web_contents->GetMainFrame()->GetRoutingID();
     summary.navigation_id =
-        CreateNavigationID(process_id, frame_id, main_frame_url.spec());
+        NavigationID(web_contents, main_frame_url, base::TimeTicks::Now());

[ahemery, 2016/12/19 14:53:41]

Using constructor directly with web_contents. Forgot to update it in previous
patch

     return summary;
   }
 
   GURL GetRedirectEndpoint(const GURL& initial_url) const {
     GURL current = initial_url;
     while (true) {
       auto it = redirects_.find(current);
       if (it == redirects_.end())
         break;
       current = it->second.url;
     }
     return current;
   }
 
   std::unique_ptr<net::test_server::HttpResponse> HandleResourceRequest(
       const net::test_server::HttpRequest& request) const {
-    auto resource_it = resources_.find(request.GetURL());
+    GURL resource_url = request.GetURL();

[ahemery, 2016/12/19 14:53:42]

At this stage, URL has been resolved to 127.0.0.1, making the resource search
fail. We are updating the hostname to match the ones we have stored. We avoid
storing every resource with 127.0.0.1 in case we need to test cross site
fetching further.

+    GURL::Replacements replace_host;
+    replace_host.SetHostStr(current_host_name_);
+    resource_url = resource_url.ReplaceComponents(replace_host);
+
+    auto resource_it = resources_.find(resource_url);
     if (resource_it == resources_.end())
       return nullptr;
 
     const ResourceSummary& summary = resource_it->second;
     if (summary.is_external)
       return nullptr;
 
     auto http_response =
         base::MakeUnique<net::test_server::BasicHttpResponse>();
     http_response->set_code(net::HTTP_OK);
@@ skipping 25 matching lines @@
         base::MakeUnique<net::test_server::BasicHttpResponse>();
     http_response->set_code(redirect_it->second.code);
     http_response->AddCustomHeader("Location", redirect_it->second.url.spec());
     return std::move(http_response);
   }
 
   size_t UpdateAndGetVisitCount(const GURL& main_frame_url) {
     return ++visit_count_[main_frame_url];
   }
 
+  std::string current_host_name_;
   ResourcePrefetchPredictor* predictor_;
   std::unique_ptr<net::EmbeddedTestServer> https_server_;
   std::map<GURL, ResourceSummary> resources_;
   std::map<GURL, RedirectEdge> redirects_;
   std::map<GURL, size_t> visit_count_;
+  std::map<NavigationID, int> navigation_id_history_;
+};
+
+// TODO(ahemery): Remove when https://codereview.chromium.org/2553083002/
+// is landed by Alex. LearningObserver should be used by default.
+// His change will disable actual prefetching by default
+class ResourcePrefetchPredictorLearningBrowserTest
+    : public ResourcePrefetchPredictorBrowserTest {
+ protected:
+  void SetUpCommandLine(base::CommandLine* command_line) override {
+    command_line->AppendSwitchASCII(
+        switches::kSpeculativeResourcePrefetching,
+        switches::kSpeculativeResourcePrefetchingLearning);

[ahemery, 2016/12/19 14:53:41]

This is the critical line. Disables actual prefetching. This way we can have
absolutely no caching, as the browser cache clear does not prevent the prefetch
cache to be used.

+  }
 };
 
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest, LearningSimple) {
   // These resources have default priorities that correspond to
   // blink::typeToPriority function.
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
@@ skipping 114 matching lines @@
   AddResource(GetURL(kImagePath2), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath2), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath2), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   // Included from <iframe src="html_subresources.html"> and not recored.
   AddUnrecordedResources({GetURL(kImagePath), GetURL(kStylePath),
                           GetURL(kScriptPath), GetURL(kFontPath)});
   NavigateToURLAndCheckSubresources(GetURL(kHtmlIframePath));
 }
 
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
+                       CrossSiteLearning) {
+  SetCurrentHostName("foo.com");
+  AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
+  AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
+              net::HIGHEST);
+  AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
+              net::HIGHEST);
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlSubresourcesPath));
+  ClearResources();
+
+  SetCurrentHostName("bar.com");
+  AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
+  AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
+              net::HIGHEST);
+  AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
+              net::HIGHEST);
+
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlSubresourcesPath));
+}
+
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorLearningBrowserTest,

[ahemery, 2016/12/19 14:53:41]

When we stay in the same tab, tab_id shouldn't be different but should be
different if we change tab/window

+                       TabIdBehavingAsExpected) {
+  AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
+  AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
+              net::HIGHEST);
+  AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
+              net::HIGHEST);
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlSubresourcesPath));
+  EXPECT_EQ(GetHistoryCount(), 1U);
+  ClearCache();
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlSubresourcesPath));
+  EXPECT_EQ(GetHistoryCount(), 1U);
+  ClearCache();
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlSubresourcesPath),
+                                    WindowOpenDisposition::NEW_FOREGROUND_TAB);
+  EXPECT_EQ(GetHistoryCount(), 2U);
+  ClearCache();
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlSubresourcesPath),
+                                    WindowOpenDisposition::NEW_WINDOW);
+  EXPECT_EQ(GetHistoryCount(), 3U);
+  ClearCache();
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlSubresourcesPath),
+                                    WindowOpenDisposition::NEW_POPUP);
+  EXPECT_EQ(GetHistoryCount(), 4U);
+}
+
 }  // namespace predictors