predictors: Add browsertest that tests prefetching.

chrome/browser/predictors/resource_prefetch_predictor_browsertest.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/command_line.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor_test_util.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/ui/browser.h"
+#include "chrome/browser/ui/browser_commands.h"
 #include "chrome/browser/ui/tabs/tab_strip_model.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/test/base/in_process_browser_test.h"
 #include "chrome/test/base/ui_test_utils.h"
 #include "content/public/browser/render_frame_host.h"
 #include "content/public/browser/render_process_host.h"
 #include "net/test/embedded_test_server/http_request.h"
 #include "net/test/embedded_test_server/http_response.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ skipping 45 matching lines @@
   bool is_external;
   bool should_be_recorded;
 };
 
 struct RedirectEdge {
   // This response code should be returned by previous url in the chain.
   net::HttpStatusCode code;
   GURL url;
 };
 
+// Helper class to track and allow waiting for ResourcePrefetchPredictor
+// initialization. WARNING: OnPredictorInitialized event will not be fired if
+// ResourcePrefetchPredictor is initialized before observer creation.

[Benoit L, 2016/12/19 12:32:00]

nit: before the observer creation.

[alexilin, 2016/12/19 12:58:09]

Done.

 class InitializationObserver : public TestObserver {
  public:
   explicit InitializationObserver(ResourcePrefetchPredictor* predictor)
       : TestObserver(predictor) {}
 
   void OnPredictorInitialized() override { run_loop_.Quit(); }
 
   void Wait() { run_loop_.Run(); }
 
  private:
@@ skipping 52 matching lines @@
     ModifySubresourceForComparison(&subresource, match_navigation_id);
   for (auto& subresource : expected_subresources)
     ModifySubresourceForComparison(&subresource, match_navigation_id);
 
   EXPECT_THAT(actual_subresources,
               testing::UnorderedElementsAreArray(expected_subresources));
 }
 
 }  // namespace
 
-// Helper class to track and allow waiting for ResourcePrefetchPredictor events.
-// These events are also used to verify that ResourcePrefetchPredictor works as
-// expected.
-class ResourcePrefetchPredictorTestObserver : public TestObserver {
+// Helper class to track and allow waiting for a single OnNavigationLearned
+// event. The information provided by this event is also used to verify that
+// ResourcePrefetchPredictor works as expected.
+class LearningObserver : public TestObserver {
  public:
   using PageRequestSummary = ResourcePrefetchPredictor::PageRequestSummary;
 
-  explicit ResourcePrefetchPredictorTestObserver(
-      ResourcePrefetchPredictor* predictor,
-      const size_t expected_url_visit_count,
-      const PageRequestSummary& expected_summary,
-      bool match_navigation_id)
+  explicit LearningObserver(ResourcePrefetchPredictor* predictor,
+                            const size_t expected_url_visit_count,
+                            const PageRequestSummary& expected_summary,
+                            bool match_navigation_id)
       : TestObserver(predictor),
         url_visit_count_(expected_url_visit_count),
         summary_(expected_summary),
         match_navigation_id_(match_navigation_id) {}
 
   // TestObserver:
   void OnNavigationLearned(size_t url_visit_count,
                            const PageRequestSummary& summary) override {
     EXPECT_EQ(url_visit_count, url_visit_count_);
     EXPECT_EQ(summary.main_frame_url, summary_.main_frame_url);
     EXPECT_EQ(summary.initial_url, summary_.initial_url);
     CompareSubresources(summary.subresource_requests,
                         summary_.subresource_requests, match_navigation_id_);
     run_loop_.Quit();
   }
 
   void Wait() { run_loop_.Run(); }
 
  private:
   base::RunLoop run_loop_;
   size_t url_visit_count_;
   PageRequestSummary summary_;
   bool match_navigation_id_;
 
-  DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictorTestObserver);
+  DISALLOW_COPY_AND_ASSIGN(LearningObserver);
+};
+
+// Helper class to track and allow waiting for a single OnPrefetchingFinished
+// event. No learning events should be fired while this observer is active.
+class PrefetchingObserver : public TestObserver {
+ public:
+  explicit PrefetchingObserver(ResourcePrefetchPredictor* predictor,
+                               const GURL& expected_main_frame_url)
+      : TestObserver(predictor), main_frame_url_(expected_main_frame_url) {}
+
+  // TestObserver:
+  void OnNavigationLearned(size_t url_visit_count,
+                           const PageRequestSummary& summary) override {
+    ADD_FAILURE() << "Prefetching shouldn't activate learning";
+  }
+
+  void OnPrefetchingFinished(const GURL& main_frame_url) override {
+    EXPECT_EQ(main_frame_url_, main_frame_url);
+    run_loop_.Quit();
+  }
+
+  void Wait() { run_loop_.Run(); }
+
+ private:
+  base::RunLoop run_loop_;
+  GURL main_frame_url_;
+
+  DISALLOW_COPY_AND_ASSIGN(PrefetchingObserver);
 };
 
 class ResourcePrefetchPredictorBrowserTest : public InProcessBrowserTest {
  protected:
   using URLRequestSummary = ResourcePrefetchPredictor::URLRequestSummary;
 
   void SetUpCommandLine(base::CommandLine* command_line) override {
     command_line->AppendSwitchASCII(
         switches::kSpeculativeResourcePrefetching,
         switches::kSpeculativeResourcePrefetchingEnabled);
   }
 
   void SetUpOnMainThread() override {
     embedded_test_server()->RegisterRequestHandler(
         base::Bind(&ResourcePrefetchPredictorBrowserTest::HandleRedirectRequest,
                    base::Unretained(this)));
     embedded_test_server()->RegisterRequestHandler(
         base::Bind(&ResourcePrefetchPredictorBrowserTest::HandleResourceRequest,
                    base::Unretained(this)));
     ASSERT_TRUE(embedded_test_server()->Start());
     predictor_ =
         ResourcePrefetchPredictorFactory::GetForProfile(browser()->profile());
     ASSERT_TRUE(predictor_);
     EnsurePredictorInitialized();
   }
 
+  void TestLearningAndPrefetching(const GURL& main_frame_url) {
+    // Navigate to |main_frame_url| and check all the expectations.
+    NavigateToURLAndCheckSubresources(main_frame_url);
+    ClearCache();
+    // It is needed to have at least two resource hits to trigger prefetch.
+    NavigateToURLAndCheckSubresources(main_frame_url);
+    ClearCache();
+    // Prefetch all needed resources and change expectations so that all
+    // cacheable resources should be served from cache next navigation.
+    PrefetchURL(main_frame_url);
+    NavigateToURLAndCheckSubresources(main_frame_url);
+  }
+
   void NavigateToURLAndCheckSubresources(
       const GURL& main_frame_url,
       WindowOpenDisposition disposition = WindowOpenDisposition::CURRENT_TAB) {
     GURL endpoint_url = GetRedirectEndpoint(main_frame_url);
     std::vector<URLRequestSummary> url_request_summaries;
     for (const auto& kv : resources_) {
       if (kv.second.is_no_store || !kv.second.should_be_recorded)
         continue;
       url_request_summaries.push_back(
           GetURLRequestSummaryForResource(endpoint_url, kv.second));
     }
-    ResourcePrefetchPredictorTestObserver observer(
+    LearningObserver observer(
         predictor_, UpdateAndGetVisitCount(main_frame_url),
         CreatePageRequestSummary(endpoint_url.spec(), main_frame_url.spec(),
                                  url_request_summaries),
         true);  // Matching navigation id by default
     ui_test_utils::NavigateToURLWithDisposition(
         browser(), main_frame_url, disposition,
         ui_test_utils::BROWSER_TEST_NONE);
     observer.Wait();
+    for (auto& kv : resources_) {
+      if (!kv.second.is_no_store && kv.second.should_be_recorded)
+        kv.second.request.was_cached = true;
+    }
+  }
+
+  void PrefetchURL(const GURL& main_frame_url) {
+    PrefetchingObserver observer(predictor_, main_frame_url);
+    predictor_->StartPrefetching(main_frame_url, PrefetchOrigin::NAVIGATION);

[Benoit L, 2016/12/19 12:32:00]

nit: What you're doing here maps more closely to "EXTERNAL" conceptually, since
you prefetch independently from a navigation.

[alexilin, 2016/12/19 12:58:09]

I hope that the behavior of predictor in this two cases won't diverge? I'm
curious if we'll need special test cases for both.

[Benoit L, 2016/12/19 15:19:05]

Good point.
I'm sure that if either of us change this, we will add a test... or suffer the
wrath of the other one in code review.

+    observer.Wait();
+    for (auto& kv : resources_) {
+      if (!kv.second.is_no_store && kv.second.should_be_recorded)
+        kv.second.request.was_cached = true;
+    }
   }
 
   ResourceSummary* AddResource(const GURL& resource_url,
                                content::ResourceType resource_type,
                                net::RequestPriority priority) {
     auto pair_and_whether_inserted =
         resources_.insert(std::make_pair(resource_url, ResourceSummary()));
     EXPECT_TRUE(pair_and_whether_inserted.second) << resource_url
                                                   << " was inserted twice";
     ResourceSummary* resource = &pair_and_whether_inserted.first->second;
@@ skipping 25 matching lines @@
                         const std::vector<RedirectEdge>& redirect_chain) {
     ASSERT_FALSE(redirect_chain.empty());
     GURL current = initial_url;
     for (const auto& edge : redirect_chain) {
       auto result = redirects_.insert(std::make_pair(current, edge));
       EXPECT_TRUE(result.second) << current << " already has a redirect.";
       current = edge.url;
     }
   }
 
+  void ClearCache() {
+    chrome::ClearCache(browser());
+    for (auto& kv : resources_)
+      kv.second.request.was_cached = false;
+  }
+
   // Shortcut for convenience.
   GURL GetURL(const std::string& path) const {
     return embedded_test_server()->GetURL(path);
   }
 
   void EnableHttpsServer() {
     ASSERT_FALSE(https_server_);
     https_server_ = base::MakeUnique<net::EmbeddedTestServer>(
         net::EmbeddedTestServer::TYPE_HTTPS);
     https_server()->AddDefaultHandlers(
@@ skipping 104 matching lines @@
     return ++visit_count_[main_frame_url];
   }
 
   ResourcePrefetchPredictor* predictor_;
   std::unique_ptr<net::EmbeddedTestServer> https_server_;
   std::map<GURL, ResourceSummary> resources_;
   std::map<GURL, RedirectEdge> redirects_;
   std::map<GURL, size_t> visit_count_;
 };
 
-IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest, LearningSimple) {
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest, Simple) {
   // These resources have default priorities that correspond to
   // blink::typeToPriority function.
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
               net::HIGHEST);
-  NavigateToURLAndCheckSubresources(GetURL(kHtmlSubresourcesPath));
+  TestLearningAndPrefetching(GetURL(kHtmlSubresourcesPath));
 }
 
-IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
-                       LearningAfterRedirect) {
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest, Redirect) {
   AddRedirectChain(GetURL(kRedirectPath), {{net::HTTP_MOVED_PERMANENTLY,
                                             GetURL(kHtmlSubresourcesPath)}});
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
               net::HIGHEST);
-  NavigateToURLAndCheckSubresources(GetURL(kRedirectPath));
+  TestLearningAndPrefetching(GetURL(kRedirectPath));
 }
 
-IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
-                       LearningAfterRedirectChain) {
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest, RedirectChain) {
   AddRedirectChain(GetURL(kRedirectPath),
                    {{net::HTTP_FOUND, GetURL(kRedirectPath2)},
                     {net::HTTP_MOVED_PERMANENTLY, GetURL(kRedirectPath3)},
                     {net::HTTP_FOUND, GetURL(kHtmlSubresourcesPath)}});
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(GetURL(kFontPath), content::RESOURCE_TYPE_FONT_RESOURCE,
               net::HIGHEST);
-  NavigateToURLAndCheckSubresources(GetURL(kRedirectPath));
+  TestLearningAndPrefetching(GetURL(kRedirectPath));
 }
 
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
                        LearningAfterHttpToHttpsRedirect) {
   EnableHttpsServer();
   AddRedirectChain(GetURL(kRedirectPath),
-                   {{net::HTTP_FOUND, https_server()->GetURL(kRedirectPath2)},
-                    {net::HTTP_MOVED_PERMANENTLY,
+                   {{net::HTTP_MOVED_PERMANENTLY,
                      https_server()->GetURL(kHtmlSubresourcesPath)}});
   AddResource(https_server()->GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE,
               net::LOWEST);
   AddResource(https_server()->GetURL(kStylePath),
               content::RESOURCE_TYPE_STYLESHEET, net::HIGHEST);
   AddResource(https_server()->GetURL(kScriptPath),
               content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(https_server()->GetURL(kFontPath),
               content::RESOURCE_TYPE_FONT_RESOURCE, net::HIGHEST);
   NavigateToURLAndCheckSubresources(GetURL(kRedirectPath));
+  // TODO(alexilin): activate it after new NavigationID introduced.
+  // TestLearningAndPrefetching(GetURL(kRedirectPath));

[Benoit L, 2016/12/19 12:32:00]

nit: Don't put commented-out code. However the comment is fine.

[alexilin, 2016/12/19 12:58:09]

Done.
Is there some rule in styleguide about that or is this just a bad style?

[Benoit L, 2016/12/19 15:19:05]

I'm not sure, but I'm quite confident that no-one likes commented-out code. For
code that was there, we have git. For code that will be there, this is code that
has never ran, may not even compile, so has little value compared to not writing
it. And that's without taking into account all the times where I wrote code that
I thought "would be useful at some point"...


Can you update the comment as well? Something like "Test learning and
prefetching once crbug.com/650246 is fixed."

[alexilin, 2016/12/19 15:36:13]

Well, now even this never commited commented-out code is stored "forever" in the
code review tool.
Done.

 }
 
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
-                       LearningJavascriptDocumentWrite) {
+                       JavascriptDocumentWrite) {
   auto externalScript =
       AddExternalResource(GetURL(kScriptDocumentWritePath),
                           content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   externalScript->request.mime_type = kJavascriptMime;
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
-  NavigateToURLAndCheckSubresources(GetURL(kHtmlDocumentWritePath));
+  TestLearningAndPrefetching(GetURL(kHtmlDocumentWritePath));
 }
 
 // Disabled due to flakiness (crbug.com/673028).
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
-                       DISABLED_LearningJavascriptAppendChild) {
+                       JavascriptAppendChild) {
   auto externalScript =
       AddExternalResource(GetURL(kScriptAppendChildPath),
                           content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   externalScript->request.mime_type = kJavascriptMime;
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   // This script has net::LOWEST priority because it's executed asynchronously.
   AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::LOWEST);
-  NavigateToURLAndCheckSubresources(GetURL(kHtmlAppendChildPath));
+  TestLearningAndPrefetching(GetURL(kHtmlAppendChildPath));
 }
 
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
-                       LearningJavascriptInnerHtml) {
+                       JavascriptInnerHtml) {
   auto externalScript = AddExternalResource(
       GetURL(kScriptInnerHtmlPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   externalScript->request.mime_type = kJavascriptMime;
   AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   // https://www.w3.org/TR/2014/REC-html5-20141028/scripting-1.html#the-script-element
   // Script elements don't execute when inserted using innerHTML attribute.
   AddUnrecordedResources({GetURL(kScriptPath)});
-  NavigateToURLAndCheckSubresources(GetURL(kHtmlInnerHtmlPath));
+  TestLearningAndPrefetching(GetURL(kHtmlInnerHtmlPath));
 }
 
 // Requests originated by XMLHttpRequest have content::RESOURCE_TYPE_XHR.
 // Actual resource type is inferred from the mime-type.
-IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
-                       LearningJavascriptXHR) {
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest, JavascriptXHR) {
   auto externalScript = AddExternalResource(
       GetURL(kScriptXHRPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   externalScript->request.mime_type = kJavascriptMime;
   auto image = AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE,
                            net::HIGHEST);
   image->request.mime_type = kImageMime;
   auto style = AddResource(GetURL(kStylePath),
                            content::RESOURCE_TYPE_STYLESHEET, net::HIGHEST);
   style->request.mime_type = kStyleMime;
   auto script = AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT,
                             net::HIGHEST);
   script->request.mime_type = kJavascriptMime;
-  NavigateToURLAndCheckSubresources(GetURL(kHtmlXHRPath));
+  TestLearningAndPrefetching(GetURL(kHtmlXHRPath));
 }
 
 // ResourcePrefetchPredictor ignores all resources requested from subframes.
 IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
-                       LearningWithIframe) {
+                       IframeShouldBeIgnored) {
   // Included from html_iframe.html.
   AddResource(GetURL(kImagePath2), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
   AddResource(GetURL(kStylePath2), content::RESOURCE_TYPE_STYLESHEET,
               net::HIGHEST);
   AddResource(GetURL(kScriptPath2), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   // Included from <iframe src="html_subresources.html"> and not recored.
   AddUnrecordedResources({GetURL(kImagePath), GetURL(kStylePath),
                           GetURL(kScriptPath), GetURL(kFontPath)});
-  NavigateToURLAndCheckSubresources(GetURL(kHtmlIframePath));
+  TestLearningAndPrefetching(GetURL(kHtmlIframePath));
 }
 
 }  // namespace predictors