predictors: Add browsertests that check fetching through javascript.

chrome/browser/predictors/resource_prefetch_predictor_browsertest.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/command_line.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor_factory.h"
 #include "chrome/browser/predictors/resource_prefetch_predictor_test_util.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/browser/ui/browser.h"
 #include "chrome/browser/ui/tabs/tab_strip_model.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/test/base/in_process_browser_test.h"
 #include "chrome/test/base/ui_test_utils.h"
 #include "content/public/browser/render_frame_host.h"
 #include "content/public/browser/render_process_host.h"
 #include "net/test/embedded_test_server/http_request.h"
 #include "net/test/embedded_test_server/http_response.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace predictors {
 
 namespace {
 
-static const char kImagePath[] = "/predictors/image.png";
-static const char kStylePath[] = "/predictors/style.css";
-static const char kScriptPath[] = "/predictors/script.js";
-static const char kFontPath[] = "/predictors/font.ttf";
+// Paths to resources handled by a custom request handler. They return empty
+// responses with controllable response headers.
+static const char kImagePath[] = "/handled-by-test/image.png";
+static const char kImagePath2[] = "/handled-by-test/image2.png";
+static const char kStylePath[] = "/handled-by-test/style.css";
+static const char kStylePath2[] = "/handled-by-test/style2.css";
+static const char kScriptPath[] = "/handled-by-test/script.js";
+static const char kScriptPath2[] = "/handled-by-test/script2.js";
+static const char kFontPath[] = "/handled-by-test/font.ttf";
+static const char kRedirectPath[] = "/handled-by-test/redirect.html";
+static const char kRedirectPath2[] = "/handled-by-test/redirect2.html";
+static const char kRedirectPath3[] = "/handled-by-test/redirect3.html";
+
+// These are loaded from a file by the test server.
 static const char kHtmlSubresourcesPath[] =
     "/predictors/html_subresources.html";
-static const char kRedirectPath[] = "/predictors/redirect.html";
-static const char kRedirectPath2[] = "/predictors/redirect2.html";
-static const char kRedirectPath3[] = "/predictors/redirect3.html";
+static const char kHtmlDocumentWritePath[] = "/predictors/document_write.html";
+static const char kScriptDocumentWritePath[] = "/predictors/document_write.js";
+static const char kHtmlAppendChildPath[] = "/predictors/append_child.html";
+static const char kScriptAppendChildPath[] = "/predictors/append_child.js";
+static const char kHtmlInnerHtmlPath[] = "/predictors/inner_html.html";
+static const char kScriptInnerHtmlPath[] = "/predictors/inner_html.js";
+static const char kHtmlFetchPath[] = "/predictors/fetch.html";
+static const char kScriptFetchPath[] = "/predictors/fetch.js";
+static const char kHtmlIframePath[] = "/predictors/html_iframe.html";
 
 struct ResourceSummary {
-  ResourceSummary() : is_no_store(false), version(0) {}
+  ResourceSummary()
+      : is_no_store(false),
+        version(0),
+        is_external(false),
+        should_be_recorded(true) {}
 
   ResourcePrefetchPredictor::URLRequestSummary request;
   std::string content;
   bool is_no_store;
   size_t version;
+  bool is_external;
+  bool should_be_recorded;
 };
 
 struct RedirectEdge {
   // This response code should be returned by previous url in the chain.
   net::HttpStatusCode code;
   GURL url;
 };
 
 class InitializationObserver : public TestObserver {
  public:
@@ skipping 92 matching lines @@
     predictor_ =
         ResourcePrefetchPredictorFactory::GetForProfile(browser()->profile());
     ASSERT_TRUE(predictor_);
     EnsurePredictorInitialized();
   }
 
   void NavigateToURLAndCheckSubresources(const GURL& main_frame_url) {
     GURL endpoint_url = GetRedirectEndpoint(main_frame_url);
     std::vector<URLRequestSummary> url_request_summaries;
     for (const auto& kv : resources_) {
-      if (kv.second.is_no_store)
+      if (kv.second.is_no_store || !kv.second.should_be_recorded)
         continue;
       url_request_summaries.push_back(
           GetURLRequestSummaryForResource(endpoint_url, kv.second));
     }
     ResourcePrefetchPredictorTestObserver observer(
         predictor_, UpdateAndGetVisitCount(main_frame_url),
         CreatePageRequestSummary(endpoint_url.spec(), main_frame_url.spec(),
                                  url_request_summaries));
     ui_test_utils::NavigateToURL(browser(), main_frame_url);
     observer.Wait();
   }
 
   ResourceSummary* AddResource(const GURL& resource_url,
                                content::ResourceType resource_type,
                                net::RequestPriority priority) {
     auto pair_and_whether_inserted =
         resources_.insert(std::make_pair(resource_url, ResourceSummary()));
     EXPECT_TRUE(pair_and_whether_inserted.second) << resource_url
                                                   << " was inserted twice";
     ResourceSummary* resource = &pair_and_whether_inserted.first->second;
     resource->request.resource_url = resource_url;
     resource->request.resource_type = resource_type;
     resource->request.priority = priority;
     resource->request.has_validators = true;
     return resource;
   }
 
+  ResourceSummary* AddExternalResource(const GURL& resource_url,
+                                       content::ResourceType resource_type,
+                                       net::RequestPriority priority) {
+    auto resource = AddResource(resource_url, resource_type, priority);
+    resource->is_external = true;
+    return resource;
+  }
+
+  void AddUnrecordedResources(const std::vector<GURL>& resource_urls) {
+    for (const GURL& resource_url : resource_urls) {
+      auto resource =
+          AddResource(resource_url, content::RESOURCE_TYPE_SUB_RESOURCE,
+                      net::DEFAULT_PRIORITY);
+      resource->should_be_recorded = false;
+    }
+  }
+
   void AddRedirectChain(const GURL& initial_url,
                         const std::vector<RedirectEdge>& redirect_chain) {
     ASSERT_FALSE(redirect_chain.empty());
     GURL current = initial_url;
     for (const auto& edge : redirect_chain) {
       auto result = redirects_.insert(std::make_pair(current, edge));
       EXPECT_TRUE(result.second) << current << " already has a redirect.";
       current = edge.url;
     }
   }
@@ skipping 67 matching lines @@
     return current;
   }
 
   std::unique_ptr<net::test_server::HttpResponse> HandleResourceRequest(
       const net::test_server::HttpRequest& request) const {
     auto resource_it = resources_.find(request.GetURL());
     if (resource_it == resources_.end())
       return nullptr;
 
     const ResourceSummary& summary = resource_it->second;
+    if (summary.is_external)
+      return nullptr;
+
     auto http_response =
         base::MakeUnique<net::test_server::BasicHttpResponse>();
     http_response->set_code(net::HTTP_OK);
     if (!summary.request.mime_type.empty())
       http_response->set_content_type(summary.request.mime_type);
     if (!summary.content.empty())
       http_response->set_content(summary.content);
     if (summary.is_no_store)
       http_response->AddCustomHeader("Cache-Control", "no-store");
     if (summary.request.has_validators) {
@@ skipping 83 matching lines @@
               net::LOWEST);
   AddResource(https_server()->GetURL(kStylePath),
               content::RESOURCE_TYPE_STYLESHEET, net::HIGHEST);
   AddResource(https_server()->GetURL(kScriptPath),
               content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
   AddResource(https_server()->GetURL(kFontPath),
               content::RESOURCE_TYPE_FONT_RESOURCE, net::HIGHEST);
   NavigateToURLAndCheckSubresources(GetURL(kRedirectPath));
 }
 
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
+                       LearningJavascriptDocumentWrite) {
+  auto externalScript =
+      AddExternalResource(GetURL(kScriptDocumentWritePath),
+                          content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  externalScript->request.mime_type = "application/javascript";
+  AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
+  AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
+              net::HIGHEST);
+  AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlDocumentWritePath));
+}
+
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
+                       LearningJavascriptAppendChild) {
+  auto externalScript =
+      AddExternalResource(GetURL(kScriptAppendChildPath),
+                          content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  externalScript->request.mime_type = "application/javascript";
+  AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
+  AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
+              net::HIGHEST);
+  // This script has net::LOWEST priority because it's executed asynchronously.
+  AddResource(GetURL(kScriptPath), content::RESOURCE_TYPE_SCRIPT, net::LOWEST);
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlAppendChildPath));
+}
+
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
+                       LearningJavascriptInnerHtml) {
+  auto externalScript = AddExternalResource(
+      GetURL(kScriptInnerHtmlPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  externalScript->request.mime_type = "application/javascript";
+  AddResource(GetURL(kImagePath), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
+  AddResource(GetURL(kStylePath), content::RESOURCE_TYPE_STYLESHEET,
+              net::HIGHEST);
+  // The script isn't requested. Need furhter investigation.

[Benoit L, 2016/12/05 14:03:30]

nit: further.

[alexilin, 2016/12/06 09:10:44]

Done.

+  AddUnrecordedResources({GetURL(kScriptPath)});
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlInnerHtmlPath));
+}
+
+// Javascript fetch requests are ignored by the ResourcePrefetchPredictor
+// because they have unsupported resource type content::RESOURCE_TYPE_XHR.

[Benoit L, 2016/12/05 14:03:30]

Oh, that's a good find!
I wonder what we should do about that.

[alexilin, 2016/12/06 09:10:44]

Already done:
https://codereview.chromium.org/2539263004/

+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
+                       LearningJavascriptFetchIgnored) {
+  auto externalScript = AddExternalResource(
+      GetURL(kScriptFetchPath), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  externalScript->request.mime_type = "application/javascript";
+  AddUnrecordedResources(
+      {GetURL(kImagePath), GetURL(kStylePath), GetURL(kScriptPath)});
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlFetchPath));
+}
+
+// ResourcePrefetchPredictor ignores all resources requested from subframes.
+IN_PROC_BROWSER_TEST_F(ResourcePrefetchPredictorBrowserTest,
+                       LearningWithIframe) {
+  // Included from html_iframe.html.
+  AddResource(GetURL(kImagePath2), content::RESOURCE_TYPE_IMAGE, net::LOWEST);
+  AddResource(GetURL(kStylePath2), content::RESOURCE_TYPE_STYLESHEET,
+              net::HIGHEST);
+  AddResource(GetURL(kScriptPath2), content::RESOURCE_TYPE_SCRIPT, net::MEDIUM);
+  // Included from <iframe src="html_subresources.html"> and not recored.
+  AddUnrecordedResources({GetURL(kImagePath), GetURL(kStylePath),
+                          GetURL(kScriptPath), GetURL(kFontPath)});
+  NavigateToURLAndCheckSubresources(GetURL(kHtmlIframePath));
+}
+
 }  // namespace predictors