predictors: Several improvements for redirects database.

chrome/browser/predictors/resource_prefetch_predictor_unittest.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/predictors/resource_prefetch_predictor.h"
 
 #include <iostream>
 #include <memory>
 #include <utility>
 
@@ skipping 216 matching lines @@
     profile_->BlockUntilHistoryProcessesPendingRequests();
   }
 
   void ResetPredictor() {
     ResourcePrefetchPredictorConfig config;
     config.max_urls_to_track = 3;
     config.max_hosts_to_track = 2;
     config.min_url_visit_count = 2;
     config.max_resources_per_entry = 4;
     config.max_consecutive_misses = 2;
+    config.max_redirect_consecutive_misses = 2;
     config.min_resource_confidence_to_trigger_prefetch = 0.5;
     config.is_url_learning_enabled = true;
 
     config.mode |= ResourcePrefetchPredictorConfig::LEARNING;
     predictor_.reset(new ResourcePrefetchPredictor(config, profile_.get()));
     predictor_->set_mock_tables(mock_tables_);
   }
 
   void InitializeSampleData();
   void TestRedirectStatusHistogram(
@@ skipping 346 matching lines @@
   InitializeResourceData(
       host_data.add_resources(), "https://google.com/script1.js",
       content::RESOURCE_TYPE_SCRIPT, 1, 0, 0, 2.0, net::MEDIUM, false, false);
   InitializeResourceData(
       host_data.add_resources(), "https://google.com/script2.js",
       content::RESOURCE_TYPE_SCRIPT, 1, 0, 0, 3.0, net::MEDIUM, false, false);
   EXPECT_CALL(*mock_tables_.get(),
               UpdateData(empty_resource_data_, host_data, empty_redirect_data_,
                          empty_redirect_data_));
 
+  RedirectData host_redirect_data = CreateRedirectData("www.google.com");
+  InitializeRedirectStat(host_redirect_data.add_redirect_endpoints(),
+                         "www.google.com", 1, 0, 0);
+  EXPECT_CALL(*mock_tables_.get(),
+              UpdateData(empty_resource_data_, empty_resource_data_,
+                         empty_redirect_data_, host_redirect_data));
+
   predictor_->RecordMainFrameLoadComplete(main_frame.navigation_id);
   profile_->BlockUntilHistoryProcessesPendingRequests();
 }
 
 // Single navigation that will be recorded. Will check for duplicate
 // resources and also for number of resources saved.
 TEST_F(ResourcePrefetchPredictorTest, NavigationUrlNotInDB) {
   const int kVisitCount = 4;
   AddUrlToHistory("http://www.google.com", kVisitCount);
 
@@ skipping 58 matching lines @@
   EXPECT_CALL(*mock_tables_.get(),
               UpdateData(url_data, empty_resource_data_, empty_redirect_data_,
                          empty_redirect_data_));
 
   PrefetchData host_data = CreatePrefetchData("www.google.com");
   host_data.mutable_resources()->CopyFrom(url_data.resources());
   EXPECT_CALL(*mock_tables_.get(),
               UpdateData(empty_resource_data_, host_data, empty_redirect_data_,
                          empty_redirect_data_));
 
+  RedirectData url_redirect_data = CreateRedirectData("http://www.google.com/");
+  InitializeRedirectStat(url_redirect_data.add_redirect_endpoints(),
+                         "http://www.google.com/", 1, 0, 0);
+  EXPECT_CALL(*mock_tables_.get(),
+              UpdateData(empty_resource_data_, empty_resource_data_,
+                         url_redirect_data, empty_redirect_data_));
+
+  RedirectData host_redirect_data = CreateRedirectData("www.google.com");
+  InitializeRedirectStat(host_redirect_data.add_redirect_endpoints(),
+                         "www.google.com", 1, 0, 0);
+  EXPECT_CALL(*mock_tables_.get(),
+              UpdateData(empty_resource_data_, empty_resource_data_,
+                         empty_redirect_data_, host_redirect_data));
+
   predictor_->RecordMainFrameLoadComplete(main_frame.navigation_id);
   profile_->BlockUntilHistoryProcessesPendingRequests();
 }
 
 // Tests that navigation is recorded correctly for URL already present in
 // the database cache.
 TEST_F(ResourcePrefetchPredictorTest, NavigationUrlInDB) {
   const int kVisitCount = 4;
   AddUrlToHistory("http://www.google.com", kVisitCount);
 
@@ skipping 87 matching lines @@
       host_data.add_resources(), "http://google.com/script2.js",
       content::RESOURCE_TYPE_SCRIPT, 1, 0, 0, 3.0, net::MEDIUM, false, false);
   InitializeResourceData(host_data.add_resources(),
                          "http://google.com/style2.css",
                          content::RESOURCE_TYPE_STYLESHEET, 1, 0, 0, 7.0,
                          net::MEDIUM, false, false);
   EXPECT_CALL(*mock_tables_.get(),
               UpdateData(empty_resource_data_, host_data, empty_redirect_data_,
                          empty_redirect_data_));
 
+  RedirectData url_redirect_data = CreateRedirectData("http://www.google.com/");
+  InitializeRedirectStat(url_redirect_data.add_redirect_endpoints(),
+                         "http://www.google.com/", 1, 0, 0);
+  EXPECT_CALL(*mock_tables_.get(),
+              UpdateData(empty_resource_data_, empty_resource_data_,
+                         url_redirect_data, empty_redirect_data_));
+
+  RedirectData host_redirect_data = CreateRedirectData("www.google.com");
+  InitializeRedirectStat(host_redirect_data.add_redirect_endpoints(),
+                         "www.google.com", 1, 0, 0);
+  EXPECT_CALL(*mock_tables_.get(),
+              UpdateData(empty_resource_data_, empty_resource_data_,
+                         empty_redirect_data_, host_redirect_data));
+
   predictor_->RecordMainFrameLoadComplete(main_frame.navigation_id);
   profile_->BlockUntilHistoryProcessesPendingRequests();
 }
 
 // Tests that a URL is deleted before another is added if the cache is full.
 TEST_F(ResourcePrefetchPredictorTest, NavigationUrlNotInDBAndDBFull) {
   const int kVisitCount = 4;
   AddUrlToHistory("http://www.nike.com/", kVisitCount);
 
   EXPECT_CALL(*mock_tables_.get(),
@@ skipping 49 matching lines @@
   EXPECT_CALL(*mock_tables_.get(),
               UpdateData(url_data, empty_resource_data_, empty_redirect_data_,
                          empty_redirect_data_));
 
   PrefetchData host_data = CreatePrefetchData("www.nike.com");
   host_data.mutable_resources()->CopyFrom(url_data.resources());
   EXPECT_CALL(*mock_tables_.get(),
               UpdateData(empty_resource_data_, host_data, empty_redirect_data_,
                          empty_redirect_data_));
 
+  RedirectData url_redirect_data = CreateRedirectData("http://www.nike.com/");
+  InitializeRedirectStat(url_redirect_data.add_redirect_endpoints(),
+                         "http://www.nike.com/", 1, 0, 0);
+  EXPECT_CALL(*mock_tables_.get(),
+              UpdateData(empty_resource_data_, empty_resource_data_,
+                         url_redirect_data, empty_redirect_data_));
+
+  RedirectData host_redirect_data = CreateRedirectData("www.nike.com");
+  InitializeRedirectStat(host_redirect_data.add_redirect_endpoints(),
+                         "www.nike.com", 1, 0, 0);
+  EXPECT_CALL(*mock_tables_.get(),
+              UpdateData(empty_resource_data_, empty_resource_data_,
+                         empty_redirect_data_, host_redirect_data));
+
   predictor_->RecordMainFrameLoadComplete(main_frame.navigation_id);
   profile_->BlockUntilHistoryProcessesPendingRequests();
 }
 
 TEST_F(ResourcePrefetchPredictorTest, RedirectUrlNotInDB) {
   const int kVisitCount = 4;
   AddUrlToHistory("https://facebook.com/google", kVisitCount);
 
   URLRequestSummary fb1 = CreateURLRequestSummary(1, "http://fb.com/google");
   predictor_->RecordURLRequest(fb1);
@@ skipping 759 matching lines @@
                                                      test_data, &urls));
   EXPECT_TRUE(urls.empty());
 
   urls.clear();
   EXPECT_FALSE(predictor_->PopulatePrefetcherRequest("http://404.com",
                                                      test_data, &urls));
   EXPECT_TRUE(urls.empty());
 }
 
 TEST_F(ResourcePrefetchPredictorTest, GetRedirectEndpoint) {
-  // The data to be requested for the most confident endpoint.
-  RedirectData google = CreateRedirectData("http://google.com/", 1);
+  // The data to be requested for the confident endpoint.
+  RedirectData nyt = CreateRedirectData("http://nyt.com", 1);
+  InitializeRedirectStat(nyt.add_redirect_endpoints(),
+                         "https://mobile.nytimes.com", 10, 0, 0);
+
+  // The data to be sure that other RedirectData won't be affected.
+  RedirectData gogle = CreateRedirectData("http://gogle.com", 2);
+  InitializeRedirectStat(gogle.add_redirect_endpoints(), "https://google.com",
+                         100, 0, 0);
+
+  // The data to check negative result due not enough confidence.
+  RedirectData facebook = CreateRedirectData("http://fb.com", 3);
+  InitializeRedirectStat(facebook.add_redirect_endpoints(),
+                         "http://facebook.com", 5, 5, 0);
+
+  // The data to check negative result due ambiguity.
+  RedirectData google = CreateRedirectData("http://google.com", 4);
   InitializeRedirectStat(google.add_redirect_endpoints(), "https://google.com",
                          10, 0, 0);
   InitializeRedirectStat(google.add_redirect_endpoints(), "https://google.fr",
                          10, 1, 0);
   InitializeRedirectStat(google.add_redirect_endpoints(), "https://google.ws",
                          20, 20, 0);
 
-  // The data to be sure that other RedirectData won't be affected.
-  RedirectData gogle = CreateRedirectData("http://gogle.com", 2);
-  InitializeRedirectStat(gogle.add_redirect_endpoints(), "https://google.com",
-                         100, 0, 0);
-
-  // The data to check negative result.
-  RedirectData facebook = CreateRedirectData("http://fb.com/", 3);
-  InitializeRedirectStat(facebook.add_redirect_endpoints(),
-                         "http://facebook.com", 5, 5,
-                         0);  // not enough confidence
-
   RedirectDataMap data_map;
-  data_map.insert(std::make_pair(google.primary_key(), google));
+  data_map.insert(std::make_pair(nyt.primary_key(), nyt));
   data_map.insert(std::make_pair(gogle.primary_key(), gogle));
   data_map.insert(std::make_pair(facebook.primary_key(), facebook));
+  data_map.insert(std::make_pair(google.primary_key(), google));
 
   std::string redirect_endpoint;
-  EXPECT_TRUE(predictor_->GetRedirectEndpoint("http://google.com/", data_map,
+  EXPECT_TRUE(predictor_->GetRedirectEndpoint("http://nyt.com", data_map,
                                               &redirect_endpoint));
-  EXPECT_EQ(redirect_endpoint, "https://google.com");
+  EXPECT_EQ(redirect_endpoint, "https://mobile.nytimes.com");
+
+  // Returns the initial url if data_map doesn't contain an entry for the url.
+  EXPECT_TRUE(predictor_->GetRedirectEndpoint("http://bbc.com", data_map,
+                                              &redirect_endpoint));
+  EXPECT_EQ(redirect_endpoint, "http://bbc.com");
 
   EXPECT_FALSE(predictor_->GetRedirectEndpoint("http://fb.com", data_map,
                                                &redirect_endpoint));
-  EXPECT_FALSE(predictor_->GetRedirectEndpoint("http://404.com", data_map,
+  EXPECT_FALSE(predictor_->GetRedirectEndpoint("http://google.com", data_map,
                                                &redirect_endpoint));
 }
 
 TEST_F(ResourcePrefetchPredictorTest, GetPrefetchData) {
   const GURL main_frame_url("http://google.com/?query=cats");
   ResourcePrefetchPredictor::Prediction prediction;
   std::vector<GURL>& urls = prediction.subresource_urls;
   // No prefetch data.
   EXPECT_FALSE(predictor_->GetPrefetchData(main_frame_url, &prediction));
 
   // Add a resource associated with the main frame host.
   PrefetchData google_host = CreatePrefetchData("google.com", 1);
   const std::string script_url = "https://cdn.google.com/script.js";
   InitializeResourceData(google_host.add_resources(), script_url,
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert(
       std::make_pair(google_host.primary_key(), google_host));
 
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(script_url)));
 
-  // Add a resource associated with the main frame url.
-  PrefetchData google_url =
-      CreatePrefetchData("http://google.com/?query=cats", 2);
-  const std::string image_url = "https://cdn.google.com/image.png";
-  InitializeResourceData(google_url.add_resources(), image_url,
-                         content::RESOURCE_TYPE_IMAGE, 10, 0, 1, 2.1,
-                         net::MEDIUM, false, false);
-  predictor_->url_table_cache_->insert(
-      std::make_pair(google_url.primary_key(), google_url));
-
-  urls.clear();
-  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
-  EXPECT_THAT(urls, UnorderedElementsAre(GURL(image_url)));
-
   // Add host-based redirect.
   RedirectData host_redirect = CreateRedirectData("google.com", 3);
   InitializeRedirectStat(host_redirect.add_redirect_endpoints(),
                          "www.google.com", 10, 0, 0);
   predictor_->host_redirect_table_cache_->insert(
       std::make_pair(host_redirect.primary_key(), host_redirect));
 
-  // Nothing changed: new redirect endpoint doesn't have any associated
-  // resources
+  // Prediction failed: no data associated with the host redirect endpoint.
   urls.clear();
-  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
-  EXPECT_THAT(urls, UnorderedElementsAre(GURL(image_url)));
+  EXPECT_FALSE(predictor_->GetPrefetchData(main_frame_url, &prediction));
 
   // Add a resource associated with host redirect endpoint.
   PrefetchData www_google_host = CreatePrefetchData("www.google.com", 4);
   const std::string style_url = "https://cdn.google.com/style.css";
   InitializeResourceData(www_google_host.add_resources(), style_url,
                          content::RESOURCE_TYPE_STYLESHEET, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert(
       std::make_pair(www_google_host.primary_key(), www_google_host));
 
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(style_url)));
 
+  // Add a resource associated with the main frame url.
+  PrefetchData google_url =
+      CreatePrefetchData("http://google.com/?query=cats", 2);
+  const std::string image_url = "https://cdn.google.com/image.png";
+  InitializeResourceData(google_url.add_resources(), image_url,
+                         content::RESOURCE_TYPE_IMAGE, 10, 0, 1, 2.1,
+                         net::MEDIUM, false, false);
+  predictor_->url_table_cache_->insert(
+      std::make_pair(google_url.primary_key(), google_url));
+
+  urls.clear();
+  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
+  EXPECT_THAT(urls, UnorderedElementsAre(GURL(image_url)));
+
   // Add url-based redirect.
   RedirectData url_redirect =
       CreateRedirectData("http://google.com/?query=cats", 5);
   InitializeRedirectStat(url_redirect.add_redirect_endpoints(),
                          "https://www.google.com/?query=cats", 10, 0, 0);
   predictor_->url_redirect_table_cache_->insert(
       std::make_pair(url_redirect.primary_key(), url_redirect));
 
-  // Url redirect endpoint doesn't have associated resources.
+  // Url redirect endpoint doesn't have associated resources so we get
+  // host-based data.
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(style_url)));
 
   // Add a resource associated with url redirect endpoint.
   PrefetchData www_google_url =
       CreatePrefetchData("https://www.google.com/?query=cats", 4);
   const std::string font_url = "https://cdn.google.com/comic-sans-ms.woff";
   InitializeResourceData(www_google_url.add_resources(), font_url,
                          content::RESOURCE_TYPE_FONT_RESOURCE, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->url_table_cache_->insert(
       std::make_pair(www_google_url.primary_key(), www_google_url));
 
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(font_url)));
 }
 
 TEST_F(ResourcePrefetchPredictorTest, TestPrecisionRecallHistograms) {
   using testing::_;
-  EXPECT_CALL(*mock_tables_.get(), UpdateData(_, _, _, _));
+  EXPECT_CALL(*mock_tables_.get(), UpdateData(_, _, _, _)).Times(2);
 
   // Fill the database with 3 resources: 1 useful, 2 useless.
   const std::string main_frame_url = "http://google.com/?query=cats";
   PrefetchData google = CreatePrefetchData("google.com", 1);
 
   const std::string script_url = "https://cdn.google.com/script.js";
   InitializeResourceData(google.add_resources(), script_url,
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   InitializeResourceData(google.add_resources(), script_url + "foo",
@@ skipping 86 matching lines @@
   predictor_->host_table_cache_->insert(
       std::make_pair(google.primary_key(), google));
 
   predictor_->StartPrefetching(GURL(main_frame_url), PrefetchOrigin::EXTERNAL);
   predictor_->StopPrefetching(GURL(main_frame_url));
   histogram_tester_->ExpectTotalCount(
       internal::kResourcePrefetchPredictorPrefetchingDurationHistogram, 1);
 }
 
 }  // namespace predictors