predictors: Prefetch resources from manifest.

chrome/browser/predictors/resource_prefetch_predictor_unittest.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/predictors/resource_prefetch_predictor.h"
 
 #include <iostream>
 #include <memory>
 #include <utility>
 
@@ skipping 1077 matching lines @@
   InitializeRedirectStat(host_redirect_data.add_redirect_endpoints(),
                          "facebook.com", 1, 0, 0);
   EXPECT_CALL(*mock_tables_.get(),
               UpdateRedirectData(host_redirect_data, PREFETCH_KEY_TYPE_HOST));
 
   predictor_->RecordMainFrameLoadComplete(fb_end);
   profile_->BlockUntilHistoryProcessesPendingRequests();
 }
 
 TEST_F(ResourcePrefetchPredictorTest, ManifestHostNotInDB) {
-  precache::PrecacheManifest manifest = CreateManifestData(1);
+  precache::PrecacheManifest manifest =
+      CreateManifestData(base::Time::Now().ToDoubleT());
   InitializePrecacheResource(manifest.add_resource(),
                              "http://cdn.google.com/script.js", 0.9);
   InitializePrecacheResource(manifest.add_resource(),
                              "http://cdn.google.com/style.css", 0.75);
 
   EXPECT_CALL(*mock_tables_.get(), UpdateManifestData("google.com", manifest));
 
   predictor_->OnManifestFetched("google.com", manifest);
 }
 
 TEST_F(ResourcePrefetchPredictorTest, ManifestHostInDB) {
   EXPECT_CALL(*mock_tables_.get(),
               GetAllData(Pointee(ContainerEq(PrefetchDataMap())),
                          Pointee(ContainerEq(PrefetchDataMap())),
                          Pointee(ContainerEq(RedirectDataMap())),
                          Pointee(ContainerEq(RedirectDataMap())),
                          Pointee(ContainerEq(ManifestDataMap())),
                          Pointee(ContainerEq(OriginDataMap()))))
       .WillOnce(SetArgPointee<4>(test_manifest_data_));
   ResetPredictor();
   InitializePredictor();
   EXPECT_EQ(2U, predictor_->manifest_table_cache_->size());
 
-  precache::PrecacheManifest manifest = CreateManifestData(1);
+  precache::PrecacheManifest manifest =
+      CreateManifestData(base::Time::Now().ToDoubleT());
   InitializePrecacheResource(manifest.add_resource(),
                              "http://google.com/image.jpg", 0.1);
 
   EXPECT_CALL(*mock_tables_.get(), UpdateManifestData("google.com", manifest));
 
   predictor_->OnManifestFetched("google.com", manifest);
 }
 
 TEST_F(ResourcePrefetchPredictorTest, ManifestHostNotInDBAndDBFull) {
   EXPECT_CALL(*mock_tables_.get(),
               GetAllData(Pointee(ContainerEq(PrefetchDataMap())),
                          Pointee(ContainerEq(PrefetchDataMap())),
                          Pointee(ContainerEq(RedirectDataMap())),
                          Pointee(ContainerEq(RedirectDataMap())),
                          Pointee(ContainerEq(ManifestDataMap())),
                          Pointee(ContainerEq(OriginDataMap()))))
       .WillOnce(SetArgPointee<4>(test_manifest_data_));
   ResetPredictor();
   InitializePredictor();
   EXPECT_EQ(2U, predictor_->manifest_table_cache_->size());
 
-  precache::PrecacheManifest manifest = CreateManifestData(1);
+  precache::PrecacheManifest manifest =
+      CreateManifestData(base::Time::Now().ToDoubleT());
   InitializePrecacheResource(manifest.add_resource(),
                              "http://en.wikipedia.org/logo.png", 1.0);
 
   EXPECT_CALL(*mock_tables_.get(),
               DeleteManifestData(std::vector<std::string>({"google.com"})));
 
   EXPECT_CALL(*mock_tables_.get(),
               UpdateManifestData("en.wikipedia.org", manifest));
 
   predictor_->OnManifestFetched("en.wikipedia.org", manifest);
 }
 
 TEST_F(ResourcePrefetchPredictorTest, ManifestUnknownFieldsRemoved) {
-  precache::PrecacheManifest manifest = CreateManifestData(1);
+  precache::PrecacheManifest manifest =
+      CreateManifestData(base::Time::Now().ToDoubleT());
   InitializePrecacheResource(manifest.add_resource(),
                              "http://cdn.google.com/script.js", 0.9);
   InitializePrecacheResource(manifest.add_resource(),
                              "http://cdn.google.com/style.css", 0.75);
 
   precache::PrecacheManifest manifest_with_unknown_fields(manifest);
   manifest_with_unknown_fields.mutable_id()->mutable_unknown_fields()->append(
       "DATA");
   manifest_with_unknown_fields.mutable_unknown_fields()->append("DATA");
   for (auto& resource : *manifest_with_unknown_fields.mutable_resource()) {
     resource.mutable_unknown_fields()->append("DATA");
   }
 
   int manifest_size = manifest.ByteSize();
   auto match_size = [manifest_size](const precache::PrecacheManifest& m) {
     return m.ByteSize() == manifest_size;
   };
   EXPECT_CALL(
       *mock_tables_.get(),
       UpdateManifestData("google.com",
                          testing::AllOf(manifest, testing::Truly(match_size))));
 
   predictor_->OnManifestFetched("google.com", manifest_with_unknown_fields);
 }
 
+TEST_F(ResourcePrefetchPredictorTest, ManifestTooOld) {
+  base::Time old_time = base::Time::Now() - base::TimeDelta::FromDays(7);
+  precache::PrecacheManifest manifest =
+      CreateManifestData(old_time.ToDoubleT());
+  InitializePrecacheResource(manifest.add_resource(),
+                             "http://cdn.google.com/script.js", 0.9);
+  InitializePrecacheResource(manifest.add_resource(),
+                             "http://cdn.google.com/style.css", 0.75);
+
+  // No calls to DB should happen.
+  predictor_->OnManifestFetched("google.com", manifest);
+}
+
 TEST_F(ResourcePrefetchPredictorTest, ManifestUnusedRemoved) {
   const std::string& script_url = "http://cdn.google.com/script.js";
   const std::string& style_url = "http://cdn.google.com/style.css";
   PrefetchData google = CreatePrefetchData("www.google.com");
   InitializeResourceData(google.add_resources(), script_url,
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   InitializeResourceData(google.add_resources(), style_url,
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert({google.primary_key(), google});
 
-  precache::PrecacheManifest manifest = CreateManifestData(1);
+  precache::PrecacheManifest manifest =
+      CreateManifestData(base::Time::Now().ToDoubleT());
   InitializePrecacheResource(manifest.add_resource(), script_url, 0.9);
   InitializePrecacheResource(manifest.add_resource(), style_url, 0.75);
   InitializeExperiment(&manifest, internal::kUnusedRemovedExperiment,
                        {true, false});
 
   // style_url should be removed.
   google.mutable_resources()->RemoveLast();
   EXPECT_CALL(*mock_tables_.get(),
               UpdateResourceData(google, PREFETCH_KEY_TYPE_HOST));
   EXPECT_CALL(*mock_tables_.get(), UpdateManifestData("google.com", manifest));
@@ skipping 560 matching lines @@
   EXPECT_FALSE(predictor_->PopulatePrefetcherRequest(nyt.primary_key(),
                                                      test_data, &urls));
   EXPECT_TRUE(urls.empty());
 
   urls.clear();
   EXPECT_FALSE(predictor_->PopulatePrefetcherRequest("http://404.com",
                                                      test_data, &urls));
   EXPECT_TRUE(urls.empty());
 }
 
+TEST_F(ResourcePrefetchPredictorTest, PopulateFromManifest) {
+  // The data that will be used in populating.
+  precache::PrecacheManifest google =
+      CreateManifestData(base::Time::Now().ToDoubleT());
+  InitializePrecacheResource(google.add_resource(),
+                             "https://static.google.com/good", 0.9);
+  InitializePrecacheResource(google.add_resource(),
+                             "https://static.google.com/low_confidence", 0.6);
+  InitializePrecacheResource(google.add_resource(),
+                             "https://static.google.com/versionned_removed",
+                             0.8);
+  InitializePrecacheResource(google.add_resource(),
+                             "https://static.google.com/unused_removed", 0.8);
+  InitializePrecacheResource(google.add_resource(),
+                             "https://static.google.com/no_store", 0.8);
+  InitializeExperiment(&google, internal::kVersionedRemovedExperiment,
+                       {true, true, false, true, true});
+  InitializeExperiment(&google, internal::kUnusedRemovedExperiment,
+                       {true, true, true, false, true});
+  InitializeExperiment(&google, internal::kNoStoreRemovedExperiment,
+                       {true, true, true, true, false});
+
+  // The data that's too old.
+  base::Time old_time = base::Time::Now() - base::TimeDelta::FromDays(7);
+  precache::PrecacheManifest facebook =
+      CreateManifestData(old_time.ToDoubleT());
+  InitializePrecacheResource(facebook.add_resource(),
+                             "https://static.facebook.com/good", 0.9);
+
+  predictor_->manifest_table_cache_->insert({"google.com", google});
+  predictor_->manifest_table_cache_->insert({"facebook.com", facebook});
+
+  std::vector<GURL> urls;
+  EXPECT_TRUE(predictor_->PopulateFromManifest("google.com", &urls));
+  EXPECT_THAT(urls,
+              UnorderedElementsAre(GURL("https://static.google.com/good")));
+
+  urls.clear();
+  EXPECT_FALSE(predictor_->PopulateFromManifest("facebook.com", &urls));
+  EXPECT_TRUE(urls.empty());
+
+  urls.clear();
+  EXPECT_FALSE(predictor_->PopulateFromManifest("404.com", &urls));
+  EXPECT_TRUE(urls.empty());
+}
+
 TEST_F(ResourcePrefetchPredictorTest, GetRedirectEndpoint) {
   // The data to be requested for the confident endpoint.
   RedirectData nyt = CreateRedirectData("http://nyt.com", 1);
   InitializeRedirectStat(nyt.add_redirect_endpoints(),
                          "https://mobile.nytimes.com", 10, 0, 0);
 
   // The data to be sure that other RedirectData won't be affected.
   RedirectData gogle = CreateRedirectData("http://gogle.com", 2);
   InitializeRedirectStat(gogle.add_redirect_endpoints(), "https://google.com",
                          100, 0, 0);
@@ skipping 34 matching lines @@
                                                &redirect_endpoint));
 }
 
 TEST_F(ResourcePrefetchPredictorTest, GetPrefetchData) {
   const GURL main_frame_url("http://google.com/?query=cats");
   ResourcePrefetchPredictor::Prediction prediction;
   std::vector<GURL>& urls = prediction.subresource_urls;
   // No prefetch data.
   EXPECT_FALSE(predictor_->GetPrefetchData(main_frame_url, &prediction));
 
+  // Add a manifest associated with the main frame host.
+  const std::string& resource_url = "https://static.google.com/resource";
+  precache::PrecacheManifest manifest =
+      CreateManifestData(base::Time::Now().ToDoubleT());
+  InitializePrecacheResource(manifest.add_resource(), resource_url, 0.9);
+  predictor_->manifest_table_cache_->insert({"google.com", manifest});
+
+  urls.clear();
+  EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
+  EXPECT_THAT(urls, UnorderedElementsAre(GURL(resource_url)));
+
   // Add a resource associated with the main frame host.
-  PrefetchData google_host = CreatePrefetchData("google.com", 1);
+  PrefetchData google_host = CreatePrefetchData("google.com", 2);
   const std::string script_url = "https://cdn.google.com/script.js";
   InitializeResourceData(google_host.add_resources(), script_url,
                          content::RESOURCE_TYPE_SCRIPT, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert(
       std::make_pair(google_host.primary_key(), google_host));
 
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(script_url)));
 
   // Add host-based redirect.
   RedirectData host_redirect = CreateRedirectData("google.com", 3);
   InitializeRedirectStat(host_redirect.add_redirect_endpoints(),
-                         "www.google.com", 10, 0, 0);
+                         "www.google.fr", 10, 0, 0);

[Benoit L, 2017/04/19 14:48:54]

🇫🇷 🐓

[alexilin, 2017/04/19 15:35:12]

Vive la France!
Actually, I've changed this because for a manifest there is no difference
between "google.com" and "www.google.com".

   predictor_->host_redirect_table_cache_->insert(
       std::make_pair(host_redirect.primary_key(), host_redirect));
 
   // Prediction failed: no data associated with the host redirect endpoint.
   urls.clear();
   EXPECT_FALSE(predictor_->GetPrefetchData(main_frame_url, &prediction));
 
   // Add a resource associated with host redirect endpoint.
-  PrefetchData www_google_host = CreatePrefetchData("www.google.com", 4);
+  PrefetchData www_google_host = CreatePrefetchData("www.google.fr", 4);
   const std::string style_url = "https://cdn.google.com/style.css";
   InitializeResourceData(www_google_host.add_resources(), style_url,
                          content::RESOURCE_TYPE_STYLESHEET, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->host_table_cache_->insert(
       std::make_pair(www_google_host.primary_key(), www_google_host));
 
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(style_url)));
 
   // Add a resource associated with the main frame url.
   PrefetchData google_url =
-      CreatePrefetchData("http://google.com/?query=cats", 2);
+      CreatePrefetchData("http://google.com/?query=cats", 5);
   const std::string image_url = "https://cdn.google.com/image.png";
   InitializeResourceData(google_url.add_resources(), image_url,
                          content::RESOURCE_TYPE_IMAGE, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->url_table_cache_->insert(
       std::make_pair(google_url.primary_key(), google_url));
 
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(image_url)));
 
   // Add url-based redirect.
   RedirectData url_redirect =
-      CreateRedirectData("http://google.com/?query=cats", 5);
+      CreateRedirectData("http://google.com/?query=cats", 6);
   InitializeRedirectStat(url_redirect.add_redirect_endpoints(),
                          "https://www.google.com/?query=cats", 10, 0, 0);
   predictor_->url_redirect_table_cache_->insert(
       std::make_pair(url_redirect.primary_key(), url_redirect));
 
   // Url redirect endpoint doesn't have associated resources so we get
   // host-based data.
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(style_url)));
 
   // Add a resource associated with url redirect endpoint.
   PrefetchData www_google_url =
-      CreatePrefetchData("https://www.google.com/?query=cats", 4);
+      CreatePrefetchData("https://www.google.com/?query=cats", 7);
   const std::string font_url = "https://cdn.google.com/comic-sans-ms.woff";
   InitializeResourceData(www_google_url.add_resources(), font_url,
                          content::RESOURCE_TYPE_FONT_RESOURCE, 10, 0, 1, 2.1,
                          net::MEDIUM, false, false);
   predictor_->url_table_cache_->insert(
       std::make_pair(www_google_url.primary_key(), www_google_url));
 
   urls.clear();
   EXPECT_TRUE(predictor_->GetPrefetchData(main_frame_url, &prediction));
   EXPECT_THAT(urls, UnorderedElementsAre(GURL(font_url)));
@@ skipping 103 matching lines @@
   predictor_->host_table_cache_->insert(
       std::make_pair(google.primary_key(), google));
 
   predictor_->StartPrefetching(GURL(main_frame_url), PrefetchOrigin::EXTERNAL);
   predictor_->StopPrefetching(GURL(main_frame_url));
   histogram_tester_->ExpectTotalCount(
       internal::kResourcePrefetchPredictorPrefetchingDurationHistogram, 1);
 }
 
 }  // namespace predictors