Send the list of used and unused resources for precache

components/precache/core/precache_database_unittest.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_database.h"
 
 #include <stdint.h>
 
 #include <map>
 #include <memory>
@@ skipping 14 matching lines @@
 #include "url/gurl.h"
 
 namespace {
 
 using ::testing::ContainerEq;
 using ::testing::ElementsAre;
 using base::Bucket;
 using net::HttpResponseInfo;
 
 const GURL kURL("http://url.com");
+const int kReferrerID = 1;
 const base::TimeDelta kLatency = base::TimeDelta::FromMilliseconds(5);
 const base::Time kFetchTime = base::Time() + base::TimeDelta::FromHours(1000);
 const base::Time kOldFetchTime = kFetchTime - base::TimeDelta::FromDays(1);
+const base::Time kNewFetchTime =
+    base::Time() + base::TimeDelta::FromHours(2000);
 const int64_t kSize = 5000;
 const int64_t kFreshnessBucket10K = 9089;
 // One of the possible CacheEntryStatus for when the fetch was served from the
 // network.
 const HttpResponseInfo::CacheEntryStatus kFromNetwork =
     HttpResponseInfo::CacheEntryStatus::ENTRY_UPDATED;
 
 std::map<GURL, base::Time> BuildURLTableMap(const GURL& url,
                                             const base::Time& precache_time) {
   std::map<GURL, base::Time> url_table_map;
@@ skipping 38 matching lines @@
   PrecacheDatabaseTest() {}
   ~PrecacheDatabaseTest() override {}
 
  protected:
   void SetUp() override {
     precache_database_.reset(new PrecacheDatabase());
 
     ASSERT_TRUE(scoped_temp_dir_.CreateUniqueTempDir());
     base::FilePath db_path = scoped_temp_dir_.path().Append(
         base::FilePath(FILE_PATH_LITERAL("precache_database")));
-    precache_database_->Init(db_path);
+    ASSERT_TRUE(precache_database_->Init(db_path));
   }
 
   std::map<GURL, base::Time> GetActualURLTableMap() {
     // Flush any buffered writes so that the URL table will be up to date.
     precache_database_->Flush();
 
     std::map<GURL, base::Time> url_table_map;
     precache_url_table()->GetAllDataForTesting(&url_table_map);
     return url_table_map;
   }
 
   PrecacheURLTable* precache_url_table() {
     return &precache_database_->precache_url_table_;
   }
 
+  void Flush() { precache_database_->Flush(); }
+
   // Convenience methods for recording different types of URL fetches. These
   // exist to improve the readability of the tests.
   void RecordPrecacheFromNetwork(const GURL& url,
                                  base::TimeDelta latency,
                                  const base::Time& fetch_time,
                                  int64_t size);
   void RecordPrecacheFromCache(const GURL& url,
                                const base::Time& fetch_time,
                                int64_t size);
   void RecordFetchFromNetwork(const GURL& url,
@@ skipping 39 matching lines @@
   }
 };
 
 void PrecacheDatabaseTest::RecordPrecacheFromNetwork(
     const GURL& url,
     base::TimeDelta latency,
     const base::Time& fetch_time,
     int64_t size) {
   const HttpResponseInfo info = CreateHttpResponseInfo(
       false /* was_cached */, false /* network_accessed */);
-  precache_database_->RecordURLPrefetch(url, latency, fetch_time, info, size);
+  precache_database_->RecordURLPrefetchMetrics(info, latency);
+  precache_database_->RecordURLPrefetch(url, std::string(), fetch_time,
+                                        info.was_cached, size);
 }
 
 void PrecacheDatabaseTest::RecordPrecacheFromCache(const GURL& url,
                                                    const base::Time& fetch_time,
                                                    int64_t size) {
   const HttpResponseInfo info = CreateHttpResponseInfo(
       true /* was_cached */, false /* network_accessed */);
-  precache_database_->RecordURLPrefetch(url, base::TimeDelta() /* latency */,
-                                        fetch_time, info, size);
+  precache_database_->RecordURLPrefetchMetrics(info,
+                                               base::TimeDelta() /* latency */);
+  precache_database_->RecordURLPrefetch(url, std::string(), fetch_time,
+                                        info.was_cached, size);
 }
 
 void PrecacheDatabaseTest::RecordFetchFromNetwork(const GURL& url,
                                                   base::TimeDelta latency,
                                                   const base::Time& fetch_time,
                                                   int64_t size) {
   const HttpResponseInfo info = CreateHttpResponseInfo(
       false /* was_cached */, false /* network_accessed */);
   precache_database_->RecordURLNonPrefetch(url, latency, fetch_time, info, size,
                                            history::kMaxTopHosts,
@@ skipping 52 matching lines @@
 
   EXPECT_EQ(BuildURLTableMap(kURL, kFetchTime), GetActualURLTableMap());
 
   ExpectNewSample("Precache.DownloadedPrecacheMotivated", kSize);
   ExpectNewSample("Precache.Latency.Prefetch", kLatency.InMilliseconds());
   ExpectNewSample("Precache.Freshness.Prefetch", kFreshnessBucket10K);
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, PrecacheFromCacheWithURLTableEntry) {
-  precache_url_table()->AddURL(kURL, kOldFetchTime);
+  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime);
   RecordPrecacheFromCache(kURL, kFetchTime, kSize);
 
   // The URL table entry should have been updated to have |kFetchTime| as the
   // timestamp.
   EXPECT_EQ(BuildURLTableMap(kURL, kFetchTime), GetActualURLTableMap());
 
   ExpectNewSample("Precache.Latency.Prefetch", 0);
   ExpectNewSample("Precache.Freshness.Prefetch", kFreshnessBucket10K);
   ExpectNoOtherSamples();
 }
@@ skipping 42 matching lines @@
   ExpectNewSample("Precache.DownloadedNonPrecache", kSize);
   ExpectNewSample("Precache.DownloadedNonPrecache.Cellular", kSize);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch", kFromNetwork);
   ExpectNewSample("Precache.Latency.NonPrefetch", kLatency.InMilliseconds());
   ExpectNewSample("Precache.Latency.NonPrefetch.NonTopHosts",
                   kLatency.InMilliseconds());
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, FetchOverNetworkWithURLTableEntry) {
-  precache_url_table()->AddURL(kURL, kOldFetchTime);
+  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime);
   RecordFetchFromNetwork(kURL, kLatency, kFetchTime, kSize);
 
   // The URL table entry should have been deleted.
   EXPECT_TRUE(GetActualURLTableMap().empty());
 
   ExpectNewSample("Precache.DownloadedNonPrecache", kSize);
   ExpectNewSample("Precache.Latency.NonPrefetch", kLatency.InMilliseconds());
   ExpectNewSample("Precache.Latency.NonPrefetch.NonTopHosts",
                   kLatency.InMilliseconds());
   ExpectNewSample("Precache.CacheStatus.NonPrefetch", kFromNetwork);
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, FetchFromCacheWithURLTableEntry_NonCellular) {
-  precache_url_table()->AddURL(kURL, kOldFetchTime);
+  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime);
   RecordFetchFromCache(kURL, kFetchTime, kSize);
 
   // The URL table entry should have been deleted.
   EXPECT_TRUE(GetActualURLTableMap().empty());
 
   ExpectNewSample("Precache.Latency.NonPrefetch", 0);
   ExpectNewSample("Precache.Latency.NonPrefetch.NonTopHosts", 0);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch",
                   HttpResponseInfo::CacheEntryStatus::ENTRY_USED);
   ExpectNewSample("Precache.Saved", kSize);
   ExpectNewSample("Precache.Saved.Freshness", kFreshnessBucket10K);
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, FetchFromCacheWithURLTableEntry_Cellular) {
-  precache_url_table()->AddURL(kURL, kOldFetchTime);
+  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime);
   RecordFetchFromCacheCellular(kURL, kFetchTime, kSize);
 
   // The URL table entry should have been deleted.
   EXPECT_TRUE(GetActualURLTableMap().empty());
 
   ExpectNewSample("Precache.Latency.NonPrefetch", 0);
   ExpectNewSample("Precache.Latency.NonPrefetch.NonTopHosts", 0);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch",
                   HttpResponseInfo::CacheEntryStatus::ENTRY_USED);
   ExpectNewSample("Precache.Saved", kSize);
@@ skipping 12 matching lines @@
   ExpectNewSample("Precache.CacheStatus.NonPrefetch",
                   HttpResponseInfo::CacheEntryStatus::ENTRY_USED);
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, DeleteExpiredPrecacheHistory) {
   const base::Time kToday = base::Time() + base::TimeDelta::FromDays(1000);
   const base::Time k59DaysAgo = kToday - base::TimeDelta::FromDays(59);
   const base::Time k61DaysAgo = kToday - base::TimeDelta::FromDays(61);
 
-  precache_url_table()->AddURL(GURL("http://expired-precache.com"), k61DaysAgo);
-  precache_url_table()->AddURL(GURL("http://old-precache.com"), k59DaysAgo);
+  precache_url_table()->AddURL(GURL("http://expired-precache.com"), kReferrerID,
+                               true, k61DaysAgo);
+  precache_url_table()->AddURL(GURL("http://old-precache.com"), kReferrerID,
+                               true, k59DaysAgo);
 
   precache_database_->DeleteExpiredPrecacheHistory(kToday);
 
   EXPECT_EQ(BuildURLTableMap(GURL("http://old-precache.com"), k59DaysAgo),
             GetActualURLTableMap());
 }
 
 TEST_F(PrecacheDatabaseTest, SampleInteraction) {
   const GURL kURL1("http://url1.com");
   const int64_t kSize1 = 1;
@@ skipping 82 matching lines @@
 
 TEST_F(PrecacheDatabaseTest, PrecacheFreshnessPrefetch) {
   auto info = CreateHttpResponseInfo(false /* was_cached */,
                                      false /* network_accessed */);
   RecordPrecacheFromNetwork(kURL, kLatency, kFetchTime, kSize);
 
   EXPECT_THAT(histograms_.GetAllSamples("Precache.Freshness.Prefetch"),
               ElementsAre(Bucket(kFreshnessBucket10K, 1)));
 }
 
+TEST_F(PrecacheDatabaseTest, UpdateAndGetReferrerHost) {
+  // Invalid ID should be returned for referrer host that does not exist.
+  EXPECT_EQ(PrecacheReferrerHostEntry::kInvalidId,
+            precache_database_->GetReferrerHost(std::string()).id);
+  EXPECT_EQ(PrecacheReferrerHostEntry::kInvalidId,
+            precache_database_->GetReferrerHost("not_created_host.com").id);
+
+  // Create a new entry.
+  precache_database_->UpdatePrecacheReferrerHost("foo.com", 1, kFetchTime);
+  Flush();
+  auto actual_entry = precache_database_->GetReferrerHost("foo.com");
+  EXPECT_EQ("foo.com", actual_entry.referrer_host);
+  EXPECT_EQ(1, actual_entry.manifest_id);
+  EXPECT_EQ(kFetchTime, actual_entry.time);
+
+  // Update the existing entry.
+  precache_database_->UpdatePrecacheReferrerHost("foo.com", 2, kNewFetchTime);
+  Flush();
+  actual_entry = precache_database_->GetReferrerHost("foo.com");
+  EXPECT_EQ("foo.com", actual_entry.referrer_host);
+  EXPECT_EQ(2, actual_entry.manifest_id);
+  EXPECT_EQ(kNewFetchTime, actual_entry.time);
+}
+
+TEST_F(PrecacheDatabaseTest, GetURLListForReferrerHost) {
+  precache_database_->UpdatePrecacheReferrerHost("foo.com", 1, kFetchTime);
+  precache_database_->UpdatePrecacheReferrerHost("bar.com", 2, kNewFetchTime);
+  precache_database_->UpdatePrecacheReferrerHost("foobar.com", 3,
+                                                 kNewFetchTime);
+  precache_database_->UpdatePrecacheReferrerHost("empty.com", 3, kNewFetchTime);
+
+  struct test_resource_info {
+    std::string url;
+    bool is_network_fetched;
+    bool is_cellular_fetched;
+    bool expected_in_used;
+  };
+
+  const struct {
+    std::string hostname;
+    std::vector<test_resource_info> resource_info;
+  } tests[] = {
+      {
+          "foo.com",
+          {
+              {"http://foo.com/from-cache.js", false, false, true},
+              {"http://some-cdn.com/from-network.js", true, false, false},
+              {"http://foo.com/from-cache-cellular.js", false, true, true},
+              {"http://foo.com/from-network-cellular.js", true, true, false},
+          },
+      },
+      {
+          "bar.com",
+          {
+              {"http://bar.com/a.js", false, false, true},
+              {"http://some-cdn.com/b.js", false, true, true},
+          },
+      },
+      {
+          "foobar.com",
+          {
+              {"http://some-cdn.com/not-used.js", true, true, false},
+          },
+      },
+      {
+          "empty.com", {},
+      },
+  };
+  // Add precached resources.
+  for (const auto& test : tests) {
+    for (const auto& resource : test.resource_info) {
+      precache_database_->RecordURLPrefetch(GURL(resource.url), test.hostname,
+                                            base::Time(), false, kSize);
+    }
+  }
+  // Update some resources as used due to user browsing.
+  for (const auto& test : tests) {
+    for (const auto& resource : test.resource_info) {
+      if (!resource.is_network_fetched && !resource.is_cellular_fetched) {
+        RecordFetchFromCache(GURL(resource.url), kFetchTime, kSize);
+      } else if (!resource.is_network_fetched && resource.is_cellular_fetched) {
+        RecordFetchFromCacheCellular(GURL(resource.url), kFetchTime, kSize);
+      } else if (resource.is_network_fetched && !resource.is_cellular_fetched) {
+        RecordFetchFromNetwork(GURL(resource.url), kLatency, kFetchTime, kSize);
+      } else if (resource.is_network_fetched && resource.is_cellular_fetched) {
+        RecordFetchFromNetworkCellular(GURL(resource.url), kLatency, kFetchTime,
+                                       kSize);
+      }
+    }
+  }
+  Flush();
+  // Verify the used and unused resources.
+  for (const auto& test : tests) {
+    std::vector<GURL> expected_used_urls, expected_unused_urls;
+    for (const auto& resource : test.resource_info) {
+      if (resource.expected_in_used) {
+        expected_used_urls.push_back(GURL(resource.url));
+      } else {
+        expected_unused_urls.push_back(GURL(resource.url));
+      }
+    }
+    std::vector<GURL> actual_used_urls, actual_unused_urls;
+    auto referrer_id = precache_database_->GetReferrerHost(test.hostname).id;
+    EXPECT_NE(PrecacheReferrerHostEntry::kInvalidId, referrer_id);
+    precache_database_->GetURLListForReferrerHost(
+        referrer_id, &actual_used_urls, &actual_unused_urls);
+    EXPECT_THAT(expected_used_urls, ::testing::ContainerEq(actual_used_urls));
+    EXPECT_THAT(expected_unused_urls,
+                ::testing::ContainerEq(actual_unused_urls));
+  }
+  // Subsequent manifest updates should clear the used and unused resources.
+  for (const auto& test : tests) {
+    precache_database_->UpdatePrecacheReferrerHost(test.hostname, 100,
+                                                   kNewFetchTime);
+    Flush();
+    std::vector<GURL> actual_used_urls, actual_unused_urls;
+    auto referrer_id = precache_database_->GetReferrerHost(test.hostname).id;
+    EXPECT_NE(PrecacheReferrerHostEntry::kInvalidId, referrer_id);
+    precache_database_->GetURLListForReferrerHost(
+        referrer_id, &actual_used_urls, &actual_unused_urls);
+    EXPECT_TRUE(actual_used_urls.empty());
+    EXPECT_TRUE(actual_unused_urls.empty());
+  }
+}
+
 }  // namespace
 
 }  // namespace precache