Report downloaded resources at most once

components/precache/core/precache_database_unittest.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_database.h"
 
 #include <stdint.h>
 
 #include <map>
 #include <memory>
 
+#include "base/bind.h"
+#include "base/bind_helpers.h"
 #include "base/containers/hash_tables.h"
 #include "base/files/file_path.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/message_loop/message_loop.h"
 #include "base/metrics/histogram_base.h"
 #include "base/test/histogram_tester.h"
 #include "base/time/time.h"
 #include "components/history/core/browser/history_constants.h"
 #include "net/http/http_response_headers.h"
 #include "net/http/http_response_info.h"
@@ skipping 234 matching lines @@
 
   EXPECT_EQ(BuildURLTableMap(kURL, kFetchTime), GetActualURLTableMap());
 
   ExpectNewSample("Precache.DownloadedPrecacheMotivated", kSize);
   ExpectNewSample("Precache.Latency.Prefetch", kLatency.InMilliseconds());
   ExpectNewSample("Precache.Freshness.Prefetch", kFreshnessBucket10K);
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, PrecacheFromCacheWithURLTableEntry) {
-  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime);
+  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime, false);
   RecordPrecacheFromCache(kURL, kFetchTime, kSize);
 
   // The URL table entry should have been updated to have |kFetchTime| as the
   // timestamp.
   EXPECT_EQ(BuildURLTableMap(kURL, kFetchTime), GetActualURLTableMap());
 
   ExpectNewSample("Precache.Latency.Prefetch", 0);
   ExpectNewSample("Precache.Freshness.Prefetch", kFreshnessBucket10K);
   ExpectNoOtherSamples();
 }
@@ skipping 42 matching lines @@
   ExpectNewSample("Precache.DownloadedNonPrecache", kSize);
   ExpectNewSample("Precache.DownloadedNonPrecache.Cellular", kSize);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch", kFromNetwork);
   ExpectNewSample("Precache.Latency.NonPrefetch", kLatency.InMilliseconds());
   ExpectNewSample("Precache.Latency.NonPrefetch.NonTopHosts",
                   kLatency.InMilliseconds());
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, FetchOverNetworkWithURLTableEntry) {
-  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime);
+  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime, false);
   RecordFetchFromNetwork(kURL, kLatency, kFetchTime, kSize);
 
   // The URL table entry should have been deleted.
   EXPECT_TRUE(GetActualURLTableMap().empty());
 
   ExpectNewSample("Precache.DownloadedNonPrecache", kSize);
   ExpectNewSample("Precache.Latency.NonPrefetch", kLatency.InMilliseconds());
   ExpectNewSample("Precache.Latency.NonPrefetch.NonTopHosts",
                   kLatency.InMilliseconds());
   ExpectNewSample("Precache.CacheStatus.NonPrefetch", kFromNetwork);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch.FromPrecache",
                   kFromNetwork);
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, FetchFromCacheWithURLTableEntry_NonCellular) {
-  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime);
+  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime, false);
   RecordFetchFromCache(kURL, kFetchTime, kSize);
 
   // The URL table entry should have been deleted.
   EXPECT_TRUE(GetActualURLTableMap().empty());
 
   ExpectNewSample("Precache.Latency.NonPrefetch", 0);
   ExpectNewSample("Precache.Latency.NonPrefetch.NonTopHosts", 0);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch",
                   HttpResponseInfo::CacheEntryStatus::ENTRY_USED);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch.FromPrecache",
                   HttpResponseInfo::CacheEntryStatus::ENTRY_USED);
   ExpectNewSample("Precache.Saved", kSize);
   ExpectNewSample("Precache.Saved.Freshness", kFreshnessBucket10K);
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, FetchFromCacheWithURLTableEntry_Cellular) {
-  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime);
+  precache_url_table()->AddURL(kURL, kReferrerID, true, kOldFetchTime, false);
   RecordFetchFromCacheCellular(kURL, kFetchTime, kSize);
 
   // The URL table entry should have been deleted.
   EXPECT_TRUE(GetActualURLTableMap().empty());
 
   ExpectNewSample("Precache.Latency.NonPrefetch", 0);
   ExpectNewSample("Precache.Latency.NonPrefetch.NonTopHosts", 0);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch",
                   HttpResponseInfo::CacheEntryStatus::ENTRY_USED);
   ExpectNewSample("Precache.CacheStatus.NonPrefetch.FromPrecache",
@@ skipping 15 matching lines @@
                   HttpResponseInfo::CacheEntryStatus::ENTRY_USED);
   ExpectNoOtherSamples();
 }
 
 TEST_F(PrecacheDatabaseTest, DeleteExpiredPrecacheHistory) {
   const base::Time kToday = base::Time() + base::TimeDelta::FromDays(1000);
   const base::Time k59DaysAgo = kToday - base::TimeDelta::FromDays(59);
   const base::Time k61DaysAgo = kToday - base::TimeDelta::FromDays(61);
 
   precache_url_table()->AddURL(GURL("http://expired-precache.com"), kReferrerID,
-                               true, k61DaysAgo);
+                               true, k61DaysAgo, false);
   precache_url_table()->AddURL(GURL("http://old-precache.com"), kReferrerID,
-                               true, k59DaysAgo);
+                               true, k59DaysAgo, false);
 
   precache_database_->DeleteExpiredPrecacheHistory(kToday);
 
   EXPECT_EQ(BuildURLTableMap(GURL("http://old-precache.com"), k59DaysAgo),
             GetActualURLTableMap());
 }
 
 TEST_F(PrecacheDatabaseTest, SampleInteraction) {
   const GURL kURL1("http://url1.com");
   const int64_t kSize1 = 1;
@@ skipping 113 matching lines @@
   EXPECT_EQ(kNewFetchTime, actual_entry.time);
 }
 
 TEST_F(PrecacheDatabaseTest, GetURLListForReferrerHost) {
   precache_database_->UpdatePrecacheReferrerHost("foo.com", 1, kFetchTime);
   precache_database_->UpdatePrecacheReferrerHost("bar.com", 2, kNewFetchTime);
   precache_database_->UpdatePrecacheReferrerHost("foobar.com", 3,
                                                  kNewFetchTime);
   precache_database_->UpdatePrecacheReferrerHost("empty.com", 3, kNewFetchTime);
 
+  // Add two resources that shouldn't appear in downloaded.
+  Flush();  // We need to write the referrer_host_id.
+  const std::string already_reported_and_not_refetch =
+      "http://foo.com/already-reported-and-not-refetch.js";
+
+  precache_database_->RecordURLPrefetch(GURL(already_reported_and_not_refetch),
+                                        "foo.com", kPrecacheTime, false, kSize);
+  const std::string already_reported_and_in_cache =
+      "http://foo.com/already-reported-and-in-cache.js";
+  precache_database_->RecordURLPrefetch(GURL(already_reported_and_in_cache),
+                                        "foo.com", kPrecacheTime, false, kSize);
+  {
+    // Let's mark existing resources as is_download_reported = 1 by calling
+    // GetURLListForReferrerHost.
+    std::vector<GURL> unused_a, unused_b;
+    auto id = precache_database_->GetReferrerHost("foo.com").id;
+    ASSERT_NE(PrecacheReferrerHostEntry::kInvalidId, id);
+    precache_database_->GetURLListForReferrerHost(id, &unused_a, &unused_b);
+  }
+
   struct test_resource_info {
     std::string url;
     bool is_user_browsed;
     bool is_network_fetched;
     bool is_cellular_fetched;
     bool expected_in_used;
   };
 
   const struct {
     std::string hostname;
@@ skipping 29 matching lines @@
           "empty.com", std::vector<test_resource_info>{},
       },
   };
   // Add precached resources.
   for (const auto& test : tests) {
     for (const auto& resource : test.resource_info) {
       precache_database_->RecordURLPrefetch(GURL(resource.url), test.hostname,
                                             kPrecacheTime, false, kSize);
     }
   }
+  precache_database_->RecordURLPrefetch(GURL(already_reported_and_in_cache),
+                                        "foo.com", kPrecacheTime, true, kSize);
   // Update some resources as used due to user browsing.
   for (const auto& test : tests) {
     for (const auto& resource : test.resource_info) {
       if (!resource.is_user_browsed)
         continue;
       if (!resource.is_network_fetched && !resource.is_cellular_fetched) {
         RecordFetchFromCache(GURL(resource.url), kFetchTime, kSize);
       } else if (!resource.is_network_fetched && resource.is_cellular_fetched) {
         RecordFetchFromCacheCellular(GURL(resource.url), kFetchTime, kSize);
       } else if (resource.is_network_fetched && !resource.is_cellular_fetched) {
         RecordFetchFromNetwork(GURL(resource.url), kLatency, kFetchTime, kSize);
       } else if (resource.is_network_fetched && resource.is_cellular_fetched) {
         RecordFetchFromNetworkCellular(GURL(resource.url), kLatency, kFetchTime,
                                        kSize);
       }
     }
   }
   Flush();
-  // Verify the used and unused resources.
+  // Verify the used and downloaded resources.
   for (const auto& test : tests) {
-    std::vector<GURL> expected_used_urls, expected_unused_urls;
+    std::vector<GURL> expected_used_urls, expected_downloaded_urls;
     for (const auto& resource : test.resource_info) {
       if (resource.expected_in_used) {
         expected_used_urls.push_back(GURL(resource.url));
-      } else {
-        expected_unused_urls.push_back(GURL(resource.url));
       }
+      expected_downloaded_urls.push_back(GURL(resource.url));
     }
-    std::vector<GURL> actual_used_urls, actual_unused_urls;
+    std::vector<GURL> actual_used_urls, actual_downloaded_urls;
     auto referrer_id = precache_database_->GetReferrerHost(test.hostname).id;
     EXPECT_NE(PrecacheReferrerHostEntry::kInvalidId, referrer_id);
     precache_database_->GetURLListForReferrerHost(
-        referrer_id, &actual_used_urls, &actual_unused_urls);
-    EXPECT_THAT(expected_used_urls, ::testing::ContainerEq(actual_used_urls));
-    EXPECT_THAT(expected_unused_urls,
-                ::testing::ContainerEq(actual_unused_urls));
+        referrer_id, &actual_used_urls, &actual_downloaded_urls);
+    EXPECT_THAT(actual_used_urls, ::testing::ContainerEq(expected_used_urls));
+    EXPECT_THAT(actual_downloaded_urls,
+                ::testing::ContainerEq(expected_downloaded_urls))
+        << "Host: " << test.hostname;
   }
-  // Subsequent manifest updates should clear the used and unused resources.
+  // Subsequent manifest updates should clear the used and downloaded resources.
   for (const auto& test : tests) {
     precache_database_->UpdatePrecacheReferrerHost(test.hostname, 100,
                                                    kNewFetchTime);
     Flush();
-    std::vector<GURL> actual_used_urls, actual_unused_urls;
+    std::vector<GURL> actual_used_urls, actual_downloaded_urls;
     auto referrer_id = precache_database_->GetReferrerHost(test.hostname).id;
     EXPECT_NE(PrecacheReferrerHostEntry::kInvalidId, referrer_id);
     precache_database_->GetURLListForReferrerHost(
-        referrer_id, &actual_used_urls, &actual_unused_urls);
+        referrer_id, &actual_used_urls, &actual_downloaded_urls);
     EXPECT_TRUE(actual_used_urls.empty());

[twifkak, 2016/12/20 22:48:09]

change this to EXPECT_THAT(actual_used_urls, IsEmpty()) (for a better error
message) and add a check for actual_downloaded_urls too

[jamartin, 2016/12/20 23:14:07]

Done.

   }
   // Resources that were precached previously and not seen in user browsing
   // should be still marked as precached.
   std::map<GURL, base::Time> expected_url_table_map;
   for (const auto& test : tests) {
     for (const auto& resource : test.resource_info) {
       if (!resource.is_user_browsed)
         expected_url_table_map[GURL(resource.url)] = kPrecacheTime;
     }
   }
+  expected_url_table_map[GURL(already_reported_and_not_refetch)] =
+      kPrecacheTime;
+  expected_url_table_map[GURL(already_reported_and_in_cache)] = kPrecacheTime;
   EXPECT_EQ(expected_url_table_map, GetActualURLTableMap());

[twifkak, 2016/12/20 22:48:09]

Add a resource for which is_download_reported gets reset to 0.

Also, this test is getting pretty long. Maybe you want to split out the new URLs
in a separate test looking only at downloaded_urls?

[jamartin, 2016/12/20 23:14:07]

Done.

 }
 
 }  // namespace
 
 }  // namespace precache