Send the list of used and unused resources for precache

components/precache/core/precache_fetcher_unittest.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_fetcher.h"
 
 #include <stdint.h>
 
 #include <cstring>
 #include <memory>
 #include <set>
 #include <string>
 #include <vector>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
+#include "base/files/file_path.h"
+#include "base/files/scoped_temp_dir.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/run_loop.h"
 #include "base/single_thread_task_runner.h"
 #include "base/strings/stringprintf.h"
 #include "base/test/histogram_tester.h"
 #include "base/threading/thread_task_runner_handle.h"
+#include "components/precache/core/precache_database.h"
 #include "components/precache/core/precache_switches.h"
 #include "components/precache/core/proto/precache.pb.h"
 #include "components/precache/core/proto/unfinished_work.pb.h"
+#include "net/base/escape.h"
 #include "net/base/load_flags.h"
 #include "net/http/http_response_headers.h"
+#include "net/http/http_response_info.h"
 #include "net/http/http_status_code.h"
 #include "net/url_request/test_url_fetcher_factory.h"
 #include "net/url_request/url_request_status.h"
 #include "net/url_request/url_request_test_util.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "url/gurl.h"
 
 namespace precache {
 
@@ skipping 41 matching lines @@
       net::URLRequestStatus::Status status) {
     std::unique_ptr<net::FakeURLFetcher> fetcher(new net::FakeURLFetcher(
         url, delegate, response_data, response_code, status));
 
     total_response_bytes_ += response_data.size();
     requested_urls_.insert(url);
 
     return fetcher;
   }
 
-  const std::multiset<GURL>& requested_urls() const {
-    return requested_urls_;
-  }
+  const std::multiset<GURL>& requested_urls() const { return requested_urls_; }
+
+  void clear_requested_urls() { requested_urls_.clear(); }
 
   int total_response_bytes() const { return total_response_bytes_; }
 
  private:
   // Multiset with one entry for each URL requested.
   std::multiset<GURL> requested_urls_;
   int total_response_bytes_;
 };
 
 class TestPrecacheDelegate : public PrecacheFetcher::PrecacheDelegate {
@@ skipping 117 matching lines @@
   GURL url(kConfigURL);
 
   net::FakeURLFetcher* fetcher = nullptr;
   EXPECT_CALL(factory_, DoCreateURLFetcher(_, url, net::URLFetcher::GET, _))
       .WillOnce(factory_.RespondWith("", &fetcher));
   EXPECT_CALL(*this,
               Callback(Property(&PrecacheFetcher::Fetcher::network_url_fetcher,
                                 NotNull())));
 
   PrecacheFetcher::Fetcher precache_fetcher(
-      request_context_.get(), url, callback_, false /* is_resource_request */,
-      SIZE_MAX);
+      request_context_.get(), url, url.host(), callback_,
+      false /* is_resource_request */, SIZE_MAX);
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   ASSERT_NE(nullptr, fetcher);
   EXPECT_EQ(kNoTracking, fetcher->GetLoadFlags());
 }
 
 TEST_F(PrecacheFetcherFetcherTest, ResourceNotInCache) {
   GURL url(kGoodResourceURL);
 
   net::FakeURLFetcher *fetcher1 = nullptr, *fetcher2 = nullptr;
   EXPECT_CALL(factory_, DoCreateURLFetcher(_, url, net::URLFetcher::GET, _))
       .WillOnce(factory_.RespondWith("", CacheMiss, &fetcher1))
       .WillOnce(factory_.RespondWith("", &fetcher2));
   EXPECT_CALL(*this,
               Callback(Property(&PrecacheFetcher::Fetcher::network_url_fetcher,
                                 NotNull())))
       .Times(1);
 
   PrecacheFetcher::Fetcher precache_fetcher(
-      request_context_.get(), url, callback_, true /* is_resource_request */,
-      SIZE_MAX);
+      request_context_.get(), url, url.host(), callback_,
+      true /* is_resource_request */, SIZE_MAX);
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   ASSERT_NE(nullptr, fetcher1);
   EXPECT_EQ(net::LOAD_ONLY_FROM_CACHE | kNoTracking, fetcher1->GetLoadFlags());
   ASSERT_NE(nullptr, fetcher2);
   EXPECT_EQ(net::LOAD_VALIDATE_CACHE | kNoTracking, fetcher2->GetLoadFlags());
 }
 
 TEST_F(PrecacheFetcherFetcherTest, ResourceHasValidators) {
   GURL url(kGoodResourceURL);
 
   net::FakeURLFetcher *fetcher1 = nullptr, *fetcher2 = nullptr;
   EXPECT_CALL(factory_, DoCreateURLFetcher(_, url, net::URLFetcher::GET, _))
       .WillOnce(factory_.RespondWith("", HasETag, &fetcher1))
       .WillOnce(factory_.RespondWith("", &fetcher2));
   EXPECT_CALL(*this,
               Callback(Property(&PrecacheFetcher::Fetcher::network_url_fetcher,
                                 NotNull())));
 
   PrecacheFetcher::Fetcher precache_fetcher(
-      request_context_.get(), url, callback_, true /* is_resource_request */,
-      SIZE_MAX);
+      request_context_.get(), url, url.host(), callback_,
+      true /* is_resource_request */, SIZE_MAX);
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   ASSERT_NE(nullptr, fetcher1);
   EXPECT_EQ(net::LOAD_ONLY_FROM_CACHE | kNoTracking, fetcher1->GetLoadFlags());
   ASSERT_NE(nullptr, fetcher2);
   EXPECT_EQ(net::LOAD_VALIDATE_CACHE | kNoTracking, fetcher2->GetLoadFlags());
 }
 
 TEST_F(PrecacheFetcherFetcherTest, ResourceHasNoValidators) {
   GURL url(kGoodResourceURL);
 
   net::FakeURLFetcher* fetcher;
   EXPECT_CALL(factory_, DoCreateURLFetcher(_, url, net::URLFetcher::GET, _))
       .WillOnce(factory_.RespondWith("", &fetcher));
   EXPECT_CALL(*this,
               Callback(Property(&PrecacheFetcher::Fetcher::network_url_fetcher,
                                 nullptr)));  // It never reached the network.
 
   PrecacheFetcher::Fetcher precache_fetcher(
-      request_context_.get(), url, callback_, true /* is_resource_request */,
-      SIZE_MAX);
+      request_context_.get(), url, url.host(), callback_,
+      true /* is_resource_request */, SIZE_MAX);
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   EXPECT_EQ(net::LOAD_ONLY_FROM_CACHE | kNoTracking, fetcher->GetLoadFlags());
 }
 
 TEST_F(PrecacheFetcherFetcherTest, ResourceTooBig) {
   GURL url(kGoodResourceURL);
 
   EXPECT_CALL(factory_, DoCreateURLFetcher(_, url, net::URLFetcher::GET, _))
       // Cache request will fail, so that a network request is made. Only
       // network requests are byte-capped.
       .WillOnce(factory_.RespondWith("", CacheMiss, nullptr))
       .WillOnce(factory_.RespondWith(std::string(100, '.'), nullptr));
 
   // The callback should be called even though the download was cancelled, so
   // that the next download can start. The network_url_fetcher within should be
   // null, to signify that either the network was never reached (which will be
   // flagged as an error due to the expectation above) or it was requested but
   // cancelled (which is the desired behavior).
   EXPECT_CALL(*this,
               Callback(Property(&PrecacheFetcher::Fetcher::network_url_fetcher,
                                 nullptr)));
 
   PrecacheFetcher::Fetcher precache_fetcher(
-      request_context_.get(), url, callback_, true /* is_resource_request */,
-      99 /* max_bytes */);
+      request_context_.get(), url, url.host(), callback_,
+      true /* is_resource_request */, 99 /* max_bytes */);
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 }
 
 class PrecacheFetcherTest : public testing::Test {
  public:
   PrecacheFetcherTest()
-      : request_context_(new net::TestURLRequestContextGetter(
+      : task_runner_(base::ThreadTaskRunnerHandle::Get()),
+        request_context_(new net::TestURLRequestContextGetter(
             base::ThreadTaskRunnerHandle::Get())),
         factory_(NULL,
                  base::Bind(&TestURLFetcherCallback::CreateURLFetcher,
                             base::Unretained(&url_callback_))),
         expected_total_response_bytes_(0),
         parallel_fetches_beyond_capacity_(false) {}
 
+  void UpdatePrecacheReferrerHost(const std::string& hostname,
+                                  int64_t manifest_id) {
+    precache_database_.UpdatePrecacheReferrerHost(hostname, manifest_id,
+                                                  base::Time());
+  }
+
+  void RecordURLPrefetch(const GURL& url, const std::string& referrer_host) {
+    precache_database_.RecordURLPrefetch(url, referrer_host, base::Time::Now(),
+                                         false /* was_cached */,
+                                         1000 /* size */);
+  }
+
+  void RecordURLNonPrefetch(const GURL& url) {
+    net::HttpResponseInfo info;
+    info.was_cached = true;
+    info.headers = new net::HttpResponseHeaders(std::string());
+    precache_database_.RecordURLNonPrefetch(
+        url, base::TimeDelta(), base::Time::Now(), info, 1000 /* size */,
+        0 /* host_rank */, false /* is_connection_cellular */);
+  }
+
  protected:
+  void SetUp() override {
+    ASSERT_TRUE(scoped_temp_dir_.CreateUniqueTempDir());
+    base::FilePath db_path = scoped_temp_dir_.path().Append(
+        base::FilePath(FILE_PATH_LITERAL("precache_database")));
+    precache_database_.Init(db_path);
+  }
   void SetDefaultFlags() {
     base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
         switches::kPrecacheConfigSettingsURL, kConfigURL);
     base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
         switches::kPrecacheManifestURLPrefix, kManifestURLPrefix);
   }
 
   // Posts a task to check if more parallel fetches of precache manifest and
   // resource URLs were attempted beyond the fetcher pool maximum defined
   // capacity. The task will be posted repeatedly until such condition is met.
   void CheckUntilParallelFetchesBeyondCapacity(
       const PrecacheFetcher* precache_fetcher) {
     if (!precache_fetcher->pool_.IsAvailable() &&
-        !precache_fetcher->resource_urls_to_fetch_.empty() &&
-        !precache_fetcher->manifest_urls_to_fetch_.empty()) {
+        precache_fetcher->top_hosts_to_fetch_ &&
+        !precache_fetcher->top_hosts_to_fetch_->empty() &&
+        !precache_fetcher->resources_to_fetch_.empty()) {
       parallel_fetches_beyond_capacity_ = true;
       return;
     }
 
     // Check again after allowing the message loop to process some messages.
-    loop_.task_runner()->PostTask(
+    loop_.PostTask(
         FROM_HERE,
         base::Bind(
             &PrecacheFetcherTest::CheckUntilParallelFetchesBeyondCapacity,
             base::Unretained(this), precache_fetcher));
   }
 
+  const scoped_refptr<base::SingleThreadTaskRunner>& task_runner() const {
+    return task_runner_;
+  }
+
+  // Must be declared first so that it is destroyed last.
   base::MessageLoopForUI loop_;
+  scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
   scoped_refptr<net::TestURLRequestContextGetter> request_context_;
   TestURLFetcherCallback url_callback_;
   net::FakeURLFetcherFactory factory_;
   TestPrecacheDelegate precache_delegate_;
+  base::ScopedTempDir scoped_temp_dir_;
+  PrecacheDatabase precache_database_;
   int expected_total_response_bytes_;
 
   // True if more parallel fetches were attempted beyond the fetcher pool
   // maximum capacity.
   bool parallel_fetches_beyond_capacity_;
 };
 
 TEST_F(PrecacheFetcherTest, FullPrecache) {
   SetDefaultFlags();
 
@@ skipping 31 matching lines @@
                            net::URLRequestStatus::FAILED);
   factory_.SetFakeResponse(GURL(kGoodResourceURL), "good", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kForcedStartingURLManifestURL),
                            PrecacheManifest().SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
 
     // Destroy the PrecacheFetcher after it has finished, to record metrics.
   }
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kManifestFetchFailureURL));
   expected_requested_urls.insert(GURL(kBadManifestURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   expected_requested_urls.insert(GURL(kResourceFetchFailureURL));
@@ skipping 42 matching lines @@
   factory_.SetFakeResponse(GURL(kGoodResourceURL), "good", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodResourceURLB), "good URL B", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodResourceURLD), "good URL D", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
 
     // Destroy the PrecacheFetcher after it has finished, to record metrics.
   }
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   expected_requested_urls.insert(GURL(kGoodResourceURL));
   expected_requested_urls.insert(GURL(kGoodResourceURLB));
   expected_requested_urls.insert(GURL(kGoodResourceURLD));
@@ skipping 46 matching lines @@
   factory_.SetFakeResponse(GURL(kGoodResourceURLB), "good URL B", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodResourceURLC), "good URL C", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodResourceURLD), "good URL D", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
 
     // Destroy the PrecacheFetcher after it has finished, to record metrics.
   }
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   expected_requested_urls.insert(GURL(kGoodResourceURL));
   expected_requested_urls.insert(GURL(kGoodResourceURLA));
   expected_requested_urls.insert(GURL(kGoodResourceURLB));
@@ skipping 11 matching lines @@
 }
 
 TEST_F(PrecacheFetcherTest, PrecachePauseResume) {
   SetDefaultFlags();
 
   PrecacheConfigurationSettings config;
   config.set_top_sites_count(3);
 
   std::unique_ptr<PrecacheUnfinishedWork> initial_work(
       new PrecacheUnfinishedWork());
-  initial_work->add_manifest()->set_url(
-      "http://manifest-url-prefix.com/manifest1.com");
-  initial_work->add_manifest()->set_url(
-      "http://manifest-url-prefix.com/manifest2.com");
-  initial_work->add_resource()->set_url(kGoodResourceURL);
+  initial_work->add_top_host()->set_hostname("manifest1.com");
+  initial_work->add_top_host()->set_hostname("manifest2.com");
   initial_work->set_start_time(
       (base::Time::Now() - base::TimeDelta::FromHours(1)).ToInternalValue());
+  // initial_work->add_resource()->set_url(kGoodResourceURL);

[sclittle, 2016/08/15 20:13:10]

nit: remove dead code

[Raj, 2016/08/16 17:54:37]

Done.

 
   PrecacheFetcher first_fetcher(request_context_.get(), GURL(), std::string(),
                                 std::move(initial_work), kExperimentID,
+                                precache_database_.GetWeakPtr(), task_runner(),
                                 &precache_delegate_);
-
-  loop_.RunUntilIdle();
+  factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(),
+                           net::HTTP_OK, net::URLRequestStatus::SUCCESS);
+  first_fetcher.Start();
+  // base::RunLoop().RunUntilIdle();

[sclittle, 2016/08/15 20:13:10]

nit: remove dead code, here and elsewhere

[Raj, 2016/08/16 17:54:37]

Done.

   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work =
       first_fetcher.CancelPrecaching();
 
   std::multiset<GURL> expected_requested_urls;
+  expected_requested_urls.insert(GURL(kConfigURL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kBadManifestURL), "bad protobuf", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL("http://manifest-url-prefix.com/manifest1.com"),
                            "bad protobuf", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL("http://manifest-url-prefix.com/manifest2.com"),
                            "bad protobuf", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodResourceURL), "good", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
-  // Starting hosts should not be fetched.
-  unfinished_work->add_top_host()->set_hostname("bad-manifest.com");
+
+  // unfinished_work->add_top_host()->set_hostname("bad-manifest.com");
+  url_callback_.clear_requested_urls();
   PrecacheFetcher second_fetcher(request_context_.get(), GURL(), std::string(),
                                  std::move(unfinished_work), kExperimentID,
+                                 precache_database_.GetWeakPtr(), task_runner(),
                                  &precache_delegate_);
   second_fetcher.Start();
-  loop_.RunUntilIdle();
-  expected_requested_urls.insert(GURL(kConfigURL));
+  base::RunLoop().RunUntilIdle();
   expected_requested_urls.insert(
       GURL("http://manifest-url-prefix.com/manifest1.com"));
   expected_requested_urls.insert(
       GURL("http://manifest-url-prefix.com/manifest2.com"));
-  expected_requested_urls.insert(GURL(kGoodResourceURL));
+  // expected_requested_urls.insert(GURL(kGoodResourceURL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 TEST_F(PrecacheFetcherTest, ResumeWithConfigOnly) {
   SetDefaultFlags();
   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work(
       new PrecacheUnfinishedWork());
   unfinished_work->mutable_config_settings()->add_forced_site(
       "good-manifest.com");
   unfinished_work->set_start_time(base::Time::Now().ToInternalValue());
   PrecacheManifest good_manifest;
   good_manifest.add_resource()->set_url(kGoodResourceURL);
 
   factory_.SetFakeResponse(GURL(kGoodManifestURL),
                            good_manifest.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodResourceURL), "good", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
   }
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   expected_requested_urls.insert(GURL(kGoodResourceURL));
 
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 
 }
@@ skipping 14 matching lines @@
   factory_.SetFakeResponse(GURL(kCustomConfigURL), config.SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kCustomGoodManifestURL),
                            good_manifest.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodResourceURL), "good", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   PrecacheFetcher precache_fetcher(
       request_context_.get(), GURL(kCustomConfigURL), kCustomManifestURLPrefix,
-      std::move(unfinished_work), kExperimentID, &precache_delegate_);
+      std::move(unfinished_work), kExperimentID,
+      precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
   precache_fetcher.Start();
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kCustomConfigURL));
   expected_requested_urls.insert(GURL(kCustomGoodManifestURL));
   expected_requested_urls.insert(GURL(kGoodResourceURL));
 
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 TEST_F(PrecacheFetcherTest, ConfigFetchFailure) {
   SetDefaultFlags();
 
   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work(
       new PrecacheUnfinishedWork());
   unfinished_work->add_top_host()->set_hostname("good-manifest.com");
 
   factory_.SetFakeResponse(GURL(kConfigURL), "",
                            net::HTTP_INTERNAL_SERVER_ERROR,
                            net::URLRequestStatus::FAILED);
   factory_.SetFakeResponse(GURL(kGoodManifestURL), "", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
-  PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                   std::string(), std::move(unfinished_work),
-                                   kExperimentID, &precache_delegate_);
+  PrecacheFetcher precache_fetcher(
+      request_context_.get(), GURL(), std::string(), std::move(unfinished_work),
+      kExperimentID, precache_database_.GetWeakPtr(), task_runner(),
+      &precache_delegate_);
   precache_fetcher.Start();
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 TEST_F(PrecacheFetcherTest, BadConfig) {
   SetDefaultFlags();
 
   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work(
       new PrecacheUnfinishedWork());
   unfinished_work->add_top_host()->set_hostname("good-manifest.com");
 
   factory_.SetFakeResponse(GURL(kConfigURL), "bad protobuf", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodManifestURL), "", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
-  PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                   std::string(), std::move(unfinished_work),
-                                   kExperimentID, &precache_delegate_);
+  PrecacheFetcher precache_fetcher(
+      request_context_.get(), GURL(), std::string(), std::move(unfinished_work),
+      kExperimentID, precache_database_.GetWeakPtr(), task_runner(),
+      &precache_delegate_);
   precache_fetcher.Start();
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 TEST_F(PrecacheFetcherTest, Cancel) {
   SetDefaultFlags();
 
   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work(
       new PrecacheUnfinishedWork());
   unfinished_work->add_top_host()->set_hostname("starting-url.com");
 
   PrecacheConfigurationSettings config;
   config.set_top_sites_count(1);
 
   factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
     // Destroy the PrecacheFetcher, to cancel precaching. No metrics
     // should be recorded because this should not cause OnDone to be
     // called on the precache delegate.
   }
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_FALSE(precache_delegate_.was_on_done_called());
 
   histogram.ExpectTotalCount("Precache.Fetch.TimeToComplete", 0);
 }
 
 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
 
 // If the default precache configuration settings URL is defined, then test that
 // it works with the PrecacheFetcher.
 TEST_F(PrecacheFetcherTest, PrecacheUsingDefaultConfigSettingsURL) {
   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work(
       new PrecacheUnfinishedWork());
   unfinished_work->add_top_host()->set_hostname("starting-url.com");
 
   PrecacheConfigurationSettings config;
   config.set_top_sites_count(0);
 
   factory_.SetFakeResponse(GURL(PRECACHE_CONFIG_SETTINGS_URL),
                            config.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
-  PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                   std::string(), std::move(unfinished_work),
-                                   kExperimentID, &precache_delegate_);
+  PrecacheFetcher precache_fetcher(
+      request_context_.get(), GURL(), std::string(), std::move(unfinished_work),
+      kExperimentID, precache_database_.GetWeakPtr(), task_runner(),
+      &precache_delegate_);
   precache_fetcher.Start();
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(PRECACHE_CONFIG_SETTINGS_URL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 #endif  // PRECACHE_CONFIG_SETTINGS_URL
 
@@ skipping 12 matching lines @@
   PrecacheConfigurationSettings config;
   config.set_top_sites_count(1);
 
   GURL manifest_url(PRECACHE_MANIFEST_URL_PREFIX "starting-url.com");
 
   factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(manifest_url, PrecacheManifest().SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
 
-  PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                   std::string(), std::move(unfinished_work),
-                                   kExperimentID, &precache_delegate_);
+  PrecacheFetcher precache_fetcher(
+      request_context_.get(), GURL(), std::string(), std::move(unfinished_work),
+      kExperimentID, precache_database_.GetWeakPtr(), task_runner(),
+      &precache_delegate_);
   precache_fetcher.Start();
 
-  loop_.RunUntilIdle();
+  base::RunLoop().RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(manifest_url);
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 #endif  // PRECACHE_MANIFEST_URL_PREFIX
@@ skipping 20 matching lines @@
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodManifestURL),
                            good_manifest.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL("http://good-manifest.com/retrieved"), "good",
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
 
     // Destroy the PrecacheFetcher after it has finished, to record metrics.
   }
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   expected_requested_urls.insert(GURL("http://good-manifest.com/retrieved"));
   expected_requested_urls.insert(GURL("http://good-manifest.com/retrieved"));
   expected_requested_urls.insert(GURL("http://good-manifest.com/retrieved"));
@@ skipping 51 matching lines @@
                              net::HTTP_OK, net::URLRequestStatus::SUCCESS);
   }
 
   factory_.SetFakeResponse(GURL(kGoodManifestURL),
                            good_manifest.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
   }
 
   // We don't know which will make it and which won't due to the parallelism in
   // the pool of Fetchers.
   EXPECT_GT(kNumResources, url_callback_.requested_urls().size());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 
   // good-manifest.com will not have been completed.
   EXPECT_THAT(histogram.GetAllSamples("Precache.Fetch.PercentCompleted"),
@@ skipping 36 matching lines @@
     }
     factory_.SetFakeResponse(GURL(kManifestURLPrefix + top_host_url),
                              top_host_manifest[i].SerializeAsString(),
                              net::HTTP_OK, net::URLRequestStatus::SUCCESS);
     expected_requested_urls.insert(GURL(kManifestURLPrefix + top_host_url));
   }
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
     EXPECT_GT(kNumResources, precache_fetcher.pool_.max_size());
     CheckUntilParallelFetchesBeyondCapacity(&precache_fetcher);
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
 
     // Destroy the PrecacheFetcher after it has finished, to record metrics.
   }
 
   EXPECT_TRUE(parallel_fetches_beyond_capacity_);
 
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 
@@ skipping 12 matching lines @@
       new PrecacheUnfinishedWork());
   unfinished_work->add_top_host()->set_hostname("manifest.com");
   unfinished_work->set_start_time(base::Time::UnixEpoch().ToInternalValue());
 
   factory_.SetFakeResponse(GURL(kConfigURL), "", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
   }
 
   // The config is fetched, but not the invalid manifest URL.
   EXPECT_EQ(1UL, url_callback_.requested_urls().size());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 
   // manifest.com will have been failed to complete, in this case.
   EXPECT_THAT(histogram.GetAllSamples("Precache.Fetch.PercentCompleted"),
               ElementsAre(base::Bucket(0, 1)));
@@ skipping 13 matching lines @@
   PrecacheManifest bad_manifest;
   bad_manifest.add_resource()->set_url("http://");
 
   factory_.SetFakeResponse(GURL(kBadManifestURL),
                            bad_manifest.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
-    PrecacheFetcher precache_fetcher(request_context_.get(), GURL(),
-                                     std::string(), std::move(unfinished_work),
-                                     kExperimentID, &precache_delegate_);
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
     precache_fetcher.Start();
 
-    loop_.RunUntilIdle();
+    base::RunLoop().RunUntilIdle();
   }
 
   // The config and manifest are fetched, but not the invalid resource URL.
   EXPECT_EQ(2UL, url_callback_.requested_urls().size());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 
   // bad-manifest.com will have been completed.
   EXPECT_THAT(histogram.GetAllSamples("Precache.Fetch.PercentCompleted"),
               ElementsAre(base::Bucket(100, 1)));
   histogram.ExpectTotalCount("Precache.Fetch.TimeToComplete", 1);
 }
 
+TEST_F(PrecacheFetcherTest, GetResourceURLBase64Hash) {

[sclittle, 2016/08/15 20:13:10]

nit: this should just be a TEST, not a TEST_F if it doesn't need to use any
stuff from the PrecacheFetcherTest class.

You could change it to TEST(PrecacheFetcherStandaloneTest,
GetResourceURLBase64Hash)

[Raj, 2016/08/16 17:54:37]

Done.

+  // Expected base64 hash for some selected URLs.
+  EXPECT_EQ("dVSI/sC1cGk=", PrecacheFetcher::GetResourceURLBase64HashForTesting(
+                                {GURL("http://used-resource-1/a.js")}));
+  EXPECT_EQ("B/Jc6JvusZQ=", PrecacheFetcher::GetResourceURLBase64HashForTesting(
+                                {GURL("http://used-resource-1/b.js")}));
+  EXPECT_EQ("CmvACGJ4k08=", PrecacheFetcher::GetResourceURLBase64HashForTesting(
+                                {GURL("http://used-resource-1/c.js")}));
+
+  EXPECT_EQ("dVSI/sC1cGkH8lzom+6xlA==",
+            PrecacheFetcher::GetResourceURLBase64HashForTesting(
+                {GURL("http://used-resource-1/a.js"),
+                 GURL("http://used-resource-1/b.js")}));
+}
+
+TEST_F(PrecacheFetcherTest, SendUsedUnusedResourceHash) {
+  SetDefaultFlags();
+
+  std::unique_ptr<PrecacheUnfinishedWork> unfinished_work(
+      new PrecacheUnfinishedWork());
+  unfinished_work->set_start_time(base::Time::UnixEpoch().ToInternalValue());
+  unfinished_work->add_top_host()->set_hostname("top-host-1.com");
+  unfinished_work->add_top_host()->set_hostname("top-host-2.com");
+  unfinished_work->add_top_host()->set_hostname("top-host-3.com");
+
+  UpdatePrecacheReferrerHost("top-host-1.com", 1);
+  UpdatePrecacheReferrerHost("top-host-2.com", 2);
+  UpdatePrecacheReferrerHost("top-host-3.com", 3);
+
+  // Mark some resources as precached.
+  RecordURLPrefetch(GURL("http://used-resource-1/a.js"), "top-host-1.com");
+  RecordURLPrefetch(GURL("http://used-resource-1/b.js"), "top-host-1.com");
+  RecordURLPrefetch(GURL("http://unused-resource-1/c.js"), "top-host-1.com");
+  RecordURLPrefetch(GURL("http://unused-resource-2/a.js"), "top-host-2.com");
+  RecordURLPrefetch(GURL("http://unused-resource-2/b.js"), "top-host-2.com");
+  base::RunLoop().RunUntilIdle();
+
+  // Mark some resources as used during user browsing.
+  RecordURLNonPrefetch(GURL("http://used-resource-1/a.js"));
+  RecordURLNonPrefetch(GURL("http://used-resource-1/b.js"));
+  base::RunLoop().RunUntilIdle();
+
+  factory_.SetFakeResponse(GURL(kConfigURL), std::string(), net::HTTP_OK,
+                           net::URLRequestStatus::SUCCESS);
+  factory_.SetFakeResponse(
+      GURL(std::string(kManifestURLPrefix) +
+           "top-host-1.com?manifest=1&used_resources=" +
+           net::EscapeQueryParamValue(
+               PrecacheFetcher::GetResourceURLBase64HashForTesting(
+                   {GURL("http://used-resource-1/a.js"),
+                    GURL("http://used-resource-1/b.js")}),
+               true) +
+           "&unused_resources=" +
+           net::EscapeQueryParamValue(
+               PrecacheFetcher::GetResourceURLBase64HashForTesting(
+                   {GURL("http://unused-resource-1/c.js")}),
+               true)),
+      std::string(), net::HTTP_OK, net::URLRequestStatus::SUCCESS);
+  factory_.SetFakeResponse(
+      GURL(std::string(kManifestURLPrefix) +
+           "top-host-2.com?manifest=2&used_resources=&unused_resources=" +
+           net::EscapeQueryParamValue(
+               PrecacheFetcher::GetResourceURLBase64HashForTesting(
+                   {GURL("http://unused-resource-2/a.js"),
+                    GURL("http://unused-resource-2/b.js")}),
+               true)),
+      std::string(), net::HTTP_OK, net::URLRequestStatus::SUCCESS);
+  factory_.SetFakeResponse(
+      GURL(std::string(kManifestURLPrefix) +
+           "top-host-3.com?manifest=3&used_resources=&unused_resources="),
+      std::string(), net::HTTP_OK, net::URLRequestStatus::SUCCESS);
+
+  {
+    PrecacheFetcher precache_fetcher(
+        request_context_.get(), GURL(), std::string(),
+        std::move(unfinished_work), kExperimentID,
+        precache_database_.GetWeakPtr(), task_runner(), &precache_delegate_);
+    precache_fetcher.Start();
+
+    base::RunLoop().RunUntilIdle();
+  }
+}
+
 }  // namespace precache