Add pause/resume functionality to precache

components/precache/core/precache_fetcher_unittest.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_fetcher.h"
 
 #include <stdint.h>
 
 #include <cstring>
 #include <memory>
 #include <set>
 #include <string>
 #include <vector>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
+#include "base/files/file_path.h"
+#include "base/files/scoped_temp_dir.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/run_loop.h"
 #include "base/test/histogram_tester.h"
 #include "base/thread_task_runner_handle.h"
+#include "components/precache/core/precache_database.h"
 #include "components/precache/core/precache_switches.h"
 #include "components/precache/core/proto/precache.pb.h"
 #include "net/base/load_flags.h"
 #include "net/http/http_response_headers.h"
 #include "net/http/http_status_code.h"
 #include "net/url_request/test_url_fetcher_factory.h"
 #include "net/url_request/url_request_status.h"
 #include "net/url_request/url_request_test_util.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ skipping 52 matching lines @@
   int total_response_bytes() const { return total_response_bytes_; }
 
  private:
   // Multiset with one entry for each URL requested.
   std::multiset<GURL> requested_urls_;
   int total_response_bytes_;
 };
 
 class TestPrecacheDelegate : public PrecacheFetcher::PrecacheDelegate {
  public:
-  TestPrecacheDelegate() : was_on_done_called_(false) {}
+  TestPrecacheDelegate()
+      : on_done_was_called_(false),
+        on_init_done_was_called_(false),
+        on_shutdown_done_was_called_(false) {}
 
-  void OnDone() override { was_on_done_called_ = true; }
+  void OnDone() override { on_done_was_called_ = true; }
+  void OnInitDone() override { on_init_done_was_called_ = true; }
+  void OnShutdownDone() override { on_shutdown_done_was_called_ = true; }
+
 
   bool was_on_done_called() const {
-    return was_on_done_called_;
+    return on_done_was_called_;
+  }
+
+  bool on_init_done_was_called() const {
+    return on_init_done_was_called_;
+  }
+
+  bool on_shutdown_done_was_called() const {
+    return on_shutdown_done_was_called_;
   }
 
  private:
-  bool was_on_done_called_;
+  bool on_done_was_called_;
+  bool on_init_done_was_called_;
+  bool on_shutdown_done_was_called_;
 };
 
 class MockURLFetcherFactory : public net::URLFetcherFactory {
  public:
   typedef net::URLFetcher* DoURLFetcher(
       int id,
       const GURL& url,
       net::URLFetcher::RequestType request_type,
       net::URLFetcherDelegate* delegate);
 
@@ skipping 205 matching lines @@
 class PrecacheFetcherTest : public testing::Test {
  public:
   PrecacheFetcherTest()
       : request_context_(new net::TestURLRequestContextGetter(
             base::ThreadTaskRunnerHandle::Get())),
         factory_(NULL,
                  base::Bind(&TestURLFetcherCallback::CreateURLFetcher,
                             base::Unretained(&url_callback_))),
         expected_total_response_bytes_(0) {}
 
+  void SetUp() override {
+    precache_database_.reset(new PrecacheDatabase());
+
+    ASSERT_TRUE(scoped_temp_dir_.CreateUniqueTempDir());
+    base::FilePath db_path = scoped_temp_dir_.path().Append(
+        base::FilePath(FILE_PATH_LITERAL("precache_database")));
+    precache_database_->Init(db_path);
+  }
+
  protected:
   void SetDefaultFlags() {
     base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
         switches::kPrecacheConfigSettingsURL, kConfigURL);
     base::CommandLine::ForCurrentProcess()->AppendSwitchASCII(
         switches::kPrecacheManifestURLPrefix, kManifestURLPrefix);
   }
 
+  // Must be declared first so that it is destroyed last.
+  base::ScopedTempDir scoped_temp_dir_;
+  std::unique_ptr<PrecacheDatabase> precache_database_;
   base::MessageLoopForUI loop_;
   scoped_refptr<net::TestURLRequestContextGetter> request_context_;
   TestURLFetcherCallback url_callback_;
   net::FakeURLFetcherFactory factory_;
   TestPrecacheDelegate precache_delegate_;
   int expected_total_response_bytes_;
 };
 
 TEST_F(PrecacheFetcherTest, FullPrecache) {
   SetDefaultFlags();
@@ skipping 32 matching lines @@
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kForcedStartingURLManifestURL),
                            PrecacheManifest().SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
     PrecacheFetcher precache_fetcher(starting_hosts, request_context_.get(),
                                      GURL(), std::string(),
-                                     &precache_delegate_);
+                                     &precache_delegate_,
+                                     precache_database_->GetWeakPtr(),
+                                     base::ThreadTaskRunnerHandle::Get());
     precache_fetcher.Start();
 
     loop_.RunUntilIdle();
 
     // Destroy the PrecacheFetcher after it has finished, to record metrics.
   }
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kManifestFetchFailureURL));
   expected_requested_urls.insert(GURL(kBadManifestURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   expected_requested_urls.insert(GURL(kResourceFetchFailureURL));
   expected_requested_urls.insert(GURL(kGoodResourceURL));
   expected_requested_urls.insert(GURL(kForcedStartingURLManifestURL));
 
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 
   histogram.ExpectUniqueSample("Precache.Fetch.PercentCompleted", 100, 1);
   histogram.ExpectUniqueSample("Precache.Fetch.ResponseBytes.Total",
                                url_callback_.total_response_bytes(), 1);
   histogram.ExpectTotalCount("Precache.Fetch.TimeToComplete", 1);
 }
 
+TEST_F(PrecacheFetcherTest, PrecachePauseResume) {
+  SetDefaultFlags();
+
+  PrecacheConfigurationSettings config;
+  config.set_top_sites_count(3);
+  std::vector<std::string> starting_hosts;
+
+  PrecacheFetcher first_fetcher(starting_hosts, request_context_.get(),
+                                GURL(), std::string(),
+                                &precache_delegate_,
+                                precache_database_->GetWeakPtr(),
+                                base::ThreadTaskRunnerHandle::Get());
+  base::TimeTicks then = base::TimeTicks::Now() - base::TimeDelta::FromHours(1);
+  std::list<GURL> manifests, resources;
+  manifests.push_back(GURL("http://manifest-url-prefix.com/manifest1.com"));
+  manifests.push_back(GURL("http://manifest-url-prefix.com/manifest2.com"));
+  resources.push_back(GURL(kGoodResourceURL));
+  first_fetcher.AssignWorkForTest(
+      manifests, resources, 271828, 314159, 5, then);
+  first_fetcher.Shutdown();
+  loop_.RunUntilIdle();
+
+  std::multiset<GURL> expected_requested_urls;
+  EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
+
+  factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(),
+                           net::HTTP_OK, net::URLRequestStatus::SUCCESS);
+  factory_.SetFakeResponse(GURL(kBadManifestURL), "bad protobuf", net::HTTP_OK,
+                           net::URLRequestStatus::SUCCESS);
+  factory_.SetFakeResponse(GURL("http://manifest-url-prefix.com/manifest1.com"),
+                           "bad protobuf", net::HTTP_OK,
+                           net::URLRequestStatus::SUCCESS);
+  factory_.SetFakeResponse(GURL("http://manifest-url-prefix.com/manifest2.com"),
+                           "bad protobuf", net::HTTP_OK,
+                           net::URLRequestStatus::SUCCESS);
+  factory_.SetFakeResponse(GURL(kGoodResourceURL), "good", net::HTTP_OK,
+                           net::URLRequestStatus::SUCCESS);
+
+  // Starting hosts should not be fetched.
+  starting_hosts.push_back("bad-manifest.com");
+
+  PrecacheFetcher second_fetcher(starting_hosts, request_context_.get(),
+                                 GURL(), std::string(),
+                                 &precache_delegate_,
+                                 precache_database_->GetWeakPtr(),
+                                 base::ThreadTaskRunnerHandle::Get());
+  second_fetcher.Init(base::TimeDelta::FromHours(2));
+  loop_.RunUntilIdle();
+  second_fetcher.Start();
+  loop_.RunUntilIdle();
+
+  expected_requested_urls.insert(GURL(kConfigURL));
+  expected_requested_urls.insert(
+      GURL("http://manifest-url-prefix.com/manifest1.com"));
+  expected_requested_urls.insert(
+      GURL("http://manifest-url-prefix.com/manifest2.com"));
+  expected_requested_urls.insert(GURL(kGoodResourceURL));
+
+  EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
+  EXPECT_TRUE(precache_delegate_.was_on_done_called());
+}
+
 TEST_F(PrecacheFetcherTest, CustomURLs) {
   SetDefaultFlags();
 
   std::vector<std::string> starting_hosts;
   starting_hosts.push_back("good-manifest.com");
 
   PrecacheConfigurationSettings config;
 
   PrecacheManifest good_manifest;
   good_manifest.add_resource()->set_url(kGoodResourceURL);
 
   factory_.SetFakeResponse(GURL(kCustomConfigURL), config.SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kCustomGoodManifestURL),
                            good_manifest.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodResourceURL), "good", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   PrecacheFetcher precache_fetcher(
       starting_hosts, request_context_.get(), GURL(kCustomConfigURL),
-      kCustomManifestURLPrefix, &precache_delegate_);
+      kCustomManifestURLPrefix, &precache_delegate_,
+      precache_database_->GetWeakPtr(), base::ThreadTaskRunnerHandle::Get());
   precache_fetcher.Start();
 
   loop_.RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kCustomConfigURL));
   expected_requested_urls.insert(GURL(kCustomGoodManifestURL));
   expected_requested_urls.insert(GURL(kGoodResourceURL));
 
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 TEST_F(PrecacheFetcherTest, ConfigFetchFailure) {
   SetDefaultFlags();
 
   std::vector<std::string> starting_hosts(1, "good-manifest.com");
 
   factory_.SetFakeResponse(GURL(kConfigURL), "",
                            net::HTTP_INTERNAL_SERVER_ERROR,
                            net::URLRequestStatus::FAILED);
   factory_.SetFakeResponse(GURL(kGoodManifestURL), "", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   PrecacheFetcher precache_fetcher(starting_hosts, request_context_.get(),
-                                   GURL(), std::string(), &precache_delegate_);
+                                   GURL(), std::string(), &precache_delegate_,
+                                   precache_database_->GetWeakPtr(),
+                                   base::ThreadTaskRunnerHandle::Get());
   precache_fetcher.Start();
 
   loop_.RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 TEST_F(PrecacheFetcherTest, BadConfig) {
   SetDefaultFlags();
 
   std::vector<std::string> starting_hosts(1, "good-manifest.com");
 
   factory_.SetFakeResponse(GURL(kConfigURL), "bad protobuf", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL(kGoodManifestURL), "", net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   PrecacheFetcher precache_fetcher(starting_hosts, request_context_.get(),
-                                   GURL(), std::string(), &precache_delegate_);
+                                   GURL(), std::string(), &precache_delegate_,
+                                   precache_database_->GetWeakPtr(),
+                                   base::ThreadTaskRunnerHandle::Get());
   precache_fetcher.Start();
 
   loop_.RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
 
 TEST_F(PrecacheFetcherTest, Cancel) {
   SetDefaultFlags();
 
   std::vector<std::string> starting_hosts(1, "starting-url.com");
 
   PrecacheConfigurationSettings config;
   config.set_top_sites_count(1);
 
   factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
     PrecacheFetcher precache_fetcher(starting_hosts, request_context_.get(),
                                      GURL(), std::string(),
-                                     &precache_delegate_);
+                                     &precache_delegate_,
+                                     precache_database_->GetWeakPtr(),
+                                     base::ThreadTaskRunnerHandle::Get());
     precache_fetcher.Start();
 
     // Destroy the PrecacheFetcher, to cancel precaching and record metrics.
     // This should not cause OnDone to be called on the precache delegate.
   }
 
   loop_.RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
@@ skipping 14 matching lines @@
   std::vector<std::string> starting_hosts(1, "starting-url.com");
 
   PrecacheConfigurationSettings config;
   config.set_top_sites_count(0);
 
   factory_.SetFakeResponse(GURL(PRECACHE_CONFIG_SETTINGS_URL),
                            config.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   PrecacheFetcher precache_fetcher(starting_hosts, request_context_.get(),
-                                   GURL(), std::string(), &precache_delegate_);
+                                   GURL(), std::string(), &precache_delegate_,
+                                   precache_database_->GetWeakPtr(),
+                                   base::ThreadTaskRunnerHandle::Get());
   precache_fetcher.Start();
 
   loop_.RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(PRECACHE_CONFIG_SETTINGS_URL));
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 }
@@ skipping 14 matching lines @@
   config.set_top_sites_count(1);
 
   GURL manifest_url(PRECACHE_MANIFEST_URL_PREFIX "starting-url.com");
 
   factory_.SetFakeResponse(GURL(kConfigURL), config.SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(manifest_url, PrecacheManifest().SerializeAsString(),
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
 
   PrecacheFetcher precache_fetcher(starting_hosts, request_context_.get(),
-                                   GURL(), std::string(), &precache_delegate_);
+                                   GURL(), std::string(), &precache_delegate_,
+                                   precache_database_->GetWeakPtr(),
+                                   base::ThreadTaskRunnerHandle::Get());
   precache_fetcher.Start();
 
   loop_.RunUntilIdle();
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(manifest_url);
   EXPECT_EQ(expected_requested_urls, url_callback_.requested_urls());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
@@ skipping 23 matching lines @@
                            good_manifest.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
   factory_.SetFakeResponse(GURL("http://good-manifest.com/retrieved"), "good",
                            net::HTTP_OK, net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
     PrecacheFetcher precache_fetcher(starting_hosts, request_context_.get(),
                                      GURL(), std::string(),
-                                     &precache_delegate_);
+                                     &precache_delegate_,
+                                     precache_database_->GetWeakPtr(),
+                                     base::ThreadTaskRunnerHandle::Get());
     precache_fetcher.Start();
 
     loop_.RunUntilIdle();
 
     // Destroy the PrecacheFetcher after it has finished, to record metrics.
   }
 
   std::multiset<GURL> expected_requested_urls;
   expected_requested_urls.insert(GURL(kConfigURL));
   expected_requested_urls.insert(GURL(kGoodManifestURL));
@@ skipping 51 matching lines @@
 
   factory_.SetFakeResponse(GURL(kGoodManifestURL),
                            good_manifest.SerializeAsString(), net::HTTP_OK,
                            net::URLRequestStatus::SUCCESS);
 
   base::HistogramTester histogram;
 
   {
     PrecacheFetcher precache_fetcher({"good-manifest.com"},
                                      request_context_.get(), GURL(),
-                                     std::string(), &precache_delegate_);
+                                     std::string(), &precache_delegate_,
+                                     precache_database_->GetWeakPtr(),
+                                     base::ThreadTaskRunnerHandle::Get());
     precache_fetcher.Start();
 
     loop_.RunUntilIdle();
 
     // Destroy the PrecacheFetcher after it has finished, to record metrics.
   }
 
   // We don't know which will make it and which won't due to the parallelism in
   // the pool of Fetchers.
   EXPECT_GT(kNumResources, url_callback_.requested_urls().size());
 
   EXPECT_TRUE(precache_delegate_.was_on_done_called());
 
   // good-manifest.com will not have been completed.
   EXPECT_THAT(histogram.GetAllSamples("Precache.Fetch.PercentCompleted"),
               ElementsAre(base::Bucket(0, 1)));
   histogram.ExpectTotalCount("Precache.Fetch.TimeToComplete", 1);
 }
 
 }  // namespace
 
 }  // namespace precache