Add pause/resume functionality to precache

components/precache/core/precache_fetcher.cc

Index: components/precache/core/precache_fetcher.cc
diff --git a/components/precache/core/precache_fetcher.cc b/components/precache/core/precache_fetcher.cc
index e7a5a0bce807a63ca3c8913f66924d154b331601..8a30ed2eeacf416110956d5dd60a47f70ee40e76 100644
--- a/components/precache/core/precache_fetcher.cc
+++ b/components/precache/core/precache_fetcher.cc
@@ -11,15 +11,19 @@
 #include <vector>
 
 #include "base/bind.h"
+#include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/containers/hash_tables.h"
+#include "base/location.h"
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
+#include "base/memory/ref_counted.h"
 #include "base/metrics/histogram_macros.h"
 #include "components/precache/core/precache_switches.h"
 #include "components/precache/core/proto/precache.pb.h"
+#include "components/precache/core/proto/unfinished_work.pb.h"
 #include "net/base/completion_callback.h"
 #include "net/base/escape.h"
 #include "net/base/io_buffer.h"
@@ -261,9 +265,11 @@ PrecacheFetcher::PrecacheFetcher(
     net::URLRequestContextGetter* request_context,
     const GURL& config_url,
     const std::string& manifest_url_prefix,
+    const PrecacheUnfinishedWork* unfinished_work,
     PrecacheFetcher::PrecacheDelegate* precache_delegate)
     : starting_hosts_(starting_hosts),
       request_context_(request_context),
+      has_config_(unfinished_work && unfinished_work->has_config_settings()),
       config_url_(config_url),
       manifest_url_prefix_(manifest_url_prefix),
       precache_delegate_(precache_delegate),
@@ -279,10 +285,30 @@ PrecacheFetcher::PrecacheFetcher(
       << "Could not determine the precache config settings URL.";
   DCHECK_NE(std::string(), GetDefaultManifestURLPrefix())
       << "Could not determine the default precache manifest URL prefix.";
+
+  if (has_config_)

[sclittle, 2016/05/19 01:59:45]

This seems unnecessary, just check has_config_settings().

[bengr, 2016/05/19 23:19:31]

Done.

+    config_->CopyFrom(unfinished_work->config_settings());
+  for (int i = 0; i < unfinished_work->manifest_size(); ++i)

[sclittle, 2016/05/19 01:59:45]

nit: use range-based for, e.g. for (const auto& blah: ...)

[sclittle, 2016/05/19 01:59:45]

nit: braces

[bengr, 2016/05/19 23:19:31]

Done.

[bengr, 2016/05/19 23:19:31]

Done.

+    if (unfinished_work->manifest(i).has_url())
+      manifest_urls_to_fetch_.push_back(
+          GURL(unfinished_work->manifest(i).url()));
+  for (int i = 0; i < unfinished_work->resource_size(); ++i) {
+    if (unfinished_work->resource(i).has_url())
+      resource_urls_to_fetch_.push_back(
+          GURL(unfinished_work->resource(i).url()));
+  }
+  if (unfinished_work->has_total_bytes())
+    total_response_bytes_ = unfinished_work->total_bytes();
+  if (unfinished_work->has_network_bytes())
+    network_response_bytes_ = unfinished_work->network_bytes();
+  if (unfinished_work->has_num_manifest_urls())
+    num_manifest_urls_to_fetch_ = unfinished_work->num_manifest_urls();
+  if (unfinished_work->has_start_time())
+    start_time_ = base::Time::FromInternalValue(unfinished_work->start_time());
 }
 
 PrecacheFetcher::~PrecacheFetcher() {
-  base::TimeDelta time_to_fetch = base::TimeTicks::Now() - start_time_;
+  base::TimeDelta time_to_fetch = base::Time::Now() - start_time_;
   UMA_HISTOGRAM_CUSTOM_TIMES("Precache.Fetch.TimeToComplete", time_to_fetch,
                              base::TimeDelta::FromSeconds(1),
                              base::TimeDelta::FromHours(4), 50);
@@ -310,14 +336,35 @@ PrecacheFetcher::~PrecacheFetcher() {
                               100);
 }
 
+std::unique_ptr<PrecacheUnfinishedWork> PrecacheFetcher::GetUnfinishedWork() {
+  std::unique_ptr<PrecacheUnfinishedWork> unfinished_work(
+      new PrecacheUnfinishedWork());
+  if (has_config_)
+    unfinished_work->mutable_config_settings()->CopyFrom(*(config_.get()));
+  for (const auto& manifest : manifest_urls_to_fetch_)
+    unfinished_work->add_manifest()->set_url(manifest.spec());
+  for (const auto& resource : resource_urls_to_fetch_)
+    unfinished_work->add_resource()->set_url(resource.spec());
+  unfinished_work->set_total_bytes(total_response_bytes_);
+  unfinished_work->set_network_bytes(network_response_bytes_);
+  unfinished_work->set_num_manifest_urls(num_manifest_urls_to_fetch_);
+  unfinished_work->set_start_time(start_time_.ToInternalValue());
+  return unfinished_work;
+}
+
 void PrecacheFetcher::Start() {
+  if (has_config_) {
+    DetermineManifests();
+    return;
+  }
+
   GURL config_url =
       config_url_.is_empty() ? GetDefaultConfigURL() : config_url_;
 
   DCHECK(config_url.is_valid()) << "Config URL not valid: "
                                 << config_url.possibly_invalid_spec();
 
-  start_time_ = base::TimeTicks::Now();
+  start_time_ = base::Time::Now();
 
   // Fetch the precache configuration settings from the server.
   DCHECK(pool_.IsEmpty()) << "All parallel requests should be available";
@@ -373,7 +420,6 @@ void PrecacheFetcher::StartNextFetch() {
 
   StartNextResourceFetch();
   StartNextManifestFetch();
-
   if (pool_.IsEmpty()) {
     // There are no more URLs to fetch, so end the precache cycle.
     precache_delegate_->OnDone();
@@ -389,8 +435,15 @@ void PrecacheFetcher::OnConfigFetchComplete(const Fetcher& source) {
   } else {
     // Attempt to parse the config proto. On failure, continue on with the
     // default configuration.
-    ParseProtoFromFetchResponse(*source.network_url_fetcher(), config_.get());
+    has_config_ =
+        ParseProtoFromFetchResponse(
+            *source.network_url_fetcher(), config_.get());
+    pool_.Delete(source);
+    DetermineManifests();
+  }
+}
 
+void PrecacheFetcher::DetermineManifests() {
     std::string prefix = manifest_url_prefix_.empty()
                              ? GetDefaultManifestURLPrefix()
                              : manifest_url_prefix_;
@@ -403,25 +456,27 @@ void PrecacheFetcher::OnConfigFetchComplete(const Fetcher& source) {
 
     // Attempt to fetch manifests for starting hosts up to the maximum top sites
     // count. If a manifest does not exist for a particular starting host, then
-    // the fetch will fail, and that starting host will be ignored.
-    int64_t rank = 0;
-    for (const std::string& host : starting_hosts_) {
-      ++rank;
-      if (rank > config_->top_sites_count())
-        break;
-      AppendManifestURLIfNew(prefix, host, &seen_manifest_urls,
-                             &manifest_urls_to_fetch_);
-    }
-
-    for (const std::string& host : config_->forced_site())
-      AppendManifestURLIfNew(prefix, host, &seen_manifest_urls,
-                             &manifest_urls_to_fetch_);
+    // the fetch will fail, and that starting host will be ignored. Starting
+    // hosts are not added if this is a continuation from a previous precache
+    // session.
+    if (manifest_urls_to_fetch_.empty() &&
+        resource_urls_to_fetch_.empty()) {
+      int64_t rank = 0;
+      for (const std::string& host : starting_hosts_) {
+        ++rank;
+        if (rank > config_->top_sites_count())
+          break;
+        AppendManifestURLIfNew(prefix, host, &seen_manifest_urls,
+                               &manifest_urls_to_fetch_);
+      }
 
+      for (const std::string& host : config_->forced_site()) {
+        AppendManifestURLIfNew(prefix, host, &seen_manifest_urls,
+                               &manifest_urls_to_fetch_);
+      }
+    }
     num_manifest_urls_to_fetch_ = manifest_urls_to_fetch_.size();
-  }
-  pool_.Delete(source);
-
-  StartNextFetch();
+    StartNextFetch();
 }
 
 void PrecacheFetcher::OnManifestFetchComplete(const Fetcher& source) {
@@ -435,9 +490,8 @@ void PrecacheFetcher::OnManifestFetchComplete(const Fetcher& source) {
       const int32_t len =
           std::min(manifest.resource_size(), config_->top_resources_count());
       for (int i = 0; i < len; ++i) {
-        if (manifest.resource(i).has_url()) {
+        if (manifest.resource(i).has_url())
           resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
-        }
       }
     }
   }
@@ -460,4 +514,20 @@ void PrecacheFetcher::UpdateStats(int64_t response_bytes,
   network_response_bytes_ += network_response_bytes;
 }
 
+void PrecacheFetcher::AssignWorkForTest(const std::list<GURL>& manifests,
+                                        const std::list<GURL>& resources,
+                                        size_t total_response_bytes,
+                                        size_t network_response_bytes,
+                                        size_t num_manifest_urls_to_fetch,
+                                        const base::Time& start_time) {
+  DCHECK(manifest_urls_to_fetch_.empty());
+  DCHECK(resource_urls_to_fetch_.empty());
+  manifest_urls_to_fetch_ = manifests;
+  resource_urls_to_fetch_ = resources;
+  total_response_bytes_ = total_response_bytes;
+  network_response_bytes_ = network_response_bytes;
+  num_manifest_urls_to_fetch_ = num_manifest_urls_to_fetch;
+  start_time_ = start_time;
+}
+
 }  // namespace precache