Send the list of used and unused resources for precache

components/precache/core/precache_fetcher.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_fetcher.h"
 
-#include <algorithm>

[sclittle, 2016/08/11 22:52:35]

You still need <algorithm> for std::min.

[Raj, 2016/08/12 19:04:20]

Done.

 #include <limits>
-#include <string>
 #include <utility>
-#include <vector>
 
+#include "base/base64.h"
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/containers/hash_tables.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/metrics/histogram_macros.h"
+#include "base/sha1.h"
+#include "base/task_runner_util.h"
+#include "components/precache/core/precache_database.h"
 #include "components/precache/core/precache_switches.h"
 #include "components/precache/core/proto/precache.pb.h"
 #include "components/precache/core/proto/unfinished_work.pb.h"
 #include "net/base/completion_callback.h"
 #include "net/base/escape.h"
 #include "net/base/io_buffer.h"
 #include "net/base/load_flags.h"
 #include "net/base/net_errors.h"
+#include "net/base/url_util.h"
 #include "net/http/http_response_headers.h"
 #include "net/url_request/url_fetcher_response_writer.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "net/url_request/url_request_status.h"
 
 namespace precache {
 
 // The following flags are for privacy reasons. For example, if a user clears
 // their cookies, but a tracking beacon is prefetched and the beacon specifies
 // its source URL in a URL param, the beacon site would be able to rebuild a
@@ skipping 41 matching lines @@
 
 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
   return PRECACHE_MANIFEST_URL_PREFIX;
 #else
   // The precache manifest URL prefix could not be determined, so return an
   // empty string.
   return std::string();
 #endif
 }
 
-// Construct the URL of the precache manifest for the given name (either host or
-// URL). The server is expecting a request for a URL consisting of the manifest
-// URL prefix followed by the doubly escaped name.
-std::string ConstructManifestURL(const std::string& prefix,
-                                 const std::string& name) {
-  return prefix + net::EscapeQueryParamValue(
-                      net::EscapeQueryParamValue(name, false), false);
-}
-
 // Attempts to parse a protobuf message from the response string of a
 // URLFetcher. If parsing is successful, the message parameter will contain the
 // parsed protobuf and this function will return true. Otherwise, returns false.
 bool ParseProtoFromFetchResponse(const net::URLFetcher& source,
                                  ::google::protobuf::MessageLite* message) {
   std::string response_string;
 
   if (!source.GetStatus().is_success()) {
     DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
     return false;
@@ skipping 40 matching lines @@
             int num_bytes,
             const net::CompletionCallback& callback) override {
     return num_bytes;
   }
 
   int Finish(const net::CompletionCallback& callback) override {
     return net::OK;
   }
 };
 
-void AppendManifestURLIfValidAndNew(
-    const std::string& prefix,
-    const std::string& name,
-    base::hash_set<std::string>* seen_manifest_urls,
-    std::list<GURL>* unique_manifest_urls) {
-  const std::string manifest_url = ConstructManifestURL(prefix, name);
-  bool first_seen = seen_manifest_urls->insert(manifest_url).second;
-  if (first_seen) {
-    GURL url(manifest_url);
-    if (url.is_valid())
-      unique_manifest_urls->push_back(url);
+// Returns the base64 encoded resource URL hashes. The resource URLs are hashed
+// individually, and 8 bytes of each hash is appended together, which is then
+// encoded to base64.
+std::string GetResourceURLBase64Hash(const std::deque<GURL>& urls) {
+  std::string hashes;

[sclittle, 2016/08/11 22:52:35]

nit: to avoid a bunch of reallocations, you could add:

hashes.reserve(urls.size() * base::kSHA1Length);

after this line so that it won't have to reallocate as you add to the string in
the for loop.

[Raj, 2016/08/12 19:04:20]

Done.

+  for (const auto& url : urls) {
+    std::string url_spec = url.spec();

[sclittle, 2016/08/11 22:52:35]

nit: change to const std::string& to avoid string copy

[Raj, 2016/08/12 19:04:20]

Done.

+    unsigned char sha1_hash[base::kSHA1Length];
+    base::SHA1HashBytes(

[bengr, 2016/08/11 18:49:14]

test that this works on an empty string.

+        reinterpret_cast<const unsigned char*>(url_spec.c_str()),
+        url_spec.size(), sha1_hash);
+    // Each resource hash uses 8 bytes.
+    hashes.append(reinterpret_cast<const char*>(sha1_hash), 8);

[sclittle, 2016/08/11 22:52:35]

Replace "8" with "arraysize(sha1_hash)".

[Raj, 2016/08/12 19:04:21]

hmm. sha1 is actually 20 bytes. But we are using only 8 bytes of it, as a
tradeoff between sending more bytes vs to reduce hash collision.

[sclittle, 2016/08/15 20:13:09]

Oh, OK, that makes sense. Could you explain that here in a comment then?

   }
+  base::Base64Encode(hashes, &hashes);
+  return hashes;
 }
 
 }  // namespace
 
 PrecacheFetcher::Fetcher::Fetcher(
     net::URLRequestContextGetter* request_context,
     const GURL& url,
+    const std::string& referrer,
     const base::Callback<void(const Fetcher&)>& callback,
     bool is_resource_request,
     size_t max_bytes)
     : request_context_(request_context),
       url_(url),
+      referrer_(referrer),
       callback_(callback),
       is_resource_request_(is_resource_request),
       max_bytes_(max_bytes),
       response_bytes_(0),
-      network_response_bytes_(0) {
+      network_response_bytes_(0),
+      was_cached_(false) {
+  DCHECK(url.is_valid());
   if (is_resource_request_)
     LoadFromCache();
   else
     LoadFromNetwork();
 }
 
 PrecacheFetcher::Fetcher::~Fetcher() {}
 
 void PrecacheFetcher::Fetcher::LoadFromCache() {
   fetch_stage_ = FetchStage::CACHE;
@@ skipping 32 matching lines @@
     const net::URLFetcher* source,
     int64_t current,
     int64_t total) {
   // If going over the per-resource download cap.
   if (fetch_stage_ == FetchStage::NETWORK &&
       // |current| is guaranteed to be non-negative, so this cast is safe.
       static_cast<size_t>(std::max(current, total)) > max_bytes_) {
     VLOG(1) << "Cancelling " << url_ << ": (" << current << "/" << total
             << ") is over " << max_bytes_;
 
-    // Cancel the download.
-    network_url_fetcher_.reset();
-
     // Call the completion callback, to attempt the next download, or to trigger
     // cleanup in precache_delegate_->OnDone().
     response_bytes_ = network_response_bytes_ = current;
+    was_cached_ = source->WasCached();
 
+    // Cancel the download.
+    network_url_fetcher_.reset();
     callback_.Run(*this);
   }
 }
 
 void PrecacheFetcher::Fetcher::OnURLFetchComplete(
     const net::URLFetcher* source) {
   CHECK(source);
   if (fetch_stage_ == FetchStage::CACHE &&
       (source->GetStatus().error() == net::ERR_CACHE_MISS ||
        (source->GetResponseHeaders() &&
         source->GetResponseHeaders()->HasValidators()))) {
     // If the resource was not found in the cache, request it from the
     // network.
     //
     // If the resource was found in the cache, but contains validators,
     // request a refresh. The presence of validators increases the chance that
     // we get a 304 response rather than a full one, thus allowing us to
     // refresh the cache with minimal network load.
     LoadFromNetwork();
     return;
   }
 
   // If any of:
   // - The request was for a config or manifest.
   // - The resource was a cache hit without validators.
   // - The response came from the network.
   // Then Fetcher is done with this URL and can return control to the caller.
   response_bytes_ = source->GetReceivedResponseContentLength();
   network_response_bytes_ = source->GetTotalReceivedBytes();
+  was_cached_ = source->WasCached();
   callback_.Run(*this);
 }
 
 // static
 void PrecacheFetcher::RecordCompletionStatistics(
     const PrecacheUnfinishedWork& unfinished_work,
     size_t remaining_manifest_urls_to_fetch,
     size_t remaining_resource_urls_to_fetch) {
   // These may be unset in tests.
   if (!unfinished_work.has_start_time())
@@ skipping 30 matching lines @@
                               1, kMaxResponseBytes,
                               100);
 }
 
 PrecacheFetcher::PrecacheFetcher(
     net::URLRequestContextGetter* request_context,
     const GURL& config_url,
     const std::string& manifest_url_prefix,
     std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
     uint32_t experiment_id,
+    base::WeakPtr<PrecacheDatabase> precache_database,

[sclittle, 2016/08/11 22:52:35]

nit: pass by const ref to avoid extra Add/Remove ref operations.

[Raj, 2016/08/12 19:04:21]

Done.

+    const scoped_refptr<base::SingleThreadTaskRunner>& db_task_runner,
     PrecacheFetcher::PrecacheDelegate* precache_delegate)
     : request_context_(request_context),
       config_url_(config_url),
       manifest_url_prefix_(manifest_url_prefix),
+      precache_database_(precache_database),
+      db_task_runner_(std::move(db_task_runner)),
       precache_delegate_(precache_delegate),
       pool_(kMaxParallelFetches),
       experiment_id_(experiment_id) {
   DCHECK(request_context_.get());  // Request context must be non-NULL.
   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
 
   DCHECK_NE(GURL(), GetDefaultConfigURL())
       << "Could not determine the precache config settings URL.";
   DCHECK_NE(std::string(), GetDefaultManifestURLPrefix())
       << "Could not determine the default precache manifest URL prefix.";
   DCHECK(unfinished_work);
 
-  // Copy manifests and resources to member variables as a convenience.
-  // TODO(bengr): Consider accessing these directly from the proto.

[Raj, 2016/08/09 22:56:48]

Since the proto datastructure does not support removing elements from the front
in O(1), it makes sense to copy to member variables.

-  for (const auto& manifest : unfinished_work->manifest()) {
-    if (manifest.has_url())
-      manifest_urls_to_fetch_.push_back(GURL(manifest.url()));
-  }
+  // Copy resources to member variable as a convenience.

[sclittle, 2016/08/11 22:52:35]

You're already changing the member variables so much, so why not just remove
most of them altogether?

Could you just keep track of the current resource index then, and increment that
as you go along? That would make all these copies and member variables
unnecessary. You could also just call unfinished_work_->add_resource() or
whatever as new resources need to be fetched.

You might also greatly simplify CancelPrecaching, since you could just call
stuff like e.g.

unfinished_work_->resource().erase(unfinished_work_->resource().begin(),
unfinished_work_->resource().begin() + resource_index_);

instead of copying everything back to the unfinished_work_ from the member
variables.

[Raj, 2016/08/12 19:04:20]

hmm. That is a good suggestion.
Added a TODO for now, to do in a separate CL.

   for (const auto& resource : unfinished_work->resource()) {
-    if (resource.has_url())
-      resource_urls_to_fetch_.push_back(GURL(resource.url()));
+    if (resource.has_url() && resource.has_tophostname()) {
+      resources_to_fetch_.emplace_back(
+          std::make_pair(GURL(resource.url()), resource.tophostname()));
+    }
   }
   unfinished_work_ = std::move(unfinished_work);
 }
 
 PrecacheFetcher::~PrecacheFetcher() {
 }
 
 std::unique_ptr<PrecacheUnfinishedWork> PrecacheFetcher::CancelPrecaching() {
   // This could get called multiple times, and it should be handled gracefully.
   if (!unfinished_work_)
     return nullptr;
 
-  unfinished_work_->clear_manifest();
   unfinished_work_->clear_resource();
-  for (const auto& manifest : manifest_urls_to_fetch_)
-    unfinished_work_->add_manifest()->set_url(manifest.spec());
-  for (const auto& resource : resource_urls_to_fetch_)
-    unfinished_work_->add_resource()->set_url(resource.spec());
+  if (top_hosts_to_fetch_) {
+    unfinished_work_->clear_top_host();
+    for (const auto& top_host : *top_hosts_to_fetch_) {
+      unfinished_work_->add_top_host()->set_hostname(top_host.hostname);
+    }
+  }
+  for (const auto& resource : resources_to_fetch_) {
+    auto new_resource = unfinished_work_->add_resource();
+    new_resource->set_url(resource.first.spec());
+    new_resource->set_tophostname(resource.second);
+  }
   for (const auto& it : pool_.elements()) {
     const Fetcher* fetcher = it.first;
-    if (fetcher->is_resource_request())
-      unfinished_work_->add_resource()->set_url(fetcher->url().spec());
-    else if (fetcher->url() != config_url_)
-      unfinished_work_->add_manifest()->set_url(fetcher->url().spec());
+    GURL config_url =
+        config_url_.is_empty() ? GetDefaultConfigURL() : config_url_;
+    if (fetcher->is_resource_request()) {
+      auto resource = unfinished_work_->add_resource();
+      resource->set_url(fetcher->url().spec());
+      resource->set_tophostname(fetcher->referrer());
+    } else if (fetcher->url() != config_url) {
+      unfinished_work_->add_top_host()->set_hostname(fetcher->referrer());
+    }
   }
-  manifest_urls_to_fetch_.clear();
-  resource_urls_to_fetch_.clear();
+  top_hosts_to_fetch_.reset();
+  resources_to_fetch_.clear();
   pool_.DeleteAll();
   return std::move(unfinished_work_);
 }
 
 void PrecacheFetcher::Start() {
   if (unfinished_work_->has_config_settings()) {
     DCHECK(unfinished_work_->has_start_time());
     DetermineManifests();
     return;
   }
 
   GURL config_url =
       config_url_.is_empty() ? GetDefaultConfigURL() : config_url_;
 
   DCHECK(config_url.is_valid()) << "Config URL not valid: "
                                 << config_url.possibly_invalid_spec();
 
   // Fetch the precache configuration settings from the server.
   DCHECK(pool_.IsEmpty()) << "All parallel requests should be available";
   VLOG(3) << "Fetching " << config_url;
   pool_.Add(base::WrapUnique(new Fetcher(
-      request_context_.get(), config_url,
-      base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
-                 base::Unretained(this)),
+      request_context_.get(), config_url, std::string(),
+      base::Bind(&PrecacheFetcher::OnConfigFetchComplete, AsWeakPtr()),
       false /* is_resource_request */, std::numeric_limits<int32_t>::max())));
 }
 
 void PrecacheFetcher::StartNextResourceFetch() {
   DCHECK(unfinished_work_->has_config_settings());
-  while (!resource_urls_to_fetch_.empty() && pool_.IsAvailable()) {
+  while (!resources_to_fetch_.empty() && pool_.IsAvailable()) {
+    const auto& resource = resources_to_fetch_.front();
     const size_t max_bytes =
         std::min(unfinished_work_->config_settings().max_bytes_per_resource(),
                  unfinished_work_->config_settings().max_bytes_total() -
                      unfinished_work_->total_bytes());
-    VLOG(3) << "Fetching " << resource_urls_to_fetch_.front();
-    pool_.Add(base::WrapUnique(
-        new Fetcher(request_context_.get(), resource_urls_to_fetch_.front(),
-                    base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
-                               base::Unretained(this)),
-                    true /* is_resource_request */, max_bytes)));
+    VLOG(3) << "Fetching " << resource.first << " " << resource.second;
+    pool_.Add(base::WrapUnique(new Fetcher(
+        request_context_.get(), resource.first, resource.second,
+        base::Bind(&PrecacheFetcher::OnResourceFetchComplete, AsWeakPtr()),
+        true /* is_resource_request */, max_bytes)));
 
-    resource_urls_to_fetch_.pop_front();
+    resources_to_fetch_.pop_front();
   }
 }
 
 void PrecacheFetcher::StartNextManifestFetch() {
-  if (manifest_urls_to_fetch_.empty() || !pool_.IsAvailable())
+  if (!top_hosts_to_fetch_ || top_hosts_to_fetch_->empty() ||
+      !pool_.IsAvailable())
     return;
 
   // We only fetch one manifest at a time to keep the size of
-  // resource_urls_to_fetch_ as small as possible.
-  VLOG(3) << "Fetching " << manifest_urls_to_fetch_.front();
+  // resources_to_fetch_ as small as possible.
+  VLOG(3) << "Fetching " << top_hosts_to_fetch_->front().manifest_url;
   pool_.Add(base::WrapUnique(new Fetcher(
-      request_context_.get(), manifest_urls_to_fetch_.front(),
-      base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
-                 base::Unretained(this)),
+      request_context_.get(), top_hosts_to_fetch_->front().manifest_url,
+      top_hosts_to_fetch_->front().hostname,
+      base::Bind(&PrecacheFetcher::OnManifestFetchComplete, AsWeakPtr()),
       false /* is_resource_request */, std::numeric_limits<int32_t>::max())));
-
-  manifest_urls_to_fetch_.pop_front();
+  top_hosts_to_fetch_->pop_front();
 }
 
 void PrecacheFetcher::NotifyDone(
     size_t remaining_manifest_urls_to_fetch,
     size_t remaining_resource_urls_to_fetch) {
   RecordCompletionStatistics(*unfinished_work_,
                              remaining_manifest_urls_to_fetch,
                              remaining_resource_urls_to_fetch);
   precache_delegate_->OnDone();
 }
 
 void PrecacheFetcher::StartNextFetch() {
   DCHECK(unfinished_work_->has_config_settings());
   // If over the precache total size cap, then stop prefetching.
   if (unfinished_work_->total_bytes() >
       unfinished_work_->config_settings().max_bytes_total()) {
     size_t pending_manifests_in_pool = 0;
     size_t pending_resources_in_pool = 0;
     for (const auto& element_pair : pool_.elements()) {
       const Fetcher* fetcher = element_pair.first;
       if (fetcher->is_resource_request())
         pending_resources_in_pool++;
       else if (fetcher->url() != config_url_)
         pending_manifests_in_pool++;
     }
     pool_.DeleteAll();
-    NotifyDone(manifest_urls_to_fetch_.size() + pending_manifests_in_pool,
-               resource_urls_to_fetch_.size() + pending_resources_in_pool);
+    int pending_top_hosts =
+        top_hosts_to_fetch_ ? top_hosts_to_fetch_->size() : 0;
+    NotifyDone(pending_top_hosts + pending_manifests_in_pool,
+               resources_to_fetch_.size() + pending_resources_in_pool);
     return;
   }
 
   StartNextResourceFetch();
   StartNextManifestFetch();
-  if (pool_.IsEmpty()) {
+  if ((!top_hosts_to_fetch_ || top_hosts_to_fetch_->empty()) &&
+      resources_to_fetch_.empty() && pool_.IsEmpty()) {
     // There are no more URLs to fetch, so end the precache cycle.
     NotifyDone(0, 0);
     // OnDone may have deleted this PrecacheFetcher, so don't do anything after
     // it is called.
   }
 }
 
 void PrecacheFetcher::OnConfigFetchComplete(const Fetcher& source) {
   UpdateStats(source.response_bytes(), source.network_response_bytes());
   if (source.network_url_fetcher() == nullptr) {
     pool_.DeleteAll();  // Cancel any other ongoing request.
   } else {
     // Attempt to parse the config proto. On failure, continue on with the
     // default configuration.
     ParseProtoFromFetchResponse(
         *source.network_url_fetcher(),
         unfinished_work_->mutable_config_settings());
     pool_.Delete(source);
     DetermineManifests();
   }
 }
 
 void PrecacheFetcher::DetermineManifests() {
   DCHECK(unfinished_work_->has_config_settings());
-    std::string prefix = manifest_url_prefix_.empty()
-                             ? GetDefaultManifestURLPrefix()
-                             : manifest_url_prefix_;
-    DCHECK_NE(std::string(), prefix)
-        << "Could not determine the precache manifest URL prefix.";
 
-    // Keep track of manifest URLs that are being fetched, in order to elide
-    // duplicates.
-    base::hash_set<std::string> seen_manifest_urls;
+  // Keep track of manifest URLs that are being fetched, in order to elide
+  // duplicates.
+  std::set<std::string> seen_top_hosts;

[sclittle, 2016/08/11 22:52:35]

You should move this into the below if-statement since it's only used within
there.

Also, I think you might be able to just do this with an
std::set<base::StringPiece> here, and avoid a bunch of string copies, since none
of the top hosts are being removed and you're just referencing them directly
from the unfinished_work_ anyways.

[Raj, 2016/08/12 19:04:20]

Done.

+  std::unique_ptr<std::deque<std::string>> top_hosts_to_fetch(

[sclittle, 2016/08/11 22:52:35]

You can remove the std::unique_ptr around this if you just use std::move when
passing it to the callback below, and change all the methods that currently take
ownership of the std::unique_ptr to take in a "std::deque<...>" (note how
there's no "const &") and use std::move to take ownership of them in turn.

[Raj, 2016/08/12 19:04:20]

Nice. It works. But top_hosts_to_fetch is a local variable in stack now, and I
do not know how std::move could move it without copy. At least a small
std::deque structure would get copied ?

+      new std::deque<std::string>);
+  std::unique_ptr<std::deque<ManifestHostInfo>> top_hosts_info(
+      new std::deque<ManifestHostInfo>);
 
-    // Attempt to fetch manifests for starting hosts up to the maximum top sites
-    // count. If a manifest does not exist for a particular starting host, then
-    // the fetch will fail, and that starting host will be ignored. Starting
-    // hosts are not added if this is a continuation from a previous precache
-    // session.
-    if (manifest_urls_to_fetch_.empty() &&
-        resource_urls_to_fetch_.empty()) {
-      int64_t rank = 0;
-      for (const auto& host : unfinished_work_->top_host()) {
-        ++rank;
-        if (rank > unfinished_work_->config_settings().top_sites_count())
-          break;
-        AppendManifestURLIfValidAndNew(prefix, host.hostname(),
-                                       &seen_manifest_urls,
-                                       &manifest_urls_to_fetch_);
-      }
+  // Attempt to fetch manifests for starting hosts up to the maximum top sites
+  // count. If a manifest does not exist for a particular starting host, then
+  // the fetch will fail, and that starting host will be ignored. Starting
+  // hosts are not added if this is a continuation from a previous precache
+  // session.
+  if (top_hosts_to_fetch->empty() && resources_to_fetch_.empty()) {

[sclittle, 2016/08/11 22:52:35]

|top_hosts_to_fetch| will always be empty at this point, won't it?

[Raj, 2016/08/12 19:04:20]

Yep.

+    int64_t rank = 0;
+    for (const auto& host : unfinished_work_->top_host()) {
+      ++rank;
+      if (rank > unfinished_work_->config_settings().top_sites_count())
+        break;
+      if (seen_top_hosts.insert(host.hostname()).second)
+        top_hosts_to_fetch->emplace_back(host.hostname());

[sclittle, 2016/08/11 22:52:35]

nit: I think you can just use push_back here, it'll have the same effect, plus
it's simpler.

[Raj, 2016/08/12 19:04:20]

Done.

+    }
 
-      for (const std::string& host
-          : unfinished_work_->config_settings().forced_site()) {
-        AppendManifestURLIfValidAndNew(prefix, host, &seen_manifest_urls,
-                                       &manifest_urls_to_fetch_);
-      }
+    for (const std::string& host :
+         unfinished_work_->config_settings().forced_site()) {
+      if (seen_top_hosts.insert(host).second)
+        top_hosts_to_fetch->emplace_back(host);

[sclittle, 2016/08/11 22:52:35]

nit: I think you can replace emplace_back with push_back here. Same with
everywhere else you use emplace_back.

[Raj, 2016/08/12 19:04:21]

Done.

     }
-    unfinished_work_->set_num_manifest_urls(manifest_urls_to_fetch_.size());
-    StartNextFetch();
+  }
+  // We only fetch one manifest at a time to keep the size of
+  // resources_to_fetch_ as small as possible.
+  auto retrieve_manifest_callback =
+      base::Bind(&PrecacheFetcher::RetrieveManifestInfo, AsWeakPtr(),
+                 base::Passed(&top_hosts_to_fetch), top_hosts_info.get());
+  db_task_runner_->PostTaskAndReply(
+      FROM_HERE, retrieve_manifest_callback,
+      base::Bind(&PrecacheFetcher::OnManifestInfoRetrieved, AsWeakPtr(),
+                 base::Passed(&top_hosts_info)));
 }
 
+void PrecacheFetcher::RetrieveManifestInfo(
+    std::unique_ptr<std::deque<std::string>> hosts_to_fetch,
+    std::deque<ManifestHostInfo>* hosts_info) {
+  for (const auto& host : *hosts_to_fetch) {
+    auto referrer_host_info = precache_database_->GetReferrerHost(host);
+    if (referrer_host_info.id != PrecacheReferrerHostEntry::INVALID_ID) {
+      std::deque<GURL> used_urls, unused_urls;

[sclittle, 2016/08/11 22:52:35]

nit: Can this just be an std::vector? Typically an std::vector is best unless
there's a good reason since it's simpler.

[Raj, 2016/08/12 19:04:20]

Done.

+      precache_database_->GetURLListForReferrerHost(referrer_host_info.id,
+                                                    used_urls, unused_urls);
+      hosts_info->emplace_back(ManifestHostInfo(
+          referrer_host_info.id, host, GetResourceURLBase64Hash(used_urls),
+          GetResourceURLBase64Hash(unused_urls)));
+    } else {
+      hosts_info->emplace_back(
+          ManifestHostInfo(PrecacheReferrerHostEntry::INVALID_ID, host,
+                           std::string(), std::string()));
+    }
+  }
+}
+
+void PrecacheFetcher::OnManifestInfoRetrieved(
+    std::unique_ptr<std::deque<ManifestHostInfo>> manifests_info) {
+  DCHECK(manifests_info);
+  const std::string prefix = manifest_url_prefix_.empty()
+                                 ? GetDefaultManifestURLPrefix()
+                                 : manifest_url_prefix_;
+  top_hosts_to_fetch_.reset(new std::deque<ManifestHostInfo>());
+  for (auto& manifest : *manifests_info) {
+    GURL manifest_url(
+        prefix +
+        net::EscapeQueryParamValue(
+            net::EscapeQueryParamValue(manifest.hostname, false), false));

[sclittle, 2016/08/11 22:52:35]

Can't you just add all the query params when you create the manifest URL here?
Repeatedly copying the GURL is kinda expensive.

[Raj, 2016/08/12 19:04:20]

I do not see any GURL constructor that takes query parameters. Do you know of an
api.
Or do you suggest to directly form the querystring and call ReplaceComponents().

[sclittle, 2016/08/15 20:13:09]

Ok, this is probably fine then. I don't see a nicer way to do it than this,
although this is kinda expensive.

+    if (manifest_url.is_valid() &&
+        manifest.manifest_id != PrecacheReferrerHostEntry::INVALID_ID) {
+      manifest_url = net::AppendOrReplaceQueryParameter(
+          manifest_url, "manifest", std::to_string(manifest.manifest_id));
+      manifest_url = net::AppendOrReplaceQueryParameter(
+          manifest_url, "used_resources", manifest.used_url_hash);
+      manifest_url = net::AppendOrReplaceQueryParameter(
+          manifest_url, "unused_resources", manifest.unused_url_hash);
+      DCHECK(manifest_url.is_valid());
+    }
+    manifest.manifest_url = manifest_url;
+    if (manifest_url.is_valid())
+      top_hosts_to_fetch_->emplace_back(manifest);
+  }
+  unfinished_work_->set_num_manifest_urls(top_hosts_to_fetch_->size());
+  StartNextFetch();
+}
+
+ManifestHostInfo::ManifestHostInfo(int64_t manifest_id,
+                                   const std::string& hostname,
+                                   const std::string& used_url_hash,
+                                   const std::string& unused_url_hash)
+    : manifest_id(manifest_id),
+      hostname(hostname),
+      used_url_hash(used_url_hash),
+      unused_url_hash(unused_url_hash) {}
+
+ManifestHostInfo::~ManifestHostInfo() {}
+
+ManifestHostInfo::ManifestHostInfo(const ManifestHostInfo& other) = default;
+
 void PrecacheFetcher::OnManifestFetchComplete(const Fetcher& source) {
   DCHECK(unfinished_work_->has_config_settings());
   UpdateStats(source.response_bytes(), source.network_response_bytes());
   if (source.network_url_fetcher() == nullptr) {
     pool_.DeleteAll();  // Cancel any other ongoing request.
   } else {
     PrecacheManifest manifest;
 
     if (ParseProtoFromFetchResponse(*source.network_url_fetcher(), &manifest)) {
       const int32_t len =
           std::min(manifest.resource_size(),
                    unfinished_work_->config_settings().top_resources_count());
       const uint64_t resource_bitset =
           GetResourceBitset(manifest, experiment_id_);
       for (int i = 0; i < len; ++i) {
         if (((0x1ULL << i) & resource_bitset) &&
             manifest.resource(i).has_url()) {
           GURL url(manifest.resource(i).url());
-          if (url.is_valid())
-            resource_urls_to_fetch_.push_back(url);
+          if (url.is_valid()) {
+            resources_to_fetch_.emplace_back(
+                std::make_pair(url, source.referrer()));
+          }
         }
       }
+      db_task_runner_->PostTask(
+          FROM_HERE,
+          base::Bind(&PrecacheDatabase::UpdatePrecacheReferrerHost,
+                     precache_database_, source.referrer(),
+                     manifest.id().timestamp().seconds(), base::Time::Now()));
     }
   }
 
   pool_.Delete(source);
   StartNextFetch();
 }
 
 void PrecacheFetcher::OnResourceFetchComplete(const Fetcher& source) {
   UpdateStats(source.response_bytes(), source.network_response_bytes());
+
+  db_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&PrecacheDatabase::RecordURLPrefetch, precache_database_,
+                 source.url(), source.referrer(), base::Time::Now(),
+                 source.was_cached(), source.response_bytes()));
+
   pool_.Delete(source);
+
   // The resource has already been put in the cache during the fetch process, so
   // nothing more needs to be done for the resource.
   StartNextFetch();
 }
 
 void PrecacheFetcher::UpdateStats(int64_t response_bytes,
                                   int64_t network_response_bytes) {
   unfinished_work_->set_total_bytes(
       unfinished_work_->total_bytes() + response_bytes);
   unfinished_work_->set_network_bytes(
       unfinished_work_->network_bytes() + network_response_bytes);
 }
 
 }  // namespace precache