Send the list of used and unused resources for precache

components/precache/core/precache_fetcher.h

Index: components/precache/core/precache_fetcher.h
diff --git a/components/precache/core/precache_fetcher.h b/components/precache/core/precache_fetcher.h
index 24307b2a02fdd69f4b2d3b1135b6762ad280abc3..b0c6f5b4e110eca1d99b74221d385478eaddc9ff 100644
--- a/components/precache/core/precache_fetcher.h
+++ b/components/precache/core/precache_fetcher.h
@@ -8,13 +8,14 @@
 #include <stdint.h>
 
 #include <algorithm>

[sclittle, 2016/08/11 22:52:36]

Remove this include from the header file and put it in the .cc file.

[Raj, 2016/08/12 19:04:21]

Done.

-#include <list>
+#include <deque>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "base/callback.h"
 #include "base/macros.h"
+#include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/single_thread_task_runner.h"
 #include "base/time/time.h"
@@ -34,11 +35,29 @@ class URLRequestContextGetter;
 namespace precache {
 
 class PrecacheConfigurationSettings;
+class PrecacheDatabase;
 class PrecacheUnfinishedWork;
 
 // Visible for testing.
 extern const int kNoTracking;
 
+// Contains the information about manifest for a host.
+struct ManifestHostInfo {
+  ManifestHostInfo(int64_t manifest_id,
+                   const std::string& hostname,
+                   const std::string& used_url_hash,
+                   const std::string& unused_url_hash);
+  ManifestHostInfo(const ManifestHostInfo& other);

[sclittle, 2016/08/11 22:52:35]

Is it possible to make this struct movable as well? Doesn't have to be too
complicated - just also add:

ManifestHostInfo(ManifestHostInfo&&) = default;
ManifestHostInfo& operator=(ManifestHostInfo&&) = default;

(You'll probably want to just have the declarations here in the .h and the
=default definitions in the .cc like you do here for the copy constructor.)

That'll greatly reduce the number of string copies performed when dealing with
these objects, since it'll just be copying the |manifest_url| then and moving
everything else.

[Raj, 2016/08/12 19:04:21]

Nice. Done.

+
+  ~ManifestHostInfo();
+
+  const int64_t manifest_id;

[sclittle, 2016/08/11 22:52:36]

Can you change these to not be const? You can just pass around a "const
ManifestHostInfo&" if you want them not to be modified.

[Raj, 2016/08/12 19:04:21]

Done.

+  const std::string hostname;
+  GURL manifest_url;
+  const std::string used_url_hash;
+  const std::string unused_url_hash;
+};
+
 // Public interface to code that fetches resources that the user is likely to
 // want to fetch in the future, putting them in the network stack disk cache.
 // Precaching is intended to be done when Chrome is not actively in use, likely
@@ -101,16 +120,20 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
       size_t remaining_manifest_urls_to_fetch,
       size_t remaining_resource_urls_to_fetch);
 
-  // Constructs a new PrecacheFetcher. The |starting_hosts| parameter is a
+  // Constructs a new PrecacheFetcher. The |unfinished_work| contains the
   // prioritized list of hosts that the user commonly visits. These hosts are
   // used by a server side component to construct a list of resource URLs that
   // the user is likely to fetch. Takes ownership of |unfinished_work|.
-  PrecacheFetcher(net::URLRequestContextGetter* request_context,
-                  const GURL& config_url,
-                  const std::string& manifest_url_prefix,
-                  std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
-                  uint32_t experiment_id,
-                  PrecacheDelegate* precache_delegate);
+  // |precache_database| should be accessed on;y in |db_task_runner|.
+  PrecacheFetcher(
+      net::URLRequestContextGetter* request_context,
+      const GURL& config_url,
+      const std::string& manifest_url_prefix,
+      std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
+      uint32_t experiment_id,
+      base::WeakPtr<PrecacheDatabase> precache_database,
+      const scoped_refptr<base::SingleThreadTaskRunner>& db_task_runner,
+      PrecacheDelegate* precache_delegate);
 
   virtual ~PrecacheFetcher();
 
@@ -161,6 +184,16 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
   // Adds up the response sizes.
   void UpdateStats(int64_t response_bytes, int64_t network_response_bytes);
 
+  // Retrieves the manifest info in DB thread. Manifest info for each of the

[bengr, 2016/08/11 18:49:15]

in -> on the

[Raj, 2016/08/12 19:04:21]

Done.

+  // hosts in |hosts_to_fetch|, is added to |hosts_info|.
+  void RetrieveManifestInfo(
+      std::unique_ptr<std::deque<std::string>> hosts_to_fetch,
+      std::deque<ManifestHostInfo>* hosts_info);
+
+  // Callback invoked when manifest info for all the top hosts is retrieved.

[bengr, 2016/08/11 18:49:14]

when -> when the

[Raj, 2016/08/12 19:04:21]

Done.

+  void OnManifestInfoRetrieved(
+      std::unique_ptr<std::deque<ManifestHostInfo>> manifests_info);
+
   // The request context used when fetching URLs.
   const scoped_refptr<net::URLRequestContextGetter> request_context_;
 
@@ -172,11 +205,15 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
   // default flag-specified prefix will be used.
   const std::string manifest_url_prefix_;
 
+  // PrecacheDatabase should be accessed in the DB thread.

[bengr, 2016/08/11 18:49:14]

in -> on

[Raj, 2016/08/12 19:04:21]

Done.

+  base::WeakPtr<PrecacheDatabase> precache_database_;
+  scoped_refptr<base::SingleThreadTaskRunner> db_task_runner_;
+
   // Non-owning pointer. Should not be NULL.
   PrecacheDelegate* precache_delegate_;
 
-  std::list<GURL> manifest_urls_to_fetch_;
-  std::list<GURL> resource_urls_to_fetch_;
+  std::unique_ptr<std::deque<ManifestHostInfo>> top_hosts_to_fetch_;
+  std::list<std::pair<GURL, std::string>> resources_to_fetch_;

[sclittle, 2016/08/11 22:52:36]

If you do chose to keep this member variable instead of just using the
|unfinished_work_| with a |resource_to_fetch_index_|, then please change this to
also be an std::deque instead of an std::list.

[Raj, 2016/08/12 19:04:21]

Done.

 
   FetcherPool<Fetcher> pool_;
 
@@ -214,6 +251,7 @@ class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
   // the specified URL using the specified request context.
   Fetcher(net::URLRequestContextGetter* request_context,
           const GURL& url,
+          const std::string& referrer,
           const base::Callback<void(const Fetcher&)>& callback,
           bool is_resource_request,
           size_t max_bytes);
@@ -228,7 +266,9 @@ class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
     return network_url_fetcher_.get();
   }
   const GURL& url() const { return url_; }
+  const std::string& referrer() const { return referrer_; }
   bool is_resource_request() const { return is_resource_request_; }
+  bool was_cached() const { return was_cached_; }
 
  private:
   enum class FetchStage { CACHE, NETWORK };
@@ -238,6 +278,7 @@ class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
 
   net::URLRequestContextGetter* const request_context_;
   const GURL url_;
+  const std::string referrer_;
   const base::Callback<void(const Fetcher&)> callback_;
   const bool is_resource_request_;
   const size_t max_bytes_;
@@ -248,6 +289,7 @@ class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
   std::unique_ptr<net::URLFetcher> network_url_fetcher_;
   int64_t response_bytes_;
   int64_t network_response_bytes_;
+  bool was_cached_;
 
   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 };