Send the list of used and unused resources for precache

components/precache/core/precache_fetcher.h

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
 #define COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
 
 #include <stdint.h>
 
-#include <algorithm>
-#include <list>
+#include <deque>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "base/callback.h"
 #include "base/macros.h"
+#include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/single_thread_task_runner.h"
 #include "base/time/time.h"
 #include "components/precache/core/fetcher_pool.h"
 #include "net/url_request/url_fetcher.h"
 #include "net/url_request/url_fetcher_delegate.h"
 #include "url/gurl.h"
 
 namespace base {
 class SingleThreadTaskRunner;
 }
 
 namespace net {
 class URLRequestContextGetter;
 }
 
 namespace precache {
 
 class PrecacheConfigurationSettings;
+class PrecacheDatabase;
 class PrecacheUnfinishedWork;
 
 // Visible for testing.
 extern const int kNoTracking;
 
+// Contains the information about manifest for a host.
+struct ManifestHostInfo {
+  ManifestHostInfo(int64_t manifest_id,
+                   const std::string& hostname,
+                   const std::string& used_url_hash,

[sclittle, 2016/08/16 21:05:05]

nit: since these hash strings can get pretty long, you could change all these
strings in the constructor to take in just "std::string"s instead of "const
std::string&"s then move them into the members.

[Raj, 2016/08/18 00:23:23]

hmm. I don't see any benefit.
The strings will get copied to the constructor arguments due to "std::string".
Without any improvement, using move() seems unnecessary.

+                   const std::string& unused_url_hash);
+  ManifestHostInfo(ManifestHostInfo&&);
+  ManifestHostInfo& operator=(ManifestHostInfo&&);
+
+  ~ManifestHostInfo();
+
+  int64_t manifest_id;
+  std::string hostname;
+  GURL manifest_url;
+  std::string used_url_hash;
+  std::string unused_url_hash;
+};
+
 // Public interface to code that fetches resources that the user is likely to
 // want to fetch in the future, putting them in the network stack disk cache.
 // Precaching is intended to be done when Chrome is not actively in use, likely
 // hours ahead of the time when the resources are actually needed.
 //
 // This class takes as input a prioritized list of URL domains that the user
 // commonly visits, referred to as starting hosts. This class interacts with a
 // server, sending it the list of starting hosts sequentially. For each starting
 // host, the server returns a manifest of resource URLs that are good candidates
 // for precaching. Every resource returned is fetched, and responses are cached
@@ skipping 42 matching lines @@
   };
 
   // Visible for testing.
   class Fetcher;
 
   static void RecordCompletionStatistics(
       const PrecacheUnfinishedWork& unfinished_work,
       size_t remaining_manifest_urls_to_fetch,
       size_t remaining_resource_urls_to_fetch);
 
-  // Constructs a new PrecacheFetcher. The |starting_hosts| parameter is a
+  // Constructs a new PrecacheFetcher. The |unfinished_work| contains the
   // prioritized list of hosts that the user commonly visits. These hosts are
   // used by a server side component to construct a list of resource URLs that
   // the user is likely to fetch. Takes ownership of |unfinished_work|.
-  PrecacheFetcher(net::URLRequestContextGetter* request_context,
-                  const GURL& config_url,
-                  const std::string& manifest_url_prefix,
-                  std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
-                  uint32_t experiment_id,
-                  PrecacheDelegate* precache_delegate);
+  // |precache_database| should be accessed on;y in |db_task_runner|.
+  PrecacheFetcher(
+      net::URLRequestContextGetter* request_context,
+      const GURL& config_url,
+      const std::string& manifest_url_prefix,
+      std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
+      uint32_t experiment_id,
+      const base::WeakPtr<PrecacheDatabase>& precache_database,
+      const scoped_refptr<base::SingleThreadTaskRunner>& db_task_runner,
+      PrecacheDelegate* precache_delegate);
 
   virtual ~PrecacheFetcher();
 
   // Starts fetching resources to precache. URLs are fetched sequentially. Can
   // be called from any thread. Start should only be called once on a
   // PrecacheFetcher instance.
   void Start();
 
   // Stops all precaching work. The PreacheFetcher should not be used after
   // calling this method.
@@ skipping 30 matching lines @@
   // resource URLs to fetch according to the URLs in the manifest. If the fetch
   // of a manifest fails, then it skips to the next manifest.
   void OnManifestFetchComplete(const Fetcher& source);
 
   // Called when a resource has been fetched.
   void OnResourceFetchComplete(const Fetcher& source);
 
   // Adds up the response sizes.
   void UpdateStats(int64_t response_bytes, int64_t network_response_bytes);
 
+  // Retrieves the manifest info on the DB thread. Manifest info for each of the
+  // hosts in |hosts_to_fetch|, is added to |hosts_info|.
+  void RetrieveManifestInfo(std::deque<std::string> hosts_to_fetch,
+                            std::deque<ManifestHostInfo>* hosts_info);
+
+  // Callback invoked when the manifest info for all the top hosts is retrieved.
+  void OnManifestInfoRetrieved(
+      std::unique_ptr<std::deque<ManifestHostInfo>> manifests_info);
+
   // The request context used when fetching URLs.
   const scoped_refptr<net::URLRequestContextGetter> request_context_;
 
   // The custom URL to use when fetching the config. If not provided, the
   // default flag-specified URL will be used.
   const GURL config_url_;
 
   // The custom URL prefix to use when fetching manifests. If not provided, the
   // default flag-specified prefix will be used.
   const std::string manifest_url_prefix_;
 
+  // PrecacheDatabase should be accessed on the DB thread.
+  base::WeakPtr<PrecacheDatabase> precache_database_;
+  scoped_refptr<base::SingleThreadTaskRunner> db_task_runner_;
+
   // Non-owning pointer. Should not be NULL.
   PrecacheDelegate* precache_delegate_;
 
-  std::list<GURL> manifest_urls_to_fetch_;
-  std::list<GURL> resource_urls_to_fetch_;
+  std::unique_ptr<std::deque<ManifestHostInfo>> top_hosts_to_fetch_;

[sclittle, 2016/08/16 21:05:05]

nit: could you remove the std::unique_ptr here and just std::move the deque into
this?

[Raj, 2016/08/18 00:23:23]

Done.

+  std::deque<std::pair<GURL, std::string>> resources_to_fetch_;
 
   FetcherPool<Fetcher> pool_;
 
   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work_;
 
   // The fieldtrial experiment ID.
   uint32_t experiment_id_;
 
   DISALLOW_COPY_AND_ASSIGN(PrecacheFetcher);
 };
@@ skipping 17 matching lines @@
 //
 // On completion it calls the given callback. This class cancels requests whose
 // responses are or will be larger than max_bytes. In such cases,
 // network_url_fetcher() will return nullptr.
 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
  public:
   // Construct a new Fetcher. This will create and start a new URLFetcher for
   // the specified URL using the specified request context.
   Fetcher(net::URLRequestContextGetter* request_context,
           const GURL& url,
+          const std::string& referrer,
           const base::Callback<void(const Fetcher&)>& callback,
           bool is_resource_request,
           size_t max_bytes);
   ~Fetcher() override;
   void OnURLFetchDownloadProgress(const net::URLFetcher* source,
                                   int64_t current,
                                   int64_t total) override;
   void OnURLFetchComplete(const net::URLFetcher* source) override;
   int64_t response_bytes() const { return response_bytes_; }
   int64_t network_response_bytes() const { return network_response_bytes_; }
   const net::URLFetcher* network_url_fetcher() const {
     return network_url_fetcher_.get();
   }
   const GURL& url() const { return url_; }
+  const std::string& referrer() const { return referrer_; }
   bool is_resource_request() const { return is_resource_request_; }
+  bool was_cached() const { return was_cached_; }
 
  private:
   enum class FetchStage { CACHE, NETWORK };
 
   void LoadFromCache();
   void LoadFromNetwork();
 
   net::URLRequestContextGetter* const request_context_;
   const GURL url_;
+  const std::string referrer_;
   const base::Callback<void(const Fetcher&)> callback_;
   const bool is_resource_request_;
   const size_t max_bytes_;
 
   FetchStage fetch_stage_;
   // The cache_url_fetcher_ is kept alive until Fetcher destruction for testing.
   std::unique_ptr<net::URLFetcher> cache_url_fetcher_;
   std::unique_ptr<net::URLFetcher> network_url_fetcher_;
   int64_t response_bytes_;
   int64_t network_response_bytes_;
+  bool was_cached_;
 
   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 };
 
 }  // namespace precache
 
 #endif  // COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_