Send the list of used and unused resources for precache

components/precache/core/precache_fetcher.h

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
 #define COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
 
 #include <stdint.h>
 
 #include <algorithm>

[sclittle, 2016/08/11 22:52:36]

Remove this include from the header file and put it in the .cc file.

[Raj, 2016/08/12 19:04:21]

Done.

-#include <list>
+#include <deque>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "base/callback.h"
 #include "base/macros.h"
+#include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/single_thread_task_runner.h"
 #include "base/time/time.h"
 #include "components/precache/core/fetcher_pool.h"
 #include "net/url_request/url_fetcher.h"
 #include "net/url_request/url_fetcher_delegate.h"
 #include "url/gurl.h"
 
 namespace base {
 class SingleThreadTaskRunner;
 }
 
 namespace net {
 class URLRequestContextGetter;
 }
 
 namespace precache {
 
 class PrecacheConfigurationSettings;
+class PrecacheDatabase;
 class PrecacheUnfinishedWork;
 
 // Visible for testing.
 extern const int kNoTracking;
 
+// Contains the information about manifest for a host.
+struct ManifestHostInfo {
+  ManifestHostInfo(int64_t manifest_id,
+                   const std::string& hostname,
+                   const std::string& used_url_hash,
+                   const std::string& unused_url_hash);
+  ManifestHostInfo(const ManifestHostInfo& other);

[sclittle, 2016/08/11 22:52:35]

Is it possible to make this struct movable as well? Doesn't have to be too
complicated - just also add:

ManifestHostInfo(ManifestHostInfo&&) = default;
ManifestHostInfo& operator=(ManifestHostInfo&&) = default;

(You'll probably want to just have the declarations here in the .h and the
=default definitions in the .cc like you do here for the copy constructor.)

That'll greatly reduce the number of string copies performed when dealing with
these objects, since it'll just be copying the |manifest_url| then and moving
everything else.

[Raj, 2016/08/12 19:04:21]

Nice. Done.

+
+  ~ManifestHostInfo();
+
+  const int64_t manifest_id;

[sclittle, 2016/08/11 22:52:36]

Can you change these to not be const? You can just pass around a "const
ManifestHostInfo&" if you want them not to be modified.

[Raj, 2016/08/12 19:04:21]

Done.

+  const std::string hostname;
+  GURL manifest_url;
+  const std::string used_url_hash;
+  const std::string unused_url_hash;
+};
+
 // Public interface to code that fetches resources that the user is likely to
 // want to fetch in the future, putting them in the network stack disk cache.
 // Precaching is intended to be done when Chrome is not actively in use, likely
 // hours ahead of the time when the resources are actually needed.
 //
 // This class takes as input a prioritized list of URL domains that the user
 // commonly visits, referred to as starting hosts. This class interacts with a
 // server, sending it the list of starting hosts sequentially. For each starting
 // host, the server returns a manifest of resource URLs that are good candidates
 // for precaching. Every resource returned is fetched, and responses are cached
@@ skipping 42 matching lines @@
   };
 
   // Visible for testing.
   class Fetcher;
 
   static void RecordCompletionStatistics(
       const PrecacheUnfinishedWork& unfinished_work,
       size_t remaining_manifest_urls_to_fetch,
       size_t remaining_resource_urls_to_fetch);
 
-  // Constructs a new PrecacheFetcher. The |starting_hosts| parameter is a
+  // Constructs a new PrecacheFetcher. The |unfinished_work| contains the
   // prioritized list of hosts that the user commonly visits. These hosts are
   // used by a server side component to construct a list of resource URLs that
   // the user is likely to fetch. Takes ownership of |unfinished_work|.
-  PrecacheFetcher(net::URLRequestContextGetter* request_context,
-                  const GURL& config_url,
-                  const std::string& manifest_url_prefix,
-                  std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
-                  uint32_t experiment_id,
-                  PrecacheDelegate* precache_delegate);
+  // |precache_database| should be accessed on;y in |db_task_runner|.
+  PrecacheFetcher(
+      net::URLRequestContextGetter* request_context,
+      const GURL& config_url,
+      const std::string& manifest_url_prefix,
+      std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
+      uint32_t experiment_id,
+      base::WeakPtr<PrecacheDatabase> precache_database,
+      const scoped_refptr<base::SingleThreadTaskRunner>& db_task_runner,
+      PrecacheDelegate* precache_delegate);
 
   virtual ~PrecacheFetcher();
 
   // Starts fetching resources to precache. URLs are fetched sequentially. Can
   // be called from any thread. Start should only be called once on a
   // PrecacheFetcher instance.
   void Start();
 
   // Stops all precaching work. The PreacheFetcher should not be used after
   // calling this method.
@@ skipping 30 matching lines @@
   // resource URLs to fetch according to the URLs in the manifest. If the fetch
   // of a manifest fails, then it skips to the next manifest.
   void OnManifestFetchComplete(const Fetcher& source);
 
   // Called when a resource has been fetched.
   void OnResourceFetchComplete(const Fetcher& source);
 
   // Adds up the response sizes.
   void UpdateStats(int64_t response_bytes, int64_t network_response_bytes);
 
+  // Retrieves the manifest info in DB thread. Manifest info for each of the

[bengr, 2016/08/11 18:49:15]

in -> on the

[Raj, 2016/08/12 19:04:21]

Done.

+  // hosts in |hosts_to_fetch|, is added to |hosts_info|.
+  void RetrieveManifestInfo(
+      std::unique_ptr<std::deque<std::string>> hosts_to_fetch,
+      std::deque<ManifestHostInfo>* hosts_info);
+
+  // Callback invoked when manifest info for all the top hosts is retrieved.

[bengr, 2016/08/11 18:49:14]

when -> when the

[Raj, 2016/08/12 19:04:21]

Done.

+  void OnManifestInfoRetrieved(
+      std::unique_ptr<std::deque<ManifestHostInfo>> manifests_info);
+
   // The request context used when fetching URLs.
   const scoped_refptr<net::URLRequestContextGetter> request_context_;
 
   // The custom URL to use when fetching the config. If not provided, the
   // default flag-specified URL will be used.
   const GURL config_url_;
 
   // The custom URL prefix to use when fetching manifests. If not provided, the
   // default flag-specified prefix will be used.
   const std::string manifest_url_prefix_;
 
+  // PrecacheDatabase should be accessed in the DB thread.

[bengr, 2016/08/11 18:49:14]

in -> on

[Raj, 2016/08/12 19:04:21]

Done.

+  base::WeakPtr<PrecacheDatabase> precache_database_;
+  scoped_refptr<base::SingleThreadTaskRunner> db_task_runner_;
+
   // Non-owning pointer. Should not be NULL.
   PrecacheDelegate* precache_delegate_;
 
-  std::list<GURL> manifest_urls_to_fetch_;
-  std::list<GURL> resource_urls_to_fetch_;
+  std::unique_ptr<std::deque<ManifestHostInfo>> top_hosts_to_fetch_;
+  std::list<std::pair<GURL, std::string>> resources_to_fetch_;

[sclittle, 2016/08/11 22:52:36]

If you do chose to keep this member variable instead of just using the
|unfinished_work_| with a |resource_to_fetch_index_|, then please change this to
also be an std::deque instead of an std::list.

[Raj, 2016/08/12 19:04:21]

Done.

 
   FetcherPool<Fetcher> pool_;
 
   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work_;
 
   // The fieldtrial experiment ID.
   uint32_t experiment_id_;
 
   DISALLOW_COPY_AND_ASSIGN(PrecacheFetcher);
 };
@@ skipping 17 matching lines @@
 //
 // On completion it calls the given callback. This class cancels requests whose
 // responses are or will be larger than max_bytes. In such cases,
 // network_url_fetcher() will return nullptr.
 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
  public:
   // Construct a new Fetcher. This will create and start a new URLFetcher for
   // the specified URL using the specified request context.
   Fetcher(net::URLRequestContextGetter* request_context,
           const GURL& url,
+          const std::string& referrer,
           const base::Callback<void(const Fetcher&)>& callback,
           bool is_resource_request,
           size_t max_bytes);
   ~Fetcher() override;
   void OnURLFetchDownloadProgress(const net::URLFetcher* source,
                                   int64_t current,
                                   int64_t total) override;
   void OnURLFetchComplete(const net::URLFetcher* source) override;
   int64_t response_bytes() const { return response_bytes_; }
   int64_t network_response_bytes() const { return network_response_bytes_; }
   const net::URLFetcher* network_url_fetcher() const {
     return network_url_fetcher_.get();
   }
   const GURL& url() const { return url_; }
+  const std::string& referrer() const { return referrer_; }
   bool is_resource_request() const { return is_resource_request_; }
+  bool was_cached() const { return was_cached_; }
 
  private:
   enum class FetchStage { CACHE, NETWORK };
 
   void LoadFromCache();
   void LoadFromNetwork();
 
   net::URLRequestContextGetter* const request_context_;
   const GURL url_;
+  const std::string referrer_;
   const base::Callback<void(const Fetcher&)> callback_;
   const bool is_resource_request_;
   const size_t max_bytes_;
 
   FetchStage fetch_stage_;
   // The cache_url_fetcher_ is kept alive until Fetcher destruction for testing.
   std::unique_ptr<net::URLFetcher> cache_url_fetcher_;
   std::unique_ptr<net::URLFetcher> network_url_fetcher_;
   int64_t response_bytes_;
   int64_t network_response_bytes_;
+  bool was_cached_;
 
   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 };
 
 }  // namespace precache
 
 #endif  // COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_