Precache: Optionally rank resources-to-precache globally.

components/precache/core/precache_fetcher.h

Index: components/precache/core/precache_fetcher.h
diff --git a/components/precache/core/precache_fetcher.h b/components/precache/core/precache_fetcher.h
index 0f136c32b1e03359201dd6b4aa532f8144dbe1f4..1a5278b3e36181f4fefc34e34bbca164fc73bae3 100644
--- a/components/precache/core/precache_fetcher.h
+++ b/components/precache/core/precache_fetcher.h
@@ -8,6 +8,7 @@
 #include <stdint.h>
 
 #include <deque>
+#include <list>
 #include <memory>
 #include <string>
 #include <vector>
@@ -40,25 +41,41 @@ class PrecacheUnfinishedWork;
 
 // Visible for testing.
 extern const int kNoTracking;
+extern const int kMaxParallelFetches;
 
-// Contains the information about manifest for a host.
+// Information about the manifest for a host.
 struct ManifestHostInfo {
   ManifestHostInfo(int64_t manifest_id,
                    const std::string& hostname,
+                   int64_t visits,
                    const std::string& used_url_hash,
                    const std::string& unused_url_hash);
+  ~ManifestHostInfo();
   ManifestHostInfo(ManifestHostInfo&&);
   ManifestHostInfo& operator=(ManifestHostInfo&&);
-
-  ~ManifestHostInfo();
+  // Copy constructor and assignment operator are implicitly deleted.
 
   int64_t manifest_id;
   std::string hostname;
   GURL manifest_url;
+  int64_t visits;
   std::string used_url_hash;
   std::string unused_url_hash;
 };
 
+// Information about a resource to be downloaded.
+struct ResourceInfo {
+  ResourceInfo(const GURL& url, const std::string& referrer, double weight);
+  ~ResourceInfo();
+  ResourceInfo(ResourceInfo&&);
+  ResourceInfo& operator=(ResourceInfo&&);
+  // Copy constructor and assignment operator are implicitly deleted.
+
+  GURL url;              // The resource being requested.

[bengr, 2016/10/14 21:52:19]

Nit: these comments aren't lined up.

[twifkak, 2016/10/14 22:41:45]

Believe it or not, that's what clang-format produces. I've cheated and removed
"An" so the last comment fits in 80 chars after being aligned.

+  std::string referrer;  // The host of the manifest requesting this resource.
+  double weight;  // An estimate of the expected utility of this resource.
+};
+
 // Public interface to code that fetches resources that the user is likely to
 // want to fetch in the future, putting them in the network stack disk cache.
 // Precaching is intended to be done when Chrome is not actively in use, likely
@@ -110,7 +127,6 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
     // were fetched or not. If the PrecacheFetcher is destroyed before OnDone is
     // called, then precaching will be canceled and OnDone will not be called.
     virtual void OnDone() = 0;
-
   };
 
   // Visible for testing.
@@ -133,6 +149,7 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
       net::URLRequestContextGetter* request_context,
       const GURL& config_url,
       const std::string& manifest_url_prefix,
+      bool global_ranking,
       std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
       uint32_t experiment_id,
       const base::WeakPtr<PrecacheDatabase>& precache_database,
@@ -152,6 +169,8 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
 
  private:
   friend class PrecacheFetcherTest;
+  FRIEND_TEST_ALL_PREFIXES(PrecacheFetcherTest,
+                           GloballyRankResourcesAfterPauseResume);
   FRIEND_TEST_ALL_PREFIXES(PrecacheFetcherTest, FetcherPoolMaxLimitReached);
   FRIEND_TEST_ALL_PREFIXES(PrecacheFetcherTest,
                            CancelPrecachingAfterAllManifestFetch);
@@ -168,7 +187,7 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
   // the |resource_urls_to_fetch_| list, reducing the memory usage.
   void StartNextFetch();
 
-  void StartNextManifestFetch();
+  void StartNextManifestFetches();
   void StartNextResourceFetch();
 
   // Called when the precache configuration settings have been fetched.
@@ -183,7 +202,7 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
   // Called when a precache manifest has been fetched. Builds the list of
   // resource URLs to fetch according to the URLs in the manifest. If the fetch
   // of a manifest fails, then it skips to the next manifest.
-  void OnManifestFetchComplete(const Fetcher& source);
+  void OnManifestFetchComplete(int64_t host_visits, const Fetcher& source);
 
   // Called when a resource has been fetched.
   void OnResourceFetchComplete(const Fetcher& source);
@@ -208,6 +227,9 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
   // default flag-specified prefix will be used.
   const std::string manifest_url_prefix_;
 
+  // Whether to order the resource fetches by the global ranking function.
+  bool global_ranking_;
+
   // PrecacheDatabase should be accessed on the DB thread.
   base::WeakPtr<PrecacheDatabase> precache_database_;
   scoped_refptr<base::SingleThreadTaskRunner> db_task_runner_;
@@ -215,8 +237,23 @@ class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
   // Non-owning pointer. Should not be NULL.
   PrecacheDelegate* precache_delegate_;
 
+  // Top hosts for which manifests still need to be fetched (i.e. no Fetcher has
+  // been created yet).
   std::deque<ManifestHostInfo> top_hosts_to_fetch_;
-  std::deque<std::pair<GURL, std::string>> resources_to_fetch_;
+
+  // Top hosts for which manifests are currently being fetched.
+  std::list<ManifestHostInfo> top_hosts_fetching_;
+
+  // Resources to be fetched, in desired fetch order. Populated only after
+  // manifest fetching is complete.
+  std::deque<ResourceInfo> resources_to_fetch_;
+
+  // Resources currently being fetched, in the order requested.
+  std::list<ResourceInfo> resources_fetching_;
+
+  // Resources to be fetched, not yet ranked. Valid until manifest fetching is
+  // done, after which resources are sorted and places in resources_to_fetch_.
+  std::deque<ResourceInfo> resources_to_rank_;
 
   FetcherPool<Fetcher> pool_;