Precache: Optionally rank resources-to-precache globally.

components/precache/core/precache_fetcher.h

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
 #define COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
 
 #include <stdint.h>
 
 #include <deque>
+#include <list>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "base/callback.h"
 #include "base/macros.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/weak_ptr.h"
 #include "base/single_thread_task_runner.h"
 #include "base/time/time.h"
@@ skipping 12 matching lines @@
 }
 
 namespace precache {
 
 class PrecacheConfigurationSettings;
 class PrecacheDatabase;
 class PrecacheUnfinishedWork;
 
 // Visible for testing.
 extern const int kNoTracking;
+extern const int kMaxParallelFetches;
 
-// Contains the information about manifest for a host.
+// Information about the manifest for a host.
 struct ManifestHostInfo {
   ManifestHostInfo(int64_t manifest_id,
                    const std::string& hostname,
+                   int64_t visits,
                    const std::string& used_url_hash,
                    const std::string& unused_url_hash);
+  ~ManifestHostInfo();
   ManifestHostInfo(ManifestHostInfo&&);
   ManifestHostInfo& operator=(ManifestHostInfo&&);
-
-  ~ManifestHostInfo();
+  // Copy constructor and assignment operator are implicitly deleted.
 
   int64_t manifest_id;
   std::string hostname;
   GURL manifest_url;
+  int64_t visits;
   std::string used_url_hash;
   std::string unused_url_hash;
 };
 
+// Information about a resource to be downloaded.
+struct ResourceInfo {
+  ResourceInfo(const GURL& url, const std::string& referrer, double weight);
+  ~ResourceInfo();
+  ResourceInfo(ResourceInfo&&);
+  ResourceInfo& operator=(ResourceInfo&&);
+  // Copy constructor and assignment operator are implicitly deleted.
+
+  GURL url;              // The resource being requested.

[bengr, 2016/10/14 21:52:19]

Nit: these comments aren't lined up.

[twifkak, 2016/10/14 22:41:45]

Believe it or not, that's what clang-format produces. I've cheated and removed
"An" so the last comment fits in 80 chars after being aligned.

+  std::string referrer;  // The host of the manifest requesting this resource.
+  double weight;  // An estimate of the expected utility of this resource.
+};
+
 // Public interface to code that fetches resources that the user is likely to
 // want to fetch in the future, putting them in the network stack disk cache.
 // Precaching is intended to be done when Chrome is not actively in use, likely
 // hours ahead of the time when the resources are actually needed.
 //
 // This class takes as input a prioritized list of URL domains that the user
 // commonly visits, referred to as starting hosts. This class interacts with a
 // server, sending it the list of starting hosts sequentially. For each starting
 // host, the server returns a manifest of resource URLs that are good candidates
 // for precaching. Every resource returned is fetched, and responses are cached
@@ skipping 31 matching lines @@
 //   std::unique_ptr<PrecacheFetcher> fetcher_;
 // };
 class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
  public:
   class PrecacheDelegate {
    public:
     // Called when the fetching of resources has finished, whether the resources
     // were fetched or not. If the PrecacheFetcher is destroyed before OnDone is
     // called, then precaching will be canceled and OnDone will not be called.
     virtual void OnDone() = 0;
-
   };
 
   // Visible for testing.
   class Fetcher;
 
   static void RecordCompletionStatistics(
       const PrecacheUnfinishedWork& unfinished_work,
       size_t remaining_manifest_urls_to_fetch,
       size_t remaining_resource_urls_to_fetch);
 
   static std::string GetResourceURLBase64HashForTesting(
       const std::vector<GURL>& urls);
 
   // Constructs a new PrecacheFetcher. The |unfinished_work| contains the
   // prioritized list of hosts that the user commonly visits. These hosts are
   // used by a server side component to construct a list of resource URLs that
   // the user is likely to fetch. Takes ownership of |unfinished_work|.
   // |precache_database| should be accessed only in |db_task_runner|.
   PrecacheFetcher(
       net::URLRequestContextGetter* request_context,
       const GURL& config_url,
       const std::string& manifest_url_prefix,
+      bool global_ranking,
       std::unique_ptr<PrecacheUnfinishedWork> unfinished_work,
       uint32_t experiment_id,
       const base::WeakPtr<PrecacheDatabase>& precache_database,
       const scoped_refptr<base::SingleThreadTaskRunner>& db_task_runner,
       PrecacheDelegate* precache_delegate);
 
   virtual ~PrecacheFetcher();
 
   // Starts fetching resources to precache. URLs are fetched sequentially. Can
   // be called from any thread. Start should only be called once on a
   // PrecacheFetcher instance.
   void Start();
 
   // Stops all precaching work. The PreacheFetcher should not be used after
   // calling this method.
   std::unique_ptr<PrecacheUnfinishedWork> CancelPrecaching();
 
  private:
   friend class PrecacheFetcherTest;
+  FRIEND_TEST_ALL_PREFIXES(PrecacheFetcherTest,
+                           GloballyRankResourcesAfterPauseResume);
   FRIEND_TEST_ALL_PREFIXES(PrecacheFetcherTest, FetcherPoolMaxLimitReached);
   FRIEND_TEST_ALL_PREFIXES(PrecacheFetcherTest,
                            CancelPrecachingAfterAllManifestFetch);
   FRIEND_TEST_ALL_PREFIXES(PrecacheFetcherTest, DailyQuota);
 
   // Notifies the precache delete that precaching is done, and report
   // completion statistics.
   void NotifyDone(size_t remaining_manifest_urls_to_fetch,
                   size_t remaining_resource_urls_to_fetch);
 
   // Fetches the next resource or manifest URL, if any remain. Fetching is done
   // sequentially and depth-first: all resources are fetched for a manifest
   // before the next manifest is fetched. This is done to limit the length of
   // the |resource_urls_to_fetch_| list, reducing the memory usage.
   void StartNextFetch();
 
-  void StartNextManifestFetch();
+  void StartNextManifestFetches();
   void StartNextResourceFetch();
 
   // Called when the precache configuration settings have been fetched.
   // Determines the list of manifest URLs to fetch according to the list of
   // |starting_hosts_| and information from the precache configuration settings.
   // If the fetch of the configuration settings fails, then precaching ends.
   void OnConfigFetchComplete(const Fetcher& source);
 
   // Constructs manifest URLs using a manifest URL prefix, and lists of hosts.
   void DetermineManifests();
 
   // Called when a precache manifest has been fetched. Builds the list of
   // resource URLs to fetch according to the URLs in the manifest. If the fetch
   // of a manifest fails, then it skips to the next manifest.
-  void OnManifestFetchComplete(const Fetcher& source);
+  void OnManifestFetchComplete(int64_t host_visits, const Fetcher& source);
 
   // Called when a resource has been fetched.
   void OnResourceFetchComplete(const Fetcher& source);
 
   // Adds up the response sizes.
   void UpdateStats(int64_t response_bytes, int64_t network_response_bytes);
 
   // Callback invoked when the manifest info for all the top hosts is retrieved.
   void OnManifestInfoRetrieved(std::deque<ManifestHostInfo> manifests_info);
 
   // Callback invoked when the quota is retrieved.
   void OnQuotaInfoRetrieved(const PrecacheQuota& quota);
 
   // The request context used when fetching URLs.
   const scoped_refptr<net::URLRequestContextGetter> request_context_;
 
   // The custom URL to use when fetching the config. If not provided, the
   // default flag-specified URL will be used.
   const GURL config_url_;
 
   // The custom URL prefix to use when fetching manifests. If not provided, the
   // default flag-specified prefix will be used.
   const std::string manifest_url_prefix_;
 
+  // Whether to order the resource fetches by the global ranking function.
+  bool global_ranking_;
+
   // PrecacheDatabase should be accessed on the DB thread.
   base::WeakPtr<PrecacheDatabase> precache_database_;
   scoped_refptr<base::SingleThreadTaskRunner> db_task_runner_;
 
   // Non-owning pointer. Should not be NULL.
   PrecacheDelegate* precache_delegate_;
 
+  // Top hosts for which manifests still need to be fetched (i.e. no Fetcher has
+  // been created yet).
   std::deque<ManifestHostInfo> top_hosts_to_fetch_;
-  std::deque<std::pair<GURL, std::string>> resources_to_fetch_;
+
+  // Top hosts for which manifests are currently being fetched.
+  std::list<ManifestHostInfo> top_hosts_fetching_;
+
+  // Resources to be fetched, in desired fetch order. Populated only after
+  // manifest fetching is complete.
+  std::deque<ResourceInfo> resources_to_fetch_;
+
+  // Resources currently being fetched, in the order requested.
+  std::list<ResourceInfo> resources_fetching_;
+
+  // Resources to be fetched, not yet ranked. Valid until manifest fetching is
+  // done, after which resources are sorted and places in resources_to_fetch_.
+  std::deque<ResourceInfo> resources_to_rank_;
 
   FetcherPool<Fetcher> pool_;
 
   std::unique_ptr<PrecacheUnfinishedWork> unfinished_work_;
 
   // Daily quota.
   PrecacheQuota quota_;
 
   // The fieldtrial experiment ID.
   uint32_t experiment_id_;
@@ skipping 67 matching lines @@
   int64_t response_bytes_;
   int64_t network_response_bytes_;
   bool was_cached_;
 
   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 };
 
 }  // namespace precache
 
 #endif  // COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_