Add pause/resume functionality to precache

components/precache/core/precache_fetcher.h

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
 #define COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_
 
 #include <stdint.h>
 
 #include <algorithm>
 #include <list>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "base/callback.h"
 #include "base/macros.h"
-#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "base/single_thread_task_runner.h"
 #include "base/time/time.h"
 #include "components/precache/core/fetcher_pool.h"
 #include "net/url_request/url_fetcher.h"
 #include "net/url_request/url_fetcher_delegate.h"
 #include "url/gurl.h"
 
+namespace base {
+class SingleThreadTaskRunner;
+}
+
 namespace net {
 class URLRequestContextGetter;
 }
 
 namespace precache {
 
 class PrecacheConfigurationSettings;
+class PrecacheDatabase;
+class UnfinishedWork;
 
 // Visible for testing.
 extern const int kNoTracking;
 
 // Public interface to code that fetches resources that the user is likely to
 // want to fetch in the future, putting them in the network stack disk cache.
 // Precaching is intended to be done when Chrome is not actively in use, likely
 // hours ahead of the time when the resources are actually needed.
 //
 // This class takes as input a prioritized list of URL domains that the user
@@ skipping 10 matching lines @@
 // manifests are fetched from. These can be set by using command line switches
 // or by providing default values.
 //
 // Sample interaction:
 //
 // class MyPrecacheFetcherDelegate : public PrecacheFetcher::PrecacheDelegate {
 //  public:
 //   void PrecacheResourcesForTopURLs(
 //       net::URLRequestContextGetter* request_context,
 //       const std::list<GURL>& top_urls) {
-//     fetcher_.reset(new PrecacheFetcher(request_context, top_urls, this));
+//     fetcher_.reset(new PrecacheFetcher(...));
+//     fetcher_->Init();
+//   }
+//
+//   virtual void OnInitDone() {
 //     fetcher_->Start();
 //   }
 //
 //   virtual void OnDone() {
-//     // Do something when precaching is done.
+//     // Do something when precaching is done, e.g.:
+//     fetcher_->Shutdown();
+//   }
+//
+//   virtual void OnShutdownDone() {
+//     fetcher_.reset();
 //   }
 //
 //  private:
 //   std::unique_ptr<PrecacheFetcher> fetcher_;
 // };
-class PrecacheFetcher {
+class PrecacheFetcher : public base::SupportsWeakPtr<PrecacheFetcher> {
  public:
   class PrecacheDelegate {
    public:
     // Called when the fetching of resources has finished, whether the resources
     // were fetched or not. If the PrecacheFetcher is destroyed before OnDone is
     // called, then precaching will be canceled and OnDone will not be called.
     virtual void OnDone() = 0;
+
+    // Called when |PrecacheFetcher| has finished initializing and can be
+    // started.
+    virtual void OnInitDone() = 0;
+
+    // Called after |PrecacheFetcher| has shut down and can be destroyed.
+    virtual void OnShutdownDone() = 0;
   };
 
   // Visible for testing.
   class Fetcher;
 
   // Constructs a new PrecacheFetcher. The |starting_hosts| parameter is a
   // prioritized list of hosts that the user commonly visits. These hosts are
   // used by a server side component to construct a list of resource URLs that
   // the user is likely to fetch.
-  PrecacheFetcher(const std::vector<std::string>& starting_hosts,
-                  net::URLRequestContextGetter* request_context,
-                  const GURL& config_url,
-                  const std::string& manifest_url_prefix,
-                  PrecacheDelegate* precache_delegate);
+  PrecacheFetcher(
+      const std::vector<std::string>& starting_hosts,
+      net::URLRequestContextGetter* request_context,
+      const GURL& config_url,
+      const std::string& manifest_url_prefix,
+      PrecacheDelegate* precache_delegate,
+      base::WeakPtr<PrecacheDatabase> precache_database,

[sclittle, 2016/05/10 00:01:27]

It seems odd for the PrecacheFetcher to have knowledge of the database. AIUI,
ideally PrecacheFetcher would just fetch URLs that it's told to fetch. What if
the classes were reorganized like this:

- Make UnfinishedWork contain almost all the state of PrecacheFetcher, including
the starting_hosts, the config_url, and the ConfigurationSettings itself if it's
already been fetched. That way, the UnfinishedWork can represent the state right
before precaching is started.
- Make PrecacheFetcher always take in an UnfinishedWork and develop its state
around that, where UnfinishedWork could represent the state before precaching
was even started.

Then, PrecacheManager could use PrecacheDatabase to check if any partially done
UnfinishedWork was saved and use that if it exists, otherwise it could just
create a new UnfinishedWork representing the state before any precaching is
done, e.g. only the starting_hosts and stuff are set.

[bengr, 2016/05/19 01:25:44]

Done.

+      scoped_refptr<base::SingleThreadTaskRunner> database_task_runner);
 
   virtual ~PrecacheFetcher();
 
+  void Init(const base::TimeDelta& max_age);
+
   // Starts fetching resources to precache. URLs are fetched sequentially. Can
   // be called from any thread. Start should only be called once on a
   // PrecacheFetcher instance.
   void Start();
 
+  void Shutdown();
+
+  // Used for testing to put the fetcher into a state where it has work to do.
+  void AssignWorkForTest(const std::list<GURL>& manifests,
+                         const std::list<GURL>& resources,
+                         size_t total_response_bytes,
+                         size_t network_response_bytes,
+                         int num_manifest_urls_to_fetch,

[sclittle, 2016/05/10 00:01:26]

nit: Why an int? Why not a size_t?

[bengr, 2016/05/19 01:25:44]

Done.

+                         const base::TimeTicks& start_time);
+
  private:
+  void OnInitDone(scoped_refptr<UnfinishedWork> unfinished_work);
+  void OnShutdownDone();
+
   // Fetches the next resource or manifest URL, if any remain. Fetching is done
   // sequentially and depth-first: all resources are fetched for a manifest
   // before the next manifest is fetched. This is done to limit the length of
   // the |resource_urls_to_fetch_| list, reducing the memory usage.
   void StartNextFetch();
 
   void StartNextManifestFetch();
   void StartNextResourceFetch();
 
   // Called when the precache configuration settings have been fetched.
@@ skipping 26 matching lines @@
 
   // The custom URL prefix to use when fetching manifests. If not provided, the
   // default flag-specified prefix will be used.
   const std::string manifest_url_prefix_;
 
   // Non-owning pointer. Should not be NULL.
   PrecacheDelegate* precache_delegate_;
 
   std::unique_ptr<PrecacheConfigurationSettings> config_;
 
+  // The database for storing unfinished work. Should only be used on
+  // the DB thread.
+  base::WeakPtr<PrecacheDatabase> precache_database_;
+
+  const scoped_refptr<base::SingleThreadTaskRunner> database_task_runner_;
+
   // Tally of the total number of bytes contained in URL fetches, including
   // config, manifests, and resources. This the number of bytes as they would be
   // compressed over the network.
   size_t total_response_bytes_;

[sclittle, 2016/05/10 00:01:26]

nit, maybe for a followup CL: This should probably actually be an int64_t, not a
size_t. Same with |network_response_bytes_| below.

[bengr, 2016/05/19 01:25:44]

Done.

 
   // Tally of the total number of bytes received over the network from URL
   // fetches (the same ones as in total_response_bytes_). This includes response
   // headers and intermediate responses such as 30xs.
   size_t network_response_bytes_;
 
   // Time when the prefetch was started.
   base::TimeTicks start_time_;
 
   int num_manifest_urls_to_fetch_;
   std::list<GURL> manifest_urls_to_fetch_;

[sclittle, 2016/05/10 00:01:26]

nit, maybe for a followup CL: This was originally my fault when I first wrote
this class years ago, but an std::deque (or even just use an std::queue) would
almost certainly be more efficient than an std::list for this, since we only
need to add or remove elements from the beginning and end. Do you mind fixing
this?

[bengr, 2016/05/19 01:25:44]

Acknowledged.

   std::list<GURL> resource_urls_to_fetch_;
 
   FetcherPool<Fetcher> pool_;
 
   DISALLOW_COPY_AND_ASSIGN(PrecacheFetcher);
 };
 
 // Class that fetches a URL, and runs the specified callback when the fetch is
 // complete. This class exists so that a different method can be run in
 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
@@ skipping 52 matching lines @@
   std::unique_ptr<net::URLFetcher> network_url_fetcher_;
   int64_t response_bytes_;
   int64_t network_response_bytes_;
 
   DISALLOW_COPY_AND_ASSIGN(Fetcher);
 };
 
 }  // namespace precache
 
 #endif  // COMPONENTS_PRECACHE_CORE_PRECACHE_FETCHER_H_