Modifying prefetch to account for multi-profile.

chrome/browser/net/predictor.h

Index: chrome/browser/net/predictor.h
===================================================================
--- chrome/browser/net/predictor.h	(revision 96503)
+++ chrome/browser/net/predictor.h	(working copy)
@@ -27,7 +27,7 @@
 #include <vector>
 
 #include "base/gtest_prod_util.h"
-#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
 #include "chrome/browser/net/url_info.h"
 #include "chrome/browser/net/referrer.h"
 #include "chrome/common/net/predictor_common.h"
@@ -37,10 +37,18 @@
 class ListValue;
 }
 
+namespace base {
+class WaitableEvent;
+}
+
 namespace net {
 class HostResolver;
 }  // namespace net
 
+class IOThread;
+class PrefService;
+class Profile;
+
 namespace chrome_browser_net {
 
 typedef chrome_common_net::UrlList UrlList;
@@ -49,7 +57,7 @@
 
 // Note that Predictor is not thread safe, and must only be called from
 // the IO thread. Failure to do so will result in a DCHECK at runtime.

[willchan no longer on Chromium, 2011/08/12 21:51:47]

You should update this comment. You should have something like this:

Predictor is constructed during Profile construction, but is destroyed on the IO
thread when ProfileIOData goes away. All of its core state and functionality
happens on the IO thread. The only UI methods are initialization / shutdown
related, or convenience methods that internally forward calls to the IO thread.

[rpetterson, 2011/08/13 00:55:17]

Done.

-class Predictor : public base::RefCountedThreadSafe<Predictor> {
+class Predictor {
  public:
   // A version number for prefs that are saved. This should be incremented when
   // we change the format so that we discard old data.
@@ -57,53 +65,74 @@
 
   // |max_concurrent| specifies how many concurrent (parallel) prefetches will
   // be performed. Host lookups will be issued through |host_resolver|.
-  Predictor(net::HostResolver* host_resolver,
-            base::TimeDelta max_queue_delay_ms, size_t max_concurrent,
-            bool preconnect_enabled);
+  Predictor();
 
+  virtual ~Predictor();
+
+  // ------------- Start UI thread methods.
+
+  virtual void InitNetworkPredictor(PrefService* user_prefs);
+
+  // The Omnibox has proposed a given url to the user, and if it is a search
+  // URL, then it also indicates that this is preconnectable (i.e., we could
+  // preconnect to the search server).
+  virtual void AnticipateOmniboxUrl(const GURL& url, bool preconnectable);
+
+  // Preconnect a URL and all of its subresource domains.
+  virtual void PreconnectUrlAndSubresources(const GURL& url);
+
+  // Instigate pre-connection to any URLs, or pre-resolution of related host,
+  // that we predict will be needed after this navigation (typically
+  // more-embedded resources on a page).  This method will actually post a task
+  // to do the actual work, so as not to jump ahead of the frame navigation that
+  // instigated this activity.
+  virtual void PredictFrameSubresources(const GURL& url);
+
+  static UrlList GetPredictedUrlListAtStartup(PrefService* user_prefs,
+                                              PrefService* local_state);
+
+  static void set_max_queueing_delay(int max_queueing_delay_ms);
+
+  static void set_max_parallel_resolves(size_t max_parallel_resolves);
+
+  static void RegisterUserPrefs(PrefService* user_prefs);
+
+  virtual void ShutdownOnUIThread(PrefService* user_prefs);
+
+  // ------------- End UI thread methods.
+
+  // ------------- Start IO thread methods.
+
   // Cancel pending requests and prevent new ones from being made.
-  void Shutdown();
+  virtual void Shutdown();
 
   // In some circumstances, for privacy reasons, all results should be
   // discarded.  This method gracefully handles that activity.
   // Destroy all our internal state, which shows what names we've looked up, and
   // how long each has taken, etc. etc.  We also destroy records of suggesses
   // (cache hits etc.).
-  void DiscardAllResults();
+  virtual void DiscardAllResults();
 
   // Add hostname(s) to the queue for processing.
-  void ResolveList(const UrlList& urls,
-                   UrlInfo::ResolutionMotivation motivation);
-  void Resolve(const GURL& url,
-               UrlInfo::ResolutionMotivation motivation);
+  virtual void ResolveList(const UrlList& urls,
+                           UrlInfo::ResolutionMotivation motivation);
 
-  // Instigate pre-connection to any URLs, or pre-resolution of related host,
-  // that we predict will be needed after this navigation (typically
-  // more-embedded resources on a page).  This method will actually post a task
-  // to do the actual work, so as not to jump ahead of the frame navigation that
-  // instigated this activity.
-  void PredictFrameSubresources(const GURL& url);
+  virtual void Resolve(const GURL& url,
+                       UrlInfo::ResolutionMotivation motivation);
 
-  // The Omnibox has proposed a given url to the user, and if it is a search
-  // URL, then it also indicates that this is preconnectable (i.e., we could
-  // preconnect to the search server).
-  void AnticipateOmniboxUrl(const GURL& url, bool preconnectable);
-
-  // Preconnect a URL and all of its subresource domains.
-  void PreconnectUrlAndSubresources(const GURL& url);
-
   // Record details of a navigation so that we can preresolve the host name
   // ahead of time the next time the users navigates to the indicated host.
   // Should only be called when urls are distinct, and they should already be
   // canonicalized to not have a path.
-  void LearnFromNavigation(const GURL& referring_url, const GURL& target_url);
+  virtual void LearnFromNavigation(const GURL& referring_url,
+                                   const GURL& target_url);
 
   // Dump HTML table containing list of referrers for about:dns.
-  void GetHtmlReferrerLists(std::string* output);
+  virtual void GetHtmlReferrerLists(std::string* output);
 
   // Dump the list of currently known referrer domains and related prefetchable
   // domains.
-  void GetHtmlInfo(std::string* output);
+  virtual void GetHtmlInfo(std::string* output);
 
   // Discards any referrer for which all the suggested host names are currently
   // annotated with negligible expected-use.  Scales down (diminishes) the
@@ -111,32 +140,106 @@
   // factor each time we trim (moving the referrer closer to being discarded in
   // a future call).
   // The task is performed synchronously and completes before returing.
-  void TrimReferrersNow();
+  virtual void TrimReferrersNow();
 
   // Construct a ListValue object that contains all the data in the referrers_
   // so that it can be persisted in a pref.
-  void SerializeReferrers(base::ListValue* referral_list);
+  virtual void SerializeReferrers(base::ListValue* referral_list);
 
   // Process a ListValue that contains all the data from a previous reference
   // list, as constructed by SerializeReferrers(), and add all the identified
   // values into the current referrer list.
-  void DeserializeReferrers(const base::ListValue& referral_list);
+  virtual void DeserializeReferrers(const base::ListValue& referral_list);
 
-  void DeserializeReferrersThenDelete(base::ListValue* referral_list);
+  virtual void DeserializeReferrersThenDelete(base::ListValue* referral_list);
 
-  // For unit test code only.
-  size_t max_concurrent_dns_lookups() const {
-    return max_concurrent_dns_lookups_;
-  }
+  virtual void DiscardInitialNavigationHistory();
 
-  // Flag setting to use preconnection instead of just DNS pre-fetching.
-  bool preconnect_enabled() const { return preconnect_enabled_; }
+  virtual void FinalizeInitializationOnIOThread(
+      const std::vector<GURL>& urls_to_prefetch,
+      base::ListValue* referral_list,
+      IOThread* io_thread);
 
+  // During startup, we learn what the first N urls visited are, and then
+  // resolve the associated hosts ASAP during our next startup.
+  virtual void LearnAboutInitialNavigation(const GURL& url);
+
+  // Renderer bundles up list and sends to this browser API via IPC.
+  // TODO(jar): Use UrlList instead to include port and scheme.
+  virtual void DnsPrefetchList(const NameList& hostnames);
+
+  // May be called from either the IO or UI thread and will PostTask
+  // to the IO thread if necessary.
+  virtual void Predictor::DnsPrefetchMotivatedList(
+    const UrlList& urls,
+    UrlInfo::ResolutionMotivation motivation);
+
+  // May be called from either the IO or UI thread and will PostTask
+  // to the IO thread if necessary.
+  virtual void SaveStateForNextStartupAndTrim(PrefService* prefs);
+
+  virtual void SaveDnsPrefetchStateForNextStartupAndTrim(
+      base::ListValue* startup_list,
+      base::ListValue* referral_list,
+      base::WaitableEvent* completion);
+
+  // May be called from either the IO or UI thread and will PostTask
+  // to the IO thread if necessary.
+  virtual void EnablePredictor(bool enable);
+
+  virtual void EnablePredictorOnIOThread(bool enable);
+
+  // ------------- End IO thread methods.
+
+  // The following methods may be called on either the IO or UI threads.
+
   // Put URL in canonical form, including a scheme, host, and port.
   // Returns GURL::EmptyGURL() if the scheme is not http/https or if the url
   // cannot be otherwise canonicalized.
   static GURL CanonicalizeUrl(const GURL& url);
 
+  // Used for testing.
+  void SetHostResolver(net::HostResolver* host_resolver) {
+    host_resolver_ = host_resolver;
+  }
+  // Used for testing.
+  size_t max_concurrent_dns_lookups() const {
+    return max_concurrent_dns_lookups_;
+  }
+
+  // Flag setting to use preconnection instead of just DNS pre-fetching.
+  bool preconnect_enabled() const {
+    return preconnect_enabled_;
+  }
+
+  // Flag setting for whether we are prefetching dns lookups.
+  bool predictor_enabled() const {
+    return predictor_enabled_;
+  }
+
+  // Given that the underlying Chromium resolver defaults to a total maximum of

[willchan no longer on Chromium, 2011/08/12 21:17:29]

constants go above methods in class declaration order:
http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml?showone=Declar...

[rpetterson, 2011/08/13 00:55:17]

Done.

+  // 8 paralell resolutions, we will avoid any chance of starving navigational
+  // resolutions by limiting the number of paralell speculative resolutions.
+  // This is used in the field trials and testing.
+  // TODO(jar): Move this limitation into the resolver.
+  static const size_t kMaxSpeculativeParallelResolves;
+
+  // To control the congestion avoidance system, we need an estimate of how
+  // many speculative requests may arrive at once.  Since we currently only
+  // keep 8 subresource names for each frame, we'll use that as our basis.
+  // Note that when scanning search results lists, we might actually get 10 at
+  // a time, and wikipedia can often supply (during a page scan) upwards of 50.
+  // In those odd cases, we may discard some of the later speculative requests
+  // mistakenly assuming that the resolutions took too long.
+  static const int kTypicalSpeculativeGroupSize;
+
+// The next constant specifies an amount of queueing delay that is
+// "too large," and indicative of problems with resolutions (perhaps due to
+// an overloaded router, or such).  When we exceed this delay, congestion
+// avoidance will kick in and all speculations in the queue will be discarded.
+  static const int kMaxSpeculativeResolveQueueDelayMs;
+
+
  private:
   friend class base::RefCountedThreadSafe<Predictor>;

[willchan no longer on Chromium, 2011/08/12 21:17:29]

Remove this

[rpetterson, 2011/08/13 00:55:17]

Done.

   FRIEND_TEST_ALL_PREFIXES(PredictorTest, BenefitLookupTest);
@@ -178,6 +281,36 @@
   DISALLOW_COPY_AND_ASSIGN(HostNameQueue);
   };
 
+  // The InitialObserver monitors navigations made by the network stack. This
+  // is only used to identify startup time resolutions (for re-resolution
+  // during our next process startup).
+  // TODO(jar): Consider preconnecting at startup, which may be faster than
+  // waiting for render process to start and request a connection.
+  class InitialObserver {
+   public:
+    // Recording of when we observed each navigation.
+    typedef std::map<GURL, base::TimeTicks> FirstNavigations;
+
+    // Potentially add a new URL to our startup list.
+    void Append(const GURL& url, Predictor* predictor);
+
+    // Get an HTML version of our current planned first_navigations_.
+    void GetFirstResolutionsHtml(std::string* output);
+
+    // Persist the current first_navigations_ for storage in a list.
+    void GetInitialDnsResolutionList(base::ListValue* startup_list);
+
+    // Discards all initial loading history.
+    void DiscardInitialNavigationHistory() { first_navigations_.clear(); }
+
+   private:
+    // List of the first N URL resolutions observed in this run.
+    FirstNavigations first_navigations_;
+
+    // The number of URLs we'll save for pre-resolving at next startup.
+    static const size_t kStartupResolutionCount = 10;
+  };
+
   // A map that is keyed with the host/port that we've learned were the cause
   // of loading additional URLs.  The list of additional targets is held
   // in a Referrer instance, which is a value in this map.
@@ -206,13 +339,6 @@
   // Number of referring URLs processed in an incremental trimming.
   static const size_t kUrlsTrimmedPerIncrement;
 
-  ~Predictor();
-
-  // Perform actual resolution or preconnection to subresources now.  This is
-  // an internal worker method that is reached via a post task from
-  // PredictFrameSubresources().
-  void PrepareFrameSubresources(const GURL& url);
-
   // Only for testing. Returns true if hostname has been successfully resolved
   // (name found).
   bool WasFound(const GURL& url) const {
@@ -232,6 +358,13 @@
   // Only for testing;
   size_t peak_pending_lookups() const { return peak_pending_lookups_; }
 
+  // ------------- Start IO thread methods.
+
+  // Perform actual resolution or preconnection to subresources now.  This is
+  // an internal worker method that is reached via a post task from
+  // PredictFrameSubresources().
+  void PrepareFrameSubresources(const GURL& url);
+
   // Access method for use by async lookup request to pass resolution result.
   void OnLookupFinished(LookupRequest* request, const GURL& url, bool found);
 
@@ -277,6 +410,14 @@
   // continue with them shortly (i.e., it yeilds and continues).
   void IncrementalTrimReferrers(bool trim_all_now);
 
+  // ------------- End IO thread methods.
+
+  scoped_ptr<InitialObserver> initial_observer_;
+
+  // Status of speculative DNS resolution and speculative TCP/IP connection
+  // feature.
+  bool predictor_enabled_;
+
   // work_queue_ holds a list of names we need to look up.
   HostNameQueue work_queue_;
 
@@ -302,7 +443,7 @@
   const base::TimeDelta max_dns_queue_delay_;
 
   // The host resolver we warm DNS entries for.
-  net::HostResolver* const host_resolver_;
+  net::HostResolver* host_resolver_;
 
   // Are we currently using preconnection, rather than just DNS resolution, for
   // subresources and omni-box search URLs.
@@ -334,7 +475,7 @@
   // A time after which we need to do more trimming of referrers.
   base::TimeTicks next_trim_time_;
 
-  ScopedRunnableMethodFactory<Predictor> trim_task_factory_;
+  scoped_ptr<ScopedRunnableMethodFactory<Predictor> > trim_task_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(Predictor);
 };