Modifying prefetch to account for multi-profile.

chrome/browser/net/predictor.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // A Predictor object is instantiated once in the browser process, and manages
 // both preresolution of hostnames, as well as TCP/IP preconnection to expected
 // subresources.
 // Most hostname lists are provided by the renderer processes, and include URLs
 // that *might* be used in the near future by the browsing user.  One goal of
 // this class is to cause the underlying DNS structure to lookup a hostname
 // before it is really needed, and hence reduce latency in the standard lookup
 // paths.
 // Subresource relationships are usually acquired from the referrer field in a
 // navigation.  A subresource URL may be associated with a referrer URL.  Later
 // navigations may, if the likelihood of needing the subresource is high enough,
 // cause this module to speculatively create a TCP/IP connection. If there is
 // only a low likelihood, then a DNS pre-resolution operation may be performed.
 
 #ifndef CHROME_BROWSER_NET_PREDICTOR_H_
 #define CHROME_BROWSER_NET_PREDICTOR_H_
 #pragma once
 
 #include <map>
 #include <queue>
 #include <set>
 #include <string>
 #include <vector>
 
 #include "base/gtest_prod_util.h"
-#include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
 #include "chrome/browser/net/url_info.h"
 #include "chrome/browser/net/referrer.h"
 #include "chrome/common/net/predictor_common.h"
 #include "net/base/host_port_pair.h"
 
 namespace base {
 class ListValue;
 }
 
+namespace base {
+class WaitableEvent;
+}
+
 namespace net {
 class HostResolver;
 }  // namespace net
 
+class IOThread;
+class PrefService;
+class Profile;
+
 namespace chrome_browser_net {
 
 typedef chrome_common_net::UrlList UrlList;
 typedef chrome_common_net::NameList NameList;
 typedef std::map<GURL, UrlInfo> Results;
 
 // Note that Predictor is not thread safe, and must only be called from
 // the IO thread. Failure to do so will result in a DCHECK at runtime.

[willchan no longer on Chromium, 2011/08/12 21:51:47]

You should update this comment. You should have something like this:

Predictor is constructed during Profile construction, but is destroyed on the IO
thread when ProfileIOData goes away. All of its core state and functionality
happens on the IO thread. The only UI methods are initialization / shutdown
related, or convenience methods that internally forward calls to the IO thread.

[rpetterson, 2011/08/13 00:55:17]

Done.

-class Predictor : public base::RefCountedThreadSafe<Predictor> {
+class Predictor {
  public:
   // A version number for prefs that are saved. This should be incremented when
   // we change the format so that we discard old data.
   enum { PREDICTOR_REFERRER_VERSION = 2 };
 
   // |max_concurrent| specifies how many concurrent (parallel) prefetches will
   // be performed. Host lookups will be issued through |host_resolver|.
-  Predictor(net::HostResolver* host_resolver,
-            base::TimeDelta max_queue_delay_ms, size_t max_concurrent,
-            bool preconnect_enabled);
+  Predictor();
+
+  virtual ~Predictor();
+
+  // ------------- Start UI thread methods.
+
+  virtual void InitNetworkPredictor(PrefService* user_prefs);
+
+  // The Omnibox has proposed a given url to the user, and if it is a search
+  // URL, then it also indicates that this is preconnectable (i.e., we could
+  // preconnect to the search server).
+  virtual void AnticipateOmniboxUrl(const GURL& url, bool preconnectable);
+
+  // Preconnect a URL and all of its subresource domains.
+  virtual void PreconnectUrlAndSubresources(const GURL& url);
+
+  // Instigate pre-connection to any URLs, or pre-resolution of related host,
+  // that we predict will be needed after this navigation (typically
+  // more-embedded resources on a page).  This method will actually post a task
+  // to do the actual work, so as not to jump ahead of the frame navigation that
+  // instigated this activity.
+  virtual void PredictFrameSubresources(const GURL& url);
+
+  static UrlList GetPredictedUrlListAtStartup(PrefService* user_prefs,
+                                              PrefService* local_state);
+
+  static void set_max_queueing_delay(int max_queueing_delay_ms);
+
+  static void set_max_parallel_resolves(size_t max_parallel_resolves);
+
+  static void RegisterUserPrefs(PrefService* user_prefs);
+
+  virtual void ShutdownOnUIThread(PrefService* user_prefs);
+
+  // ------------- End UI thread methods.
+
+  // ------------- Start IO thread methods.
 
   // Cancel pending requests and prevent new ones from being made.
-  void Shutdown();
+  virtual void Shutdown();
 
   // In some circumstances, for privacy reasons, all results should be
   // discarded.  This method gracefully handles that activity.
   // Destroy all our internal state, which shows what names we've looked up, and
   // how long each has taken, etc. etc.  We also destroy records of suggesses
   // (cache hits etc.).
-  void DiscardAllResults();
+  virtual void DiscardAllResults();
 
   // Add hostname(s) to the queue for processing.
-  void ResolveList(const UrlList& urls,
-                   UrlInfo::ResolutionMotivation motivation);
-  void Resolve(const GURL& url,
-               UrlInfo::ResolutionMotivation motivation);
+  virtual void ResolveList(const UrlList& urls,
+                           UrlInfo::ResolutionMotivation motivation);
 
-  // Instigate pre-connection to any URLs, or pre-resolution of related host,
-  // that we predict will be needed after this navigation (typically
-  // more-embedded resources on a page).  This method will actually post a task
-  // to do the actual work, so as not to jump ahead of the frame navigation that
-  // instigated this activity.
-  void PredictFrameSubresources(const GURL& url);
-
-  // The Omnibox has proposed a given url to the user, and if it is a search
-  // URL, then it also indicates that this is preconnectable (i.e., we could
-  // preconnect to the search server).
-  void AnticipateOmniboxUrl(const GURL& url, bool preconnectable);
-
-  // Preconnect a URL and all of its subresource domains.
-  void PreconnectUrlAndSubresources(const GURL& url);
+  virtual void Resolve(const GURL& url,
+                       UrlInfo::ResolutionMotivation motivation);
 
   // Record details of a navigation so that we can preresolve the host name
   // ahead of time the next time the users navigates to the indicated host.
   // Should only be called when urls are distinct, and they should already be
   // canonicalized to not have a path.
-  void LearnFromNavigation(const GURL& referring_url, const GURL& target_url);
+  virtual void LearnFromNavigation(const GURL& referring_url,
+                                   const GURL& target_url);
 
   // Dump HTML table containing list of referrers for about:dns.
-  void GetHtmlReferrerLists(std::string* output);
+  virtual void GetHtmlReferrerLists(std::string* output);
 
   // Dump the list of currently known referrer domains and related prefetchable
   // domains.
-  void GetHtmlInfo(std::string* output);
+  virtual void GetHtmlInfo(std::string* output);
 
   // Discards any referrer for which all the suggested host names are currently
   // annotated with negligible expected-use.  Scales down (diminishes) the
   // expected-use of those that remain, so that their use will go down by a
   // factor each time we trim (moving the referrer closer to being discarded in
   // a future call).
   // The task is performed synchronously and completes before returing.
-  void TrimReferrersNow();
+  virtual void TrimReferrersNow();
 
   // Construct a ListValue object that contains all the data in the referrers_
   // so that it can be persisted in a pref.
-  void SerializeReferrers(base::ListValue* referral_list);
+  virtual void SerializeReferrers(base::ListValue* referral_list);
 
   // Process a ListValue that contains all the data from a previous reference
   // list, as constructed by SerializeReferrers(), and add all the identified
   // values into the current referrer list.
-  void DeserializeReferrers(const base::ListValue& referral_list);
+  virtual void DeserializeReferrers(const base::ListValue& referral_list);
 
-  void DeserializeReferrersThenDelete(base::ListValue* referral_list);
+  virtual void DeserializeReferrersThenDelete(base::ListValue* referral_list);
 
-  // For unit test code only.
-  size_t max_concurrent_dns_lookups() const {
-    return max_concurrent_dns_lookups_;
-  }
+  virtual void DiscardInitialNavigationHistory();
 
-  // Flag setting to use preconnection instead of just DNS pre-fetching.
-  bool preconnect_enabled() const { return preconnect_enabled_; }
+  virtual void FinalizeInitializationOnIOThread(
+      const std::vector<GURL>& urls_to_prefetch,
+      base::ListValue* referral_list,
+      IOThread* io_thread);
+
+  // During startup, we learn what the first N urls visited are, and then
+  // resolve the associated hosts ASAP during our next startup.
+  virtual void LearnAboutInitialNavigation(const GURL& url);
+
+  // Renderer bundles up list and sends to this browser API via IPC.
+  // TODO(jar): Use UrlList instead to include port and scheme.
+  virtual void DnsPrefetchList(const NameList& hostnames);
+
+  // May be called from either the IO or UI thread and will PostTask
+  // to the IO thread if necessary.
+  virtual void Predictor::DnsPrefetchMotivatedList(
+    const UrlList& urls,
+    UrlInfo::ResolutionMotivation motivation);
+
+  // May be called from either the IO or UI thread and will PostTask
+  // to the IO thread if necessary.
+  virtual void SaveStateForNextStartupAndTrim(PrefService* prefs);
+
+  virtual void SaveDnsPrefetchStateForNextStartupAndTrim(
+      base::ListValue* startup_list,
+      base::ListValue* referral_list,
+      base::WaitableEvent* completion);
+
+  // May be called from either the IO or UI thread and will PostTask
+  // to the IO thread if necessary.
+  virtual void EnablePredictor(bool enable);
+
+  virtual void EnablePredictorOnIOThread(bool enable);
+
+  // ------------- End IO thread methods.
+
+  // The following methods may be called on either the IO or UI threads.
 
   // Put URL in canonical form, including a scheme, host, and port.
   // Returns GURL::EmptyGURL() if the scheme is not http/https or if the url
   // cannot be otherwise canonicalized.
   static GURL CanonicalizeUrl(const GURL& url);
 
+  // Used for testing.
+  void SetHostResolver(net::HostResolver* host_resolver) {
+    host_resolver_ = host_resolver;
+  }
+  // Used for testing.
+  size_t max_concurrent_dns_lookups() const {
+    return max_concurrent_dns_lookups_;
+  }
+
+  // Flag setting to use preconnection instead of just DNS pre-fetching.
+  bool preconnect_enabled() const {
+    return preconnect_enabled_;
+  }
+
+  // Flag setting for whether we are prefetching dns lookups.
+  bool predictor_enabled() const {
+    return predictor_enabled_;
+  }
+
+  // Given that the underlying Chromium resolver defaults to a total maximum of

[willchan no longer on Chromium, 2011/08/12 21:17:29]

constants go above methods in class declaration order:
http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml?showone=Declar...

[rpetterson, 2011/08/13 00:55:17]

Done.

+  // 8 paralell resolutions, we will avoid any chance of starving navigational
+  // resolutions by limiting the number of paralell speculative resolutions.
+  // This is used in the field trials and testing.
+  // TODO(jar): Move this limitation into the resolver.
+  static const size_t kMaxSpeculativeParallelResolves;
+
+  // To control the congestion avoidance system, we need an estimate of how
+  // many speculative requests may arrive at once.  Since we currently only
+  // keep 8 subresource names for each frame, we'll use that as our basis.
+  // Note that when scanning search results lists, we might actually get 10 at
+  // a time, and wikipedia can often supply (during a page scan) upwards of 50.
+  // In those odd cases, we may discard some of the later speculative requests
+  // mistakenly assuming that the resolutions took too long.
+  static const int kTypicalSpeculativeGroupSize;
+
+// The next constant specifies an amount of queueing delay that is
+// "too large," and indicative of problems with resolutions (perhaps due to
+// an overloaded router, or such).  When we exceed this delay, congestion
+// avoidance will kick in and all speculations in the queue will be discarded.
+  static const int kMaxSpeculativeResolveQueueDelayMs;
+
+
  private:
   friend class base::RefCountedThreadSafe<Predictor>;

[willchan no longer on Chromium, 2011/08/12 21:17:29]

Remove this

[rpetterson, 2011/08/13 00:55:17]

Done.

   FRIEND_TEST_ALL_PREFIXES(PredictorTest, BenefitLookupTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, ShutdownWhenResolutionIsPendingTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, SingleLookupTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, ConcurrentLookupTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, MassiveConcurrentLookupTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, PriorityQueuePushPopTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, PriorityQueueReorderTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, ReferrerSerializationTrimTest);
   friend class WaitForResolutionHelper;  // For testing.
 
@@ skipping 19 matching lines @@
    private:
     // The names in the queue that should be serviced (popped) ASAP.
     std::queue<GURL> rush_queue_;
     // The names in the queue that should only be serviced when rush_queue is
     // empty.
     std::queue<GURL> background_queue_;
 
   DISALLOW_COPY_AND_ASSIGN(HostNameQueue);
   };
 
+  // The InitialObserver monitors navigations made by the network stack. This
+  // is only used to identify startup time resolutions (for re-resolution
+  // during our next process startup).
+  // TODO(jar): Consider preconnecting at startup, which may be faster than
+  // waiting for render process to start and request a connection.
+  class InitialObserver {
+   public:
+    // Recording of when we observed each navigation.
+    typedef std::map<GURL, base::TimeTicks> FirstNavigations;
+
+    // Potentially add a new URL to our startup list.
+    void Append(const GURL& url, Predictor* predictor);
+
+    // Get an HTML version of our current planned first_navigations_.
+    void GetFirstResolutionsHtml(std::string* output);
+
+    // Persist the current first_navigations_ for storage in a list.
+    void GetInitialDnsResolutionList(base::ListValue* startup_list);
+
+    // Discards all initial loading history.
+    void DiscardInitialNavigationHistory() { first_navigations_.clear(); }
+
+   private:
+    // List of the first N URL resolutions observed in this run.
+    FirstNavigations first_navigations_;
+
+    // The number of URLs we'll save for pre-resolving at next startup.
+    static const size_t kStartupResolutionCount = 10;
+  };
+
   // A map that is keyed with the host/port that we've learned were the cause
   // of loading additional URLs.  The list of additional targets is held
   // in a Referrer instance, which is a value in this map.
   typedef std::map<GURL, Referrer> Referrers;
 
   // Depending on the expected_subresource_use_, we may either make a TCP/IP
   // preconnection, or merely pre-resolve the hostname via DNS (or even do
   // nothing).  The following are the threasholds for taking those actions.
   static const double kPreconnectWorthyExpectedValue;
   static const double kDNSPreresolutionWorthyExpectedValue;
   // Referred hosts with a subresource_use_rate_ that are less than the
   // following threshold will be discarded when we Trim() the list.
   static const double kDiscardableExpectedValue;
   // During trimming operation to discard hosts for which we don't have likely
   // subresources, we multiply the expected_subresource_use_ value by the
   // following ratio until that value is less than kDiscardableExpectedValue.
   // This number should always be less than 1, an more than 0.
   static const double kReferrerTrimRatio;
 
   // Interval between periodic trimming of our whole referrer list.
   // We only do a major trimming about once an hour, and then only when the user
   // is actively browsing.
   static const base::TimeDelta kDurationBetweenTrimmings;
   // Interval between incremental trimmings (to avoid inducing Jank).
   static const base::TimeDelta kDurationBetweenTrimmingIncrements;
   // Number of referring URLs processed in an incremental trimming.
   static const size_t kUrlsTrimmedPerIncrement;
 
-  ~Predictor();
-
-  // Perform actual resolution or preconnection to subresources now.  This is
-  // an internal worker method that is reached via a post task from
-  // PredictFrameSubresources().
-  void PrepareFrameSubresources(const GURL& url);
-
   // Only for testing. Returns true if hostname has been successfully resolved
   // (name found).
   bool WasFound(const GURL& url) const {
     Results::const_iterator it(results_.find(url));
     return (it != results_.end()) &&
             it->second.was_found();
   }
 
   // Only for testing. Return how long was the resolution
   // or UrlInfo::kNullDuration if it hasn't been resolved yet.
   base::TimeDelta GetResolutionDuration(const GURL& url) {
     if (results_.find(url) == results_.end())
       return UrlInfo::kNullDuration;
     return results_[url].resolve_duration();
   }
 
   // Only for testing;
   size_t peak_pending_lookups() const { return peak_pending_lookups_; }
 
+  // ------------- Start IO thread methods.
+
+  // Perform actual resolution or preconnection to subresources now.  This is
+  // an internal worker method that is reached via a post task from
+  // PredictFrameSubresources().
+  void PrepareFrameSubresources(const GURL& url);
+
   // Access method for use by async lookup request to pass resolution result.
   void OnLookupFinished(LookupRequest* request, const GURL& url, bool found);
 
   // Underlying method for both async and synchronous lookup to update state.
   void LookupFinished(LookupRequest* request,
                       const GURL& url, bool found);
 
   // Queue hostname for resolution.  If queueing was done, return the pointer
   // to the queued instance, otherwise return NULL.
   UrlInfo* AppendToResolutionQueue(const GURL& url,
@@ skipping 25 matching lines @@
   void LoadUrlsForTrimming();
 
   // Posts a task to do additional incremental trimming of referrers_.
   void PostIncrementalTrimTask();
 
   // Calls Trim() on some or all of urls_being_trimmed_.
   // If it does not process all the URLs in that vector, it posts a task to
   // continue with them shortly (i.e., it yeilds and continues).
   void IncrementalTrimReferrers(bool trim_all_now);
 
+  // ------------- End IO thread methods.
+
+  scoped_ptr<InitialObserver> initial_observer_;
+
+  // Status of speculative DNS resolution and speculative TCP/IP connection
+  // feature.
+  bool predictor_enabled_;
+
   // work_queue_ holds a list of names we need to look up.
   HostNameQueue work_queue_;
 
   // results_ contains information for existing/prior prefetches.
   Results results_;
 
   std::set<LookupRequest*> pending_lookups_;
 
   // For testing, to verify that we don't exceed the limit.
   size_t peak_pending_lookups_;
 
   // When true, we don't make new lookup requests.
   bool shutdown_;
 
   // The number of concurrent speculative lookups currently allowed to be sent
   // to the resolver.  Any additional lookups will be queued to avoid exceeding
   // this value.  The queue is a priority queue that will accelerate
   // sub-resource speculation, and retard resolutions suggested by page scans.
   const size_t max_concurrent_dns_lookups_;
 
   // The maximum queueing delay that is acceptable before we enter congestion
   // reduction mode, and discard all queued (but not yet assigned) resolutions.
   const base::TimeDelta max_dns_queue_delay_;
 
   // The host resolver we warm DNS entries for.
-  net::HostResolver* const host_resolver_;
+  net::HostResolver* host_resolver_;
 
   // Are we currently using preconnection, rather than just DNS resolution, for
   // subresources and omni-box search URLs.
   bool preconnect_enabled_;
 
   // Most recent suggestion from Omnibox provided via AnticipateOmniboxUrl().
   std::string last_omnibox_host_;
 
   // The time when the last preresolve was done for last_omnibox_host_.
   base::TimeTicks last_omnibox_preresolve_;
@@ skipping 11 matching lines @@
   // orginial hostname.
   Referrers referrers_;
 
   // List of URLs in referrers_ currently being trimmed (scaled down to
   // eventually be aged out of use).
   std::vector<GURL> urls_being_trimmed_;
 
   // A time after which we need to do more trimming of referrers.
   base::TimeTicks next_trim_time_;
 
-  ScopedRunnableMethodFactory<Predictor> trim_task_factory_;
+  scoped_ptr<ScopedRunnableMethodFactory<Predictor> > trim_task_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(Predictor);
 };
 
 }  // namespace chrome_browser_net
 
 #endif  // CHROME_BROWSER_NET_PREDICTOR_H_