Modifying prefetch to account for multi-profile.

chrome/browser/net/predictor.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // A Predictor object is instantiated once in the browser process, and manages
 // both preresolution of hostnames, as well as TCP/IP preconnection to expected
 // subresources.
 // Most hostname lists are provided by the renderer processes, and include URLs
 // that *might* be used in the near future by the browsing user.  One goal of
 // this class is to cause the underlying DNS structure to lookup a hostname
@@ skipping 10 matching lines @@
 #pragma once
 
 #include <map>
 #include <queue>
 #include <set>
 #include <string>
 #include <vector>
 
 #include "base/gtest_prod_util.h"
 #include "base/memory/ref_counted.h"
+#include "base/memory/scoped_ptr.h"
 #include "chrome/browser/net/url_info.h"
 #include "chrome/browser/net/referrer.h"
 #include "chrome/common/net/predictor_common.h"
 #include "net/base/host_port_pair.h"
 
 class ListValue;
+class PrefService;
+class Profile;
+
+namespace base {
+class WaitableEvent;
+}
 
 namespace net {
 class HostResolver;
 }  // namespace net
 
 namespace chrome_browser_net {
 
 typedef chrome_common_net::UrlList UrlList;
 typedef chrome_common_net::NameList NameList;
 typedef std::map<GURL, UrlInfo> Results;
 
 // Note that Predictor is not thread safe, and must only be called from
 // the IO thread. Failure to do so will result in a DCHECK at runtime.

[willchan no longer on Chromium, 2011/07/21 12:10:53]

This comment is wrong. It is used from both the UI and IO threads. Here's my
recommendation for lifetime management and thread organization stuff. Have
ProfileImpl construct the Predictor object (this is different from the current
world where Predictor is constructed on the IO thread, so be a bit careful with
some of the member variables that expect to be initialized and destroyed on the
same thread, such as the ScopedRunnableMethodFactory). Use the io_data_.Init()
call on
http://codesearch.google.com/codesearch#OAMlx_jo-ck/src/chrome/browser/profil...
to pass ownership of the Predictor object to the IO thread. Have ProfileImpl
keep a cached weak pointer to the Predictor object (this is why we created it on
the UI thread, so we can cache a non-owning pointer for use on the UI thread).
Note that this is safe because the ProfileImpl is destroyed on the UI thread
before the ProfileImplIOData is destroyed on the IO thread.

Note that ProfileIOData so far is always lazily initialized for startup reasons
and what not. You will want to eagerly initialize the Predictor object on the
ProfileImplIOData. Therefore, in
http://codesearch.google.com/codesearch#OAMlx_jo-ck/src/chrome/browser/profil...,
you should not use LazyParams. You should post a task to the IO thread to pass
ownership of the Predictor object to the IO thread. You should probably add a
InitOnIOThread() method to Predictor that gets invoked when this happens. This
will be analogous to the existing InitNetworkPredictorOnIOThread() that exists
in the IOThread class. Here you should initialize the member variables that
expect to be initialize on the IO thread, such as the
ScopedRunnableMethodFactory (it'll probably have to be wrapped in scoped_ptr).

Then please organize your methods here. Create one section for methods called on
the UI thread. Create another section for methods called on the IO thread. That
way it's clear to users what the thread expectations of the API are.

[rpetterson, 2011/08/10 02:24:46]

I think I've covered everything in this comment. Looking forward to your
comments on it.

 class Predictor : public base::RefCountedThreadSafe<Predictor> {
  public:
   // A version number for prefs that are saved. This should be incremented when
   // we change the format so that we discard old data.
   enum { PREDICTOR_REFERRER_VERSION = 2 };
 
   // |max_concurrent| specifies how many concurrent (parallel) prefetches will
   // be performed. Host lookups will be issued through |host_resolver|.
   Predictor(net::HostResolver* host_resolver,
             base::TimeDelta max_queue_delay_ms, size_t max_concurrent,
-            bool preconnect_enabled);
+            bool preconnect_enabled, bool predictor_enabled);
+
+  ~Predictor();
 
   // Cancel pending requests and prevent new ones from being made.
   void Shutdown();
 
   // In some circumstances, for privacy reasons, all results should be
   // discarded.  This method gracefully handles that activity.
   // Destroy all our internal state, which shows what names we've looked up, and
   // how long each has taken, etc. etc.  We also destroy records of suggesses
   // (cache hits etc.).
   void DiscardAllResults();
@@ skipping 44 matching lines @@
   // so that it can be persisted in a pref.
   void SerializeReferrers(ListValue* referral_list);
 
   // Process a ListValue that contains all the data from a previous reference
   // list, as constructed by SerializeReferrers(), and add all the identified
   // values into the current referrer list.
   void DeserializeReferrers(const ListValue& referral_list);
 
   void DeserializeReferrersThenDelete(ListValue* referral_list);
 
+  void DiscardInitialNavigationHistory();
+
+  void FinalizeInitialization(const std::vector<GURL>& urls_to_prefetch,
+                              ListValue* referral_list);
+
+  // During startup, we learn what the first N urls visited are, and then
+  // resolve the associated hosts ASAP during our next startup.
+  void LearnAboutInitialNavigation(const GURL& url);
+
+  // Renderer bundles up list and sends to this browser API via IPC.
+  // TODO(jar): Use UrlList instead to include port and scheme.
+  void DnsPrefetchList(const NameList& hostnames);
+
+  void Predictor::DnsPrefetchMotivatedList(
+    const UrlList& urls,
+    UrlInfo::ResolutionMotivation motivation);
+
+  void SaveStateForNextStartupAndTrim(PrefService* prefs, Profile* profile);
+
+  void SaveDnsPrefetchStateForNextStartupAndTrim(
+      ListValue* startup_list,
+      ListValue* referral_list,
+      base::WaitableEvent* completion);
+
   // For unit test code only.
   size_t max_concurrent_dns_lookups() const {
     return max_concurrent_dns_lookups_;
   }
 
   // Flag setting to use preconnection instead of just DNS pre-fetching.
-  bool preconnect_enabled() const { return preconnect_enabled_; }
+  bool preconnect_enabled() const {
+    return preconnect_enabled_;
+  }
+
+  // Flag setting for whether we are prefetching dns lookups.
+  bool predictor_enabled() const {
+    return predictor_enabled_;
+  }
+
+  void EnablePredictor(bool enable) {
+    predictor_enabled_ = enable;
+  }
+
+  // See variable description for explanation.
+  bool on_the_record() const {
+    return on_the_record_;
+  }
+
+  void SetOnTheRecord(bool on_the_record) {
+    on_the_record_ = on_the_record;
+  }
+
+  // Return value indicates whether to go back on the record.
+  bool HandleIncognitoBrowserClosed();
+
+  void HandleIncognitoBrowserOpened();
 
   // Put URL in canonical form, including a scheme, host, and port.
   // Returns GURL::EmptyGURL() if the scheme is not http/https or if the url
   // cannot be otherwise canonicalized.
   static GURL CanonicalizeUrl(const GURL& url);
 
+  static UrlList GetPredictedUrlListAtStartup(PrefService* user_prefs,
+                                              PrefService* local_state);
+
  private:
   friend class base::RefCountedThreadSafe<Predictor>;
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, BenefitLookupTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, ShutdownWhenResolutionIsPendingTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, SingleLookupTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, ConcurrentLookupTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, MassiveConcurrentLookupTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, PriorityQueuePushPopTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, PriorityQueueReorderTest);
   FRIEND_TEST_ALL_PREFIXES(PredictorTest, ReferrerSerializationTrimTest);
@@ skipping 21 matching lines @@
    private:
     // The names in the queue that should be serviced (popped) ASAP.
     std::queue<GURL> rush_queue_;
     // The names in the queue that should only be serviced when rush_queue is
     // empty.
     std::queue<GURL> background_queue_;
 
   DISALLOW_COPY_AND_ASSIGN(HostNameQueue);
   };
 
+  // The InitialObserver monitors navigations made by the network stack. This
+  // is only used to identify startup time resolutions (for re-resolution
+  // during our next process startup).
+  // TODO(jar): Consider preconnecting at startup, which may be faster than
+  // waiting for render process to start and request a connection.
+  class InitialObserver {
+   public:
+    // Recording of when we observed each navigation.
+    typedef std::map<GURL, base::TimeTicks> FirstNavigations;
+
+    // Potentially add a new URL to our startup list.
+    void Append(const GURL& url, Predictor* predictor);
+
+    // Get an HTML version of our current planned first_navigations_.
+    void GetFirstResolutionsHtml(std::string* output);
+
+    // Persist the current first_navigations_ for storage in a list.
+    void GetInitialDnsResolutionList(ListValue* startup_list);
+
+    // Discards all initial loading history.
+    void DiscardInitialNavigationHistory() { first_navigations_.clear(); }
+
+   private:
+    // List of the first N URL resolutions observed in this run.
+    FirstNavigations first_navigations_;
+
+    // The number of URLs we'll save for pre-resolving at next startup.
+    static const size_t kStartupResolutionCount = 10;
+  };
+
   // A map that is keyed with the host/port that we've learned were the cause
   // of loading additional URLs.  The list of additional targets is held
   // in a Referrer instance, which is a value in this map.
   typedef std::map<GURL, Referrer> Referrers;
 
   // Depending on the expected_subresource_use_, we may either make a TCP/IP
   // preconnection, or merely pre-resolve the hostname via DNS (or even do
   // nothing).  The following are the threasholds for taking those actions.
   static const double kPreconnectWorthyExpectedValue;
   static const double kDNSPreresolutionWorthyExpectedValue;
   // Referred hosts with a subresource_use_rate_ that are less than the
   // following threshold will be discarded when we Trim() the list.
   static const double kDiscardableExpectedValue;
   // During trimming operation to discard hosts for which we don't have likely
   // subresources, we multiply the expected_subresource_use_ value by the
   // following ratio until that value is less than kDiscardableExpectedValue.
   // This number should always be less than 1, an more than 0.
   static const double kReferrerTrimRatio;
 
   // Interval between periodic trimming of our whole referrer list.
   // We only do a major trimming about once an hour, and then only when the user
   // is actively browsing.
   static const base::TimeDelta kDurationBetweenTrimmings;
   // Interval between incremental trimmings (to avoid inducing Jank).
   static const base::TimeDelta kDurationBetweenTrimmingIncrements;
   // Number of referring URLs processed in an incremental trimming.
   static const size_t kUrlsTrimmedPerIncrement;
 
-  ~Predictor();
-
   // Perform actual resolution or preconnection to subresources now.  This is
   // an internal worker method that is reached via a post task from
   // PredictFrameSubresources().
   void PrepareFrameSubresources(const GURL& url);
 
   // Only for testing. Returns true if hostname has been successfully resolved
   // (name found).
   bool WasFound(const GURL& url) const {
     Results::const_iterator it(results_.find(url));
     return (it != results_.end()) &&
@@ skipping 49 matching lines @@
   void LoadUrlsForTrimming();
 
   // Posts a task to do additional incremental trimming of referrers_.
   void PostIncrementalTrimTask();
 
   // Calls Trim() on some or all of urls_being_trimmed_.
   // If it does not process all the URLs in that vector, it posts a task to
   // continue with them shortly (i.e., it yeilds and continues).
   void IncrementalTrimReferrers(bool trim_all_now);
 
+  scoped_ptr<InitialObserver> initial_observer_;
+
+  // Status of speculative DNS resolution and speculative TCP/IP connection
+  // feature.
+  bool predictor_enabled_;
+
+  // If there are any incognito windows at all, we don't want to keep prefetch
+  // information since it would be visible in about:dns, for example.
+  int off_the_record_windows_count_;
+
+  // Cached inverted copy of the off_the_record pref.
+  bool on_the_record_;
+
   // work_queue_ holds a list of names we need to look up.
   HostNameQueue work_queue_;
 
   // results_ contains information for existing/prior prefetches.
   Results results_;
 
   std::set<LookupRequest*> pending_lookups_;
 
   // For testing, to verify that we don't exceed the limit.
   size_t peak_pending_lookups_;
@@ skipping 45 matching lines @@
   base::TimeTicks next_trim_time_;
 
   ScopedRunnableMethodFactory<Predictor> trim_task_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(Predictor);
 };
 
 }  // namespace chrome_browser_net
 
 #endif  // CHROME_BROWSER_NET_PREDICTOR_H_