Modifying prefetch to account for multi-profile.

chrome/browser/net/predictor.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/net/predictor.h"
 
 #include <algorithm>
 #include <cmath>
 #include <set>
 #include <sstream>
 
 #include "base/compiler_specific.h"
 #include "base/metrics/histogram.h"
 #include "base/stringprintf.h"
+#include "base/synchronization/waitable_event.h"
 #include "base/time.h"
 #include "base/values.h"
 #include "chrome/browser/net/preconnect.h"
+#include "chrome/browser/prefs/pref_service.h"
+#include "chrome/browser/prefs/scoped_user_pref_update.h"
+#include "chrome/browser/prefs/session_startup_pref.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/common/pref_names.h"
 #include "content/browser/browser_thread.h"
 #include "net/base/address_list.h"
 #include "net/base/completion_callback.h"
 #include "net/base/host_port_pair.h"
 #include "net/base/host_resolver.h"
 #include "net/base/net_errors.h"
 #include "net/base/net_log.h"
 #include "net/base/single_request_host_resolver.h"
 
 using base::TimeDelta;
 
 namespace chrome_browser_net {
 
+static void DnsPrefetchMotivatedList(const UrlList& urls,
+                                     UrlInfo::ResolutionMotivation motivation);
 // static
 const double Predictor::kPreconnectWorthyExpectedValue = 0.8;
 // static
 const double Predictor::kDNSPreresolutionWorthyExpectedValue = 0.1;
 // static
 const double Predictor::kDiscardableExpectedValue = 0.05;
 // The goal is of trimming is to to reduce the importance (number of expected
 // subresources needed) by a factor of 2 after about 24 hours of uptime. We will
 // trim roughly once-an-hour of uptime.  The ratio to use in each trim operation
 // is then the 24th root of 0.5.  If a user only surfs for 4 hours a day, then
 // after about 6 days they will have halved all their estimates of subresource
 // connections.  Once this falls below kDiscardableExpectedValue the referrer
 // will be discarded.
 // TODO(jar): Measure size of referrer lists in the field.  Consider an adaptive
 // system that uses a higher trim ratio when the list is large.
 // static
 const double Predictor::kReferrerTrimRatio = 0.97153;
 
 // static
 const TimeDelta Predictor::kDurationBetweenTrimmings = TimeDelta::FromHours(1);
 // static
 const TimeDelta Predictor::kDurationBetweenTrimmingIncrements =
     TimeDelta::FromSeconds(15);
 // static
 const size_t Predictor::kUrlsTrimmedPerIncrement = 5u;
 
+// A version number for prefs that are saved. This should be incremented when
+// we change the format so that we discard old data.
+static const int kPredictorStartupFormatVersion = 1;
+
 class Predictor::LookupRequest {
  public:
   LookupRequest(Predictor* predictor,
                 net::HostResolver* host_resolver,
                 const GURL& url)
       : ALLOW_THIS_IN_INITIALIZER_LIST(
             net_callback_(this, &LookupRequest::OnLookupFinished)),
         predictor_(predictor),
         url_(url),
         resolver_(host_resolver) {
@@ skipping 28 matching lines @@
   const GURL url_;  // Hostname to resolve.
   net::SingleRequestHostResolver resolver_;
   net::AddressList addresses_;
 
   DISALLOW_COPY_AND_ASSIGN(LookupRequest);
 };
 
 Predictor::Predictor(net::HostResolver* host_resolver,
                      TimeDelta max_dns_queue_delay,
                      size_t max_concurrent,
-                     bool preconnect_enabled)
-    : peak_pending_lookups_(0),
+                     bool preconnect_enabled,
+                     bool predictor_enabled)
+    : predictor_enabled_(true),
+      off_the_record_windows_count_(0),
+      on_the_record_(true),
+      peak_pending_lookups_(0),
       shutdown_(false),
       max_concurrent_dns_lookups_(max_concurrent),
       max_dns_queue_delay_(max_dns_queue_delay),
       host_resolver_(host_resolver),
       preconnect_enabled_(preconnect_enabled),
       consecutive_omnibox_preconnect_count_(0),
       next_trim_time_(base::TimeTicks::Now() + kDurationBetweenTrimmings),
       ALLOW_THIS_IN_INITIALIZER_LIST(trim_task_factory_(this)) {
+  initial_observer_.reset(new InitialObserver());
 }
 
 Predictor::~Predictor() {
   DCHECK(shutdown_);
 }
 
 void Predictor::Shutdown() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DCHECK(!shutdown_);
   shutdown_ = true;
@@ skipping 19 matching lines @@
                         UrlInfo::ResolutionMotivation motivation) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   if (!url.has_host())
     return;
   AppendToResolutionQueue(url, motivation);
 }
 
 void Predictor::LearnFromNavigation(const GURL& referring_url,
                                     const GURL& target_url) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  if (!predictor_enabled_)
+    return;
   DCHECK_EQ(referring_url, Predictor::CanonicalizeUrl(referring_url));
   DCHECK_NE(referring_url, GURL::EmptyGURL());
   DCHECK_EQ(target_url, Predictor::CanonicalizeUrl(target_url));
   DCHECK_NE(target_url, GURL::EmptyGURL());
 
   referrers_[referring_url].SuggestHost(target_url);
   // Possibly do some referrer trimming.
   TrimReferrers();
 }
 
 enum SubresourceValue {
   PRECONNECTION,
   PRERESOLUTION,
   TOO_NEW,
   SUBRESOURCE_VALUE_MAX
 };
 
 void Predictor::AnticipateOmniboxUrl(const GURL& url, bool preconnectable) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  if (!predictor_enabled_)
+    return;
+  if (!url.is_valid() || !url.has_host())
+    return;
   std::string host = url.HostNoBrackets();
   bool is_new_host_request = (host != last_omnibox_host_);
   last_omnibox_host_ = host;
 
   UrlInfo::ResolutionMotivation motivation(UrlInfo::OMNIBOX_MOTIVATED);
   base::TimeTicks now = base::TimeTicks::Now();
 
   if (preconnect_enabled()) {
     if (preconnectable && !is_new_host_request) {
       ++consecutive_omnibox_preconnect_count_;
@@ skipping 48 matching lines @@
 
   // Perform at least DNS pre-resolution.
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       NewRunnableMethod(this, &Predictor::Resolve, CanonicalizeUrl(url),
                         motivation));
 }
 
 void Predictor::PreconnectUrlAndSubresources(const GURL& url) {
+  if (!predictor_enabled_)
+    return;
+  if (!url.is_valid() || !url.has_host())
+    return;
   if (preconnect_enabled()) {
     std::string host = url.HostNoBrackets();
     UrlInfo::ResolutionMotivation motivation(UrlInfo::EARLY_LOAD_MOTIVATED);
     const int kConnectionsNeeded = 1;
     PreconnectOnUIThread(CanonicalizeUrl(url), motivation,
                          kConnectionsNeeded);
     PredictFrameSubresources(url.GetWithEmptyPath());
   }
 }
 
 void Predictor::PredictFrameSubresources(const GURL& url) {
+  if (!predictor_enabled_)
+    return;
   DCHECK_EQ(url.GetWithEmptyPath(), url);
   // Add one pass through the message loop to allow current navigation to
   // proceed.
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       NewRunnableMethod(this, &Predictor::PrepareFrameSubresources, url));
 }
 
 void Predictor::PrepareFrameSubresources(const GURL& url) {
@@ skipping 155 matching lines @@
           static_cast<int>(future_url->second.preresolution_count()),
           static_cast<double>(future_url->second.subresource_use_rate()),
           future_url->first.spec().c_str());
     }
   }
   output->append("</table>");
 }
 
 void Predictor::GetHtmlInfo(std::string* output) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  if (initial_observer_.get())
+    initial_observer_->GetFirstResolutionsHtml(output);
+  // Show list of subresource predictions and stats.
+  GetHtmlReferrerLists(output);
+
   // Local lists for calling UrlInfo
   UrlInfo::UrlInfoTable name_not_found;
   UrlInfo::UrlInfoTable name_preresolved;
 
   // Get copies of all useful data.
   typedef std::map<GURL, UrlInfo, RightToLeftStringSorter> SortedUrlInfo;
   SortedUrlInfo snapshot;
   // UrlInfo supports value semantics, so we can do a shallow copy.
   for (Results::iterator it(results_.begin()); it != results_.end(); it++)
     snapshot[it->first] = it->second;
@@ skipping 260 matching lines @@
     Referrers::iterator it = referrers_.find(urls_being_trimmed_.back());
     urls_being_trimmed_.pop_back();
     if (it == referrers_.end())
       continue;  // Defensive code: It got trimmed away already.
     if (!it->second.Trim(kReferrerTrimRatio, kDiscardableExpectedValue))
       referrers_.erase(it);
   }
   PostIncrementalTrimTask();
 }
 
+void Predictor::DiscardInitialNavigationHistory() {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  if (initial_observer_.get())
+    initial_observer_->DiscardInitialNavigationHistory();
+}
+
+void Predictor::FinalizeInitialization(
+    const UrlList& startup_urls,
+    ListValue* referral_list) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  // Prefetch these hostnames on startup.
+  DnsPrefetchMotivatedList(startup_urls, UrlInfo::STARTUP_LIST_MOTIVATED);
+  DeserializeReferrersThenDelete(referral_list);
+}
+
+//------------------------------------------------------------------------------
+// This section intermingles prefetch results with actual browser HTTP
+// network activity.  It supports calculating of the benefit of a prefetch, as
+// well as recording what prefetched hostname resolutions might be potentially
+// helpful during the next chrome-startup.
+//------------------------------------------------------------------------------
+
+void Predictor::LearnAboutInitialNavigation(const GURL& url) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  if (!predictor_enabled_ || NULL == initial_observer_.get() )
+    return;
+  initial_observer_->Append(url, this);
+}
+
+// This API is only used in the browser process.
+// It is called from an IPC message originating in the renderer.  It currently
+// includes both Page-Scan, and Link-Hover prefetching.
+// TODO(jar): Separate out link-hover prefetching, and page-scan results.
+void Predictor::DnsPrefetchList(const NameList& hostnames) {
+  // TODO(jar): Push GURL transport further back into renderer, but this will
+  // require a Webkit change in the observer :-/.
+  UrlList urls;
+  for (NameList::const_iterator it = hostnames.begin();
+       it < hostnames.end();
+       ++it) {
+    urls.push_back(GURL("http://" + *it + ":80"));
+  }
+
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DnsPrefetchMotivatedList(urls, UrlInfo::PAGE_SCAN_MOTIVATED);
+}
+
+void Predictor::DnsPrefetchMotivatedList(
+    const UrlList& urls,
+    UrlInfo::ResolutionMotivation motivation) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI) ||
+         BrowserThread::CurrentlyOn(BrowserThread::IO));
+  if (!predictor_enabled_)
+    return;
+
+  if (BrowserThread::CurrentlyOn(BrowserThread::IO)) {
+    ResolveList(urls, motivation);
+  } else {
+    BrowserThread::PostTask(
+        BrowserThread::IO,
+        FROM_HERE,
+        NewRunnableMethod(this, &Predictor::ResolveList, urls, motivation));
+  }
+}
+
+//------------------------------------------------------------------------------
+// Functions to handle saving of hostnames from one session to the next, to
+// expedite startup times.
+
+static void SaveDnsPrefetchStateForNextStartupAndTrimOnIOThread(
+    ListValue* startup_list,
+    ListValue* referral_list,
+    base::WaitableEvent* completion,
+    Profile* profile) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  if (NULL == profile->GetPredictor()) {
+    completion->Signal();
+    return;
+  }
+  profile->GetPredictor()->SaveDnsPrefetchStateForNextStartupAndTrim(
+      startup_list, referral_list, completion);
+}
+
+void Predictor::SaveDnsPrefetchStateForNextStartupAndTrim(
+    ListValue* startup_list,
+    ListValue* referral_list,
+    base::WaitableEvent* completion) {
+  if (initial_observer_.get())
+    initial_observer_->GetInitialDnsResolutionList(startup_list);
+
+  // Do at least one trim at shutdown, in case the user wasn't running long
+  // enough to do any regular trimming of referrers.
+  TrimReferrersNow();
+  SerializeReferrers(referral_list);
+
+  completion->Signal();
+}
+
+void Predictor::SaveStateForNextStartupAndTrim(PrefService* prefs,
+                                               Profile* profile) {
+  if (!predictor_enabled_)
+    return;
+
+  base::WaitableEvent completion(true, false);
+
+  ListPrefUpdate update_startup_list(prefs, prefs::kDnsPrefetchingStartupList);
+  ListPrefUpdate update_referral_list(prefs,
+                                      prefs::kDnsPrefetchingHostReferralList);
+  if (BrowserThread::CurrentlyOn(BrowserThread::IO)) {
+    SaveDnsPrefetchStateForNextStartupAndTrimOnIOThread(
+        update_startup_list.Get(),
+        update_referral_list.Get(),
+        &completion,
+        profile);
+  } else {
+    bool posted = BrowserThread::PostTask(
+        BrowserThread::IO,
+        FROM_HERE,
+        NewRunnableFunction(
+            SaveDnsPrefetchStateForNextStartupAndTrimOnIOThread,
+            update_startup_list.Get(),
+            update_referral_list.Get(),
+            &completion,
+            profile));
+
+    // TODO(jar): Synchronous waiting for the IO thread is a potential source
+    // to deadlocks and should be investigated. See http://crbug.com/78451.
+    DCHECK(posted);
+    if (posted)
+      completion.Wait();
+  }
+}
+
+UrlList Predictor::GetPredictedUrlListAtStartup(
+    PrefService* user_prefs,
+    PrefService* local_state) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
+  UrlList urls;
+  // Recall list of URLs we learned about during last session.
+  // This may catch secondary hostnames, pulled in by the homepages.  It will
+  // also catch more of the "primary" home pages, since that was (presumably)
+  // rendered first (and will be rendered first this time too).
+  const ListValue* startup_list =
+      user_prefs->GetList(prefs::kDnsPrefetchingStartupList);
+
+  if (startup_list) {
+    ListValue::const_iterator it = startup_list->begin();
+    int format_version = -1;
+    if (it != startup_list->end() &&
+        (*it)->GetAsInteger(&format_version) &&
+        format_version == kPredictorStartupFormatVersion) {
+      ++it;
+      for (; it != startup_list->end(); ++it) {
+        std::string url_spec;
+        if (!(*it)->GetAsString(&url_spec)) {
+          LOG(DFATAL);
+          break;  // Format incompatibility.
+        }
+        GURL url(url_spec);
+        if (!url.has_host() || !url.has_scheme()) {
+          LOG(DFATAL);
+          break;  // Format incompatibility.
+        }
+
+        urls.push_back(url);
+      }
+    }
+  }
+
+  // Prepare for any static home page(s) the user has in prefs.  The user may
+  // have a LOT of tab's specified, so we may as well try to warm them all.
+  SessionStartupPref tab_start_pref =
+      SessionStartupPref::GetStartupPref(user_prefs);
+  if (SessionStartupPref::URLS == tab_start_pref.type) {
+    for (size_t i = 0; i < tab_start_pref.urls.size(); i++) {
+      GURL gurl = tab_start_pref.urls[i];
+      if (!gurl.is_valid() || gurl.SchemeIsFile() || gurl.host().empty())
+        continue;
+      if (gurl.SchemeIs("http") || gurl.SchemeIs("https"))
+        urls.push_back(gurl.GetWithEmptyPath());
+    }
+  }
+
+  if (urls.empty())
+    urls.push_back(GURL("http://www.google.com:80"));
+
+  return urls;
+}
+
+bool Predictor::HandleIncognitoBrowserClosed() {
+  DCHECK_LT(0, off_the_record_windows_count_);
+  if (0 >= off_the_record_windows_count_)  // Defensive coding.
+    return false;
+  if (--off_the_record_windows_count_)
+    return false;  // Still some windows are incognito.
+  return true;
+}
+
+void Predictor::HandleIncognitoBrowserOpened() {
+  off_the_record_windows_count_++;
+}
+
 //------------------------------------------------------------------------------
 
 Predictor::HostNameQueue::HostNameQueue() {
 }
 
 Predictor::HostNameQueue::~HostNameQueue() {
 }
 
 void Predictor::HostNameQueue::Push(const GURL& url,
     UrlInfo::ResolutionMotivation motivation) {
@@ skipping 21 matching lines @@
   GURL url(queue->front());
   queue->pop();
   return url;
 }
 
 void Predictor::DeserializeReferrersThenDelete(ListValue* referral_list) {
     DeserializeReferrers(*referral_list);
     delete referral_list;
 }
 
+//------------------------------------------------------------------------------
+// Member definitions for InitialObserver class.
+
+void Predictor::InitialObserver::Append(const GURL& url,
+                                        Predictor* predictor) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  // TODO(rlp): Do we really need the predictor check here?
+  if (!predictor->on_the_record() || NULL == predictor)
+    return;
+  if (kStartupResolutionCount <= first_navigations_.size())
+    return;
+
+  DCHECK(url.SchemeIs("http") || url.SchemeIs("https"));
+  DCHECK_EQ(url, Predictor::CanonicalizeUrl(url));
+  if (first_navigations_.find(url) == first_navigations_.end())
+    first_navigations_[url] = base::TimeTicks::Now();
+}
+
+void Predictor::InitialObserver::GetInitialDnsResolutionList(
+    ListValue* startup_list) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+  DCHECK(startup_list);
+  startup_list->Clear();
+  DCHECK_EQ(0u, startup_list->GetSize());
+  startup_list->Append(new FundamentalValue(kPredictorStartupFormatVersion));
+  for (FirstNavigations::iterator it = first_navigations_.begin();
+       it != first_navigations_.end();
+       ++it) {
+    DCHECK(it->first == Predictor::CanonicalizeUrl(it->first));
+    startup_list->Append(new StringValue(it->first.spec()));
+  }
+}
+
+void Predictor::InitialObserver::GetFirstResolutionsHtml(
+    std::string* output) {
+  DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
+
+  UrlInfo::UrlInfoTable resolution_list;
+  {
+    for (FirstNavigations::iterator it(first_navigations_.begin());
+         it != first_navigations_.end();
+         it++) {
+      UrlInfo info;
+      info.SetUrl(it->first);
+      info.set_time(it->second);
+      resolution_list.push_back(info);
+    }
+  }
+  UrlInfo::GetHtmlTable(resolution_list,
+      "Future startups will prefetch DNS records for ", false, output);
+}
 
 //------------------------------------------------------------------------------
 // Helper functions
 //------------------------------------------------------------------------------
 
 // static
 GURL Predictor::CanonicalizeUrl(const GURL& url) {
   if (!url.has_host())
      return GURL::EmptyGURL();
 
@@ skipping 11 matching lines @@
   // If we omit a port, it will default to 80 or 443 as appropriate.
   std::string colon_plus_port;
   if (url.has_port())
     colon_plus_port = ":" + url.port();
 
   return GURL(scheme + "://" + url.host() + colon_plus_port);
 }
 
 
 }  // namespace chrome_browser_net