OLD | NEW |
1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "chrome/browser/net/predictor.h" | 5 #include "chrome/browser/net/predictor.h" |
6 | 6 |
7 #include <algorithm> | 7 #include <algorithm> |
8 #include <cmath> | 8 #include <cmath> |
9 #include <set> | 9 #include <set> |
10 #include <sstream> | 10 #include <sstream> |
11 | 11 |
12 #include "base/bind.h" | |
13 #include "base/command_line.h" | |
14 #include "base/compiler_specific.h" | 12 #include "base/compiler_specific.h" |
15 #include "base/metrics/histogram.h" | 13 #include "base/metrics/histogram.h" |
16 #include "base/stl_util.h" | |
17 #include "base/stringprintf.h" | 14 #include "base/stringprintf.h" |
18 #include "base/synchronization/waitable_event.h" | |
19 #include "base/time.h" | 15 #include "base/time.h" |
20 #include "base/values.h" | 16 #include "base/values.h" |
21 #include "chrome/browser/io_thread.h" | |
22 #include "chrome/browser/net/preconnect.h" | 17 #include "chrome/browser/net/preconnect.h" |
23 #include "chrome/browser/prefs/browser_prefs.h" | |
24 #include "chrome/browser/prefs/pref_service.h" | |
25 #include "chrome/browser/prefs/scoped_user_pref_update.h" | |
26 #include "chrome/browser/prefs/session_startup_pref.h" | |
27 #include "chrome/common/chrome_switches.h" | |
28 #include "chrome/common/pref_names.h" | |
29 #include "content/browser/browser_thread.h" | 18 #include "content/browser/browser_thread.h" |
30 #include "net/base/address_list.h" | 19 #include "net/base/address_list.h" |
31 #include "net/base/completion_callback.h" | 20 #include "net/base/completion_callback.h" |
32 #include "net/base/host_port_pair.h" | 21 #include "net/base/host_port_pair.h" |
33 #include "net/base/host_resolver.h" | 22 #include "net/base/host_resolver.h" |
34 #include "net/base/net_errors.h" | 23 #include "net/base/net_errors.h" |
35 #include "net/base/net_log.h" | 24 #include "net/base/net_log.h" |
36 #include "net/base/single_request_host_resolver.h" | 25 #include "net/base/single_request_host_resolver.h" |
37 | 26 |
38 using base::TimeDelta; | 27 using base::TimeDelta; |
(...skipping 18 matching lines...) Expand all Loading... |
57 // static | 46 // static |
58 const double Predictor::kReferrerTrimRatio = 0.97153; | 47 const double Predictor::kReferrerTrimRatio = 0.97153; |
59 | 48 |
60 // static | 49 // static |
61 const TimeDelta Predictor::kDurationBetweenTrimmings = TimeDelta::FromHours(1); | 50 const TimeDelta Predictor::kDurationBetweenTrimmings = TimeDelta::FromHours(1); |
62 // static | 51 // static |
63 const TimeDelta Predictor::kDurationBetweenTrimmingIncrements = | 52 const TimeDelta Predictor::kDurationBetweenTrimmingIncrements = |
64 TimeDelta::FromSeconds(15); | 53 TimeDelta::FromSeconds(15); |
65 // static | 54 // static |
66 const size_t Predictor::kUrlsTrimmedPerIncrement = 5u; | 55 const size_t Predictor::kUrlsTrimmedPerIncrement = 5u; |
67 // static | |
68 const size_t Predictor::kMaxSpeculativeParallelResolves = 3; | |
69 // To control our congestion avoidance system, which discards a queue when | |
70 // resolutions are "taking too long," we need an expected resolution time. | |
71 // Common average is in the range of 300-500ms. | |
72 const int kExpectedResolutionTimeMs = 500; | |
73 // static | |
74 const int Predictor::kTypicalSpeculativeGroupSize = 8; | |
75 // static | |
76 const int Predictor::kMaxSpeculativeResolveQueueDelayMs = | |
77 (kExpectedResolutionTimeMs * Predictor::kTypicalSpeculativeGroupSize) / | |
78 Predictor::kMaxSpeculativeParallelResolves; | |
79 | |
80 static int g_max_queueing_delay_ms = 0; | |
81 static size_t g_max_parallel_resolves = 0u; | |
82 | |
83 // A version number for prefs that are saved. This should be incremented when | |
84 // we change the format so that we discard old data. | |
85 static const int kPredictorStartupFormatVersion = 1; | |
86 | 56 |
87 class Predictor::LookupRequest { | 57 class Predictor::LookupRequest { |
88 public: | 58 public: |
89 LookupRequest(Predictor* predictor, | 59 LookupRequest(Predictor* predictor, |
90 net::HostResolver* host_resolver, | 60 net::HostResolver* host_resolver, |
91 const GURL& url) | 61 const GURL& url) |
92 : ALLOW_THIS_IN_INITIALIZER_LIST( | 62 : ALLOW_THIS_IN_INITIALIZER_LIST( |
93 net_callback_(this, &LookupRequest::OnLookupFinished)), | 63 net_callback_(this, &LookupRequest::OnLookupFinished)), |
94 predictor_(predictor), | 64 predictor_(predictor), |
95 url_(url), | 65 url_(url), |
(...skipping 26 matching lines...) Expand all Loading... |
122 | 92 |
123 Predictor* predictor_; // The predictor which started us. | 93 Predictor* predictor_; // The predictor which started us. |
124 | 94 |
125 const GURL url_; // Hostname to resolve. | 95 const GURL url_; // Hostname to resolve. |
126 net::SingleRequestHostResolver resolver_; | 96 net::SingleRequestHostResolver resolver_; |
127 net::AddressList addresses_; | 97 net::AddressList addresses_; |
128 | 98 |
129 DISALLOW_COPY_AND_ASSIGN(LookupRequest); | 99 DISALLOW_COPY_AND_ASSIGN(LookupRequest); |
130 }; | 100 }; |
131 | 101 |
132 Predictor::Predictor(bool preconnect_enabled) | 102 Predictor::Predictor(net::HostResolver* host_resolver, |
133 : initial_observer_(NULL), | 103 TimeDelta max_dns_queue_delay, |
134 predictor_enabled_(true), | 104 size_t max_concurrent, |
135 peak_pending_lookups_(0), | 105 bool preconnect_enabled) |
| 106 : peak_pending_lookups_(0), |
136 shutdown_(false), | 107 shutdown_(false), |
137 max_concurrent_dns_lookups_(g_max_parallel_resolves), | 108 max_concurrent_dns_lookups_(max_concurrent), |
138 max_dns_queue_delay_( | 109 max_dns_queue_delay_(max_dns_queue_delay), |
139 TimeDelta::FromMilliseconds(g_max_queueing_delay_ms)), | 110 host_resolver_(host_resolver), |
140 host_resolver_(NULL), | |
141 preconnect_enabled_(preconnect_enabled), | 111 preconnect_enabled_(preconnect_enabled), |
142 consecutive_omnibox_preconnect_count_(0), | 112 consecutive_omnibox_preconnect_count_(0), |
143 next_trim_time_(base::TimeTicks::Now() + kDurationBetweenTrimmings) { | 113 next_trim_time_(base::TimeTicks::Now() + kDurationBetweenTrimmings), |
144 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | 114 ALLOW_THIS_IN_INITIALIZER_LIST(trim_task_factory_(this)) { |
145 } | 115 } |
146 | 116 |
147 Predictor::~Predictor() { | 117 Predictor::~Predictor() { |
148 // TODO(rlp): Add DCHECK for CurrentlyOn(BrowserThread::IO) when the | |
149 // ProfileManagerTest has been updated with a mock profile. | |
150 DCHECK(shutdown_); | 118 DCHECK(shutdown_); |
151 } | 119 } |
152 | 120 |
153 // static | 121 void Predictor::Shutdown() { |
154 Predictor* Predictor::CreatePredictor( | 122 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
155 bool preconnect_enabled, bool simple_shutdown) { | 123 DCHECK(!shutdown_); |
156 if (simple_shutdown) | 124 shutdown_ = true; |
157 return new SimpleShutdownPredictor(preconnect_enabled); | 125 |
158 return new Predictor(preconnect_enabled); | 126 std::set<LookupRequest*>::iterator it; |
| 127 for (it = pending_lookups_.begin(); it != pending_lookups_.end(); ++it) |
| 128 delete *it; |
159 } | 129 } |
160 | 130 |
161 void Predictor::RegisterUserPrefs(PrefService* user_prefs) { | 131 // Overloaded Resolve() to take a vector of names. |
162 user_prefs->RegisterListPref(prefs::kDnsPrefetchingStartupList, | 132 void Predictor::ResolveList(const UrlList& urls, |
163 PrefService::UNSYNCABLE_PREF); | 133 UrlInfo::ResolutionMotivation motivation) { |
164 user_prefs->RegisterListPref(prefs::kDnsPrefetchingHostReferralList, | 134 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
165 PrefService::UNSYNCABLE_PREF); | 135 |
| 136 for (UrlList::const_iterator it = urls.begin(); it < urls.end(); ++it) { |
| 137 AppendToResolutionQueue(*it, motivation); |
| 138 } |
166 } | 139 } |
167 | 140 |
168 // --------------------- Start UI methods. ------------------------------------ | 141 // Basic Resolve() takes an invidual name, and adds it |
169 | 142 // to the queue. |
170 void Predictor::InitNetworkPredictor(PrefService* user_prefs, | 143 void Predictor::Resolve(const GURL& url, |
171 PrefService* local_state, | 144 UrlInfo::ResolutionMotivation motivation) { |
172 IOThread* io_thread) { | 145 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
173 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | 146 if (!url.has_host()) |
174 | 147 return; |
175 predictor_enabled_ = user_prefs->GetBoolean(prefs::kNetworkPredictionEnabled); | 148 AppendToResolutionQueue(url, motivation); |
176 | |
177 // Gather the list of hostnames to prefetch on startup. | |
178 UrlList urls = GetPredictedUrlListAtStartup(user_prefs, local_state); | |
179 | |
180 base::ListValue* referral_list = | |
181 static_cast<base::ListValue*>(user_prefs->GetList( | |
182 prefs::kDnsPrefetchingHostReferralList)->DeepCopy()); | |
183 | |
184 // Remove obsolete preferences from local state if necessary. | |
185 int current_version = | |
186 local_state->GetInteger(prefs::kMultipleProfilePrefMigration); | |
187 if ((current_version & browser::DNS_PREFS) == 0) { | |
188 local_state->RegisterListPref(prefs::kDnsStartupPrefetchList, | |
189 PrefService::UNSYNCABLE_PREF); | |
190 local_state->RegisterListPref(prefs::kDnsHostReferralList, | |
191 PrefService::UNSYNCABLE_PREF); | |
192 local_state->ClearPref(prefs::kDnsStartupPrefetchList); | |
193 local_state->ClearPref(prefs::kDnsHostReferralList); | |
194 local_state->SetInteger(prefs::kMultipleProfilePrefMigration, | |
195 current_version | browser::DNS_PREFS); | |
196 } | |
197 | |
198 BrowserThread::PostTask( | |
199 BrowserThread::IO, | |
200 FROM_HERE, | |
201 base::Bind( | |
202 &Predictor::FinalizeInitializationOnIOThread, | |
203 base::Unretained(this), | |
204 urls, referral_list, | |
205 io_thread)); | |
206 } | 149 } |
207 | 150 |
| 151 void Predictor::LearnFromNavigation(const GURL& referring_url, |
| 152 const GURL& target_url) { |
| 153 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
| 154 DCHECK_EQ(referring_url, Predictor::CanonicalizeUrl(referring_url)); |
| 155 DCHECK_NE(referring_url, GURL::EmptyGURL()); |
| 156 DCHECK_EQ(target_url, Predictor::CanonicalizeUrl(target_url)); |
| 157 DCHECK_NE(target_url, GURL::EmptyGURL()); |
| 158 |
| 159 referrers_[referring_url].SuggestHost(target_url); |
| 160 // Possibly do some referrer trimming. |
| 161 TrimReferrers(); |
| 162 } |
| 163 |
| 164 enum SubresourceValue { |
| 165 PRECONNECTION, |
| 166 PRERESOLUTION, |
| 167 TOO_NEW, |
| 168 SUBRESOURCE_VALUE_MAX |
| 169 }; |
| 170 |
208 void Predictor::AnticipateOmniboxUrl(const GURL& url, bool preconnectable) { | 171 void Predictor::AnticipateOmniboxUrl(const GURL& url, bool preconnectable) { |
209 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
210 if (!predictor_enabled_) | |
211 return; | |
212 if (!url.is_valid() || !url.has_host()) | |
213 return; | |
214 std::string host = url.HostNoBrackets(); | 172 std::string host = url.HostNoBrackets(); |
215 bool is_new_host_request = (host != last_omnibox_host_); | 173 bool is_new_host_request = (host != last_omnibox_host_); |
216 last_omnibox_host_ = host; | 174 last_omnibox_host_ = host; |
217 | 175 |
218 UrlInfo::ResolutionMotivation motivation(UrlInfo::OMNIBOX_MOTIVATED); | 176 UrlInfo::ResolutionMotivation motivation(UrlInfo::OMNIBOX_MOTIVATED); |
219 base::TimeTicks now = base::TimeTicks::Now(); | 177 base::TimeTicks now = base::TimeTicks::Now(); |
220 | 178 |
221 if (preconnect_enabled()) { | 179 if (preconnect_enabled()) { |
222 if (preconnectable && !is_new_host_request) { | 180 if (preconnectable && !is_new_host_request) { |
223 ++consecutive_omnibox_preconnect_count_; | 181 ++consecutive_omnibox_preconnect_count_; |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
267 const int kMinPreresolveSeconds(10); | 225 const int kMinPreresolveSeconds(10); |
268 if (kMinPreresolveSeconds > (now - last_omnibox_preresolve_).InSeconds()) | 226 if (kMinPreresolveSeconds > (now - last_omnibox_preresolve_).InSeconds()) |
269 return; | 227 return; |
270 } | 228 } |
271 last_omnibox_preresolve_ = now; | 229 last_omnibox_preresolve_ = now; |
272 | 230 |
273 // Perform at least DNS pre-resolution. | 231 // Perform at least DNS pre-resolution. |
274 BrowserThread::PostTask( | 232 BrowserThread::PostTask( |
275 BrowserThread::IO, | 233 BrowserThread::IO, |
276 FROM_HERE, | 234 FROM_HERE, |
277 base::Bind(&Predictor::Resolve, base::Unretained(this), | 235 NewRunnableMethod(this, &Predictor::Resolve, CanonicalizeUrl(url), |
278 CanonicalizeUrl(url), motivation)); | 236 motivation)); |
279 } | 237 } |
280 | 238 |
281 void Predictor::PreconnectUrlAndSubresources(const GURL& url) { | 239 void Predictor::PreconnectUrlAndSubresources(const GURL& url) { |
282 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
283 if (!predictor_enabled_) | |
284 return; | |
285 if (!url.is_valid() || !url.has_host()) | |
286 return; | |
287 if (preconnect_enabled()) { | 240 if (preconnect_enabled()) { |
288 std::string host = url.HostNoBrackets(); | 241 std::string host = url.HostNoBrackets(); |
289 UrlInfo::ResolutionMotivation motivation(UrlInfo::EARLY_LOAD_MOTIVATED); | 242 UrlInfo::ResolutionMotivation motivation(UrlInfo::EARLY_LOAD_MOTIVATED); |
290 const int kConnectionsNeeded = 1; | 243 const int kConnectionsNeeded = 1; |
291 PreconnectOnUIThread(CanonicalizeUrl(url), motivation, | 244 PreconnectOnUIThread(CanonicalizeUrl(url), motivation, |
292 kConnectionsNeeded); | 245 kConnectionsNeeded); |
293 PredictFrameSubresources(url.GetWithEmptyPath()); | 246 PredictFrameSubresources(url.GetWithEmptyPath()); |
294 } | 247 } |
295 } | 248 } |
296 | 249 |
297 UrlList Predictor::GetPredictedUrlListAtStartup( | 250 void Predictor::PredictFrameSubresources(const GURL& url) { |
298 PrefService* user_prefs, | 251 DCHECK_EQ(url.GetWithEmptyPath(), url); |
299 PrefService* local_state) { | 252 // Add one pass through the message loop to allow current navigation to |
300 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | 253 // proceed. |
301 UrlList urls; | |
302 // Recall list of URLs we learned about during last session. | |
303 // This may catch secondary hostnames, pulled in by the homepages. It will | |
304 // also catch more of the "primary" home pages, since that was (presumably) | |
305 // rendered first (and will be rendered first this time too). | |
306 const ListValue* startup_list = | |
307 user_prefs->GetList(prefs::kDnsPrefetchingStartupList); | |
308 | |
309 if (startup_list) { | |
310 base::ListValue::const_iterator it = startup_list->begin(); | |
311 int format_version = -1; | |
312 if (it != startup_list->end() && | |
313 (*it)->GetAsInteger(&format_version) && | |
314 format_version == kPredictorStartupFormatVersion) { | |
315 ++it; | |
316 for (; it != startup_list->end(); ++it) { | |
317 std::string url_spec; | |
318 if (!(*it)->GetAsString(&url_spec)) { | |
319 LOG(DFATAL); | |
320 break; // Format incompatibility. | |
321 } | |
322 GURL url(url_spec); | |
323 if (!url.has_host() || !url.has_scheme()) { | |
324 LOG(DFATAL); | |
325 break; // Format incompatibility. | |
326 } | |
327 | |
328 urls.push_back(url); | |
329 } | |
330 } | |
331 } | |
332 | |
333 // Prepare for any static home page(s) the user has in prefs. The user may | |
334 // have a LOT of tab's specified, so we may as well try to warm them all. | |
335 SessionStartupPref tab_start_pref = | |
336 SessionStartupPref::GetStartupPref(user_prefs); | |
337 if (SessionStartupPref::URLS == tab_start_pref.type) { | |
338 for (size_t i = 0; i < tab_start_pref.urls.size(); i++) { | |
339 GURL gurl = tab_start_pref.urls[i]; | |
340 if (!gurl.is_valid() || gurl.SchemeIsFile() || gurl.host().empty()) | |
341 continue; | |
342 if (gurl.SchemeIs("http") || gurl.SchemeIs("https")) | |
343 urls.push_back(gurl.GetWithEmptyPath()); | |
344 } | |
345 } | |
346 | |
347 if (urls.empty()) | |
348 urls.push_back(GURL("http://www.google.com:80")); | |
349 | |
350 return urls; | |
351 } | |
352 | |
353 void Predictor::set_max_queueing_delay(int max_queueing_delay_ms) { | |
354 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
355 g_max_queueing_delay_ms = max_queueing_delay_ms; | |
356 } | |
357 | |
358 void Predictor::set_max_parallel_resolves(size_t max_parallel_resolves) { | |
359 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
360 g_max_parallel_resolves = max_parallel_resolves; | |
361 } | |
362 | |
363 void Predictor::ShutdownOnUIThread(PrefService* user_prefs) { | |
364 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI)); | |
365 | |
366 SaveStateForNextStartupAndTrim(user_prefs); | |
367 | |
368 BrowserThread::PostTask( | 254 BrowserThread::PostTask( |
369 BrowserThread::IO, | 255 BrowserThread::IO, |
370 FROM_HERE, | 256 FROM_HERE, |
371 base::Bind(&Predictor::Shutdown, base::Unretained(this))); | 257 NewRunnableMethod(this, &Predictor::PrepareFrameSubresources, url)); |
372 } | 258 } |
373 | 259 |
374 // ---------------------- End UI methods. ------------------------------------- | 260 void Predictor::PrepareFrameSubresources(const GURL& url) { |
| 261 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
| 262 DCHECK_EQ(url.GetWithEmptyPath(), url); |
| 263 Referrers::iterator it = referrers_.find(url); |
| 264 if (referrers_.end() == it) { |
| 265 // Only when we don't know anything about this url, make 2 connections |
| 266 // available. We could do this completely via learning (by prepopulating |
| 267 // the referrer_ list with this expected value), but it would swell the |
| 268 // size of the list with all the "Leaf" nodes in the tree (nodes that don't |
| 269 // load any subresources). If we learn about this resource, we will instead |
| 270 // provide a more carefully estimated preconnection count. |
| 271 if (preconnect_enabled_) |
| 272 PreconnectOnIOThread(url, UrlInfo::SELF_REFERAL_MOTIVATED, 2); |
| 273 return; |
| 274 } |
375 | 275 |
376 // --------------------- Start IO methods. ------------------------------------ | 276 Referrer* referrer = &(it->second); |
377 | 277 referrer->IncrementUseCount(); |
378 void Predictor::Shutdown() { | 278 const UrlInfo::ResolutionMotivation motivation = |
379 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 279 UrlInfo::LEARNED_REFERAL_MOTIVATED; |
380 DCHECK(!shutdown_); | 280 for (Referrer::iterator future_url = referrer->begin(); |
381 shutdown_ = true; | 281 future_url != referrer->end(); ++future_url) { |
382 | 282 SubresourceValue evalution(TOO_NEW); |
383 STLDeleteElements(&pending_lookups_); | 283 double connection_expectation = future_url->second.subresource_use_rate(); |
384 } | 284 UMA_HISTOGRAM_CUSTOM_COUNTS("Net.PreconnectSubresourceExpectation", |
385 | 285 static_cast<int>(connection_expectation * 100), |
386 void Predictor::DiscardAllResults() { | 286 10, 5000, 50); |
387 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 287 future_url->second.ReferrerWasObserved(); |
388 // Delete anything listed so far in this session that shows in about:dns. | 288 if (preconnect_enabled_ && |
389 referrers_.clear(); | 289 connection_expectation > kPreconnectWorthyExpectedValue) { |
390 | 290 evalution = PRECONNECTION; |
391 | 291 future_url->second.IncrementPreconnectionCount(); |
392 // Try to delete anything in our work queue. | 292 int count = static_cast<int>(std::ceil(connection_expectation)); |
393 while (!work_queue_.IsEmpty()) { | 293 if (url.host() == future_url->first.host()) |
394 // Emulate processing cycle as though host was not found. | 294 ++count; |
395 GURL url = work_queue_.Pop(); | 295 PreconnectOnIOThread(future_url->first, motivation, count); |
396 UrlInfo* info = &results_[url]; | 296 } else if (connection_expectation > kDNSPreresolutionWorthyExpectedValue) { |
397 DCHECK(info->HasUrl(url)); | 297 evalution = PRERESOLUTION; |
398 info->SetAssignedState(); | 298 future_url->second.preresolution_increment(); |
399 info->SetNoSuchNameState(); | 299 UrlInfo* queued_info = AppendToResolutionQueue(future_url->first, |
| 300 motivation); |
| 301 if (queued_info) |
| 302 queued_info->SetReferringHostname(url); |
| 303 } |
| 304 UMA_HISTOGRAM_ENUMERATION("Net.PreconnectSubresourceEval", evalution, |
| 305 SUBRESOURCE_VALUE_MAX); |
400 } | 306 } |
401 // Now every result_ is either resolved, or is being resolved | |
402 // (see LookupRequest). | |
403 | |
404 // Step through result_, recording names of all hosts that can't be erased. | |
405 // We can't erase anything being worked on. | |
406 Results assignees; | |
407 for (Results::iterator it = results_.begin(); results_.end() != it; ++it) { | |
408 GURL url(it->first); | |
409 UrlInfo* info = &it->second; | |
410 DCHECK(info->HasUrl(url)); | |
411 if (info->is_assigned()) { | |
412 info->SetPendingDeleteState(); | |
413 assignees[url] = *info; | |
414 } | |
415 } | |
416 DCHECK_LE(assignees.size(), max_concurrent_dns_lookups_); | |
417 results_.clear(); | |
418 // Put back in the names being worked on. | |
419 for (Results::iterator it = assignees.begin(); assignees.end() != it; ++it) { | |
420 DCHECK(it->second.is_marked_to_delete()); | |
421 results_[it->first] = it->second; | |
422 } | |
423 } | |
424 | |
425 // Overloaded Resolve() to take a vector of names. | |
426 void Predictor::ResolveList(const UrlList& urls, | |
427 UrlInfo::ResolutionMotivation motivation) { | |
428 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
429 | |
430 for (UrlList::const_iterator it = urls.begin(); it < urls.end(); ++it) { | |
431 AppendToResolutionQueue(*it, motivation); | |
432 } | |
433 } | |
434 | |
435 // Basic Resolve() takes an invidual name, and adds it | |
436 // to the queue. | |
437 void Predictor::Resolve(const GURL& url, | |
438 UrlInfo::ResolutionMotivation motivation) { | |
439 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
440 if (!url.has_host()) | |
441 return; | |
442 AppendToResolutionQueue(url, motivation); | |
443 } | |
444 | |
445 void Predictor::LearnFromNavigation(const GURL& referring_url, | |
446 const GURL& target_url) { | |
447 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
448 if (!predictor_enabled_) | |
449 return; | |
450 DCHECK_EQ(referring_url, Predictor::CanonicalizeUrl(referring_url)); | |
451 DCHECK_NE(referring_url, GURL::EmptyGURL()); | |
452 DCHECK_EQ(target_url, Predictor::CanonicalizeUrl(target_url)); | |
453 DCHECK_NE(target_url, GURL::EmptyGURL()); | |
454 | |
455 referrers_[referring_url].SuggestHost(target_url); | |
456 // Possibly do some referrer trimming. | |
457 TrimReferrers(); | |
458 } | |
459 | |
460 //----------------------------------------------------------------------------- | |
461 // This section supports the about:dns page. | |
462 | |
463 void Predictor::PredictorGetHtmlInfo(Predictor* predictor, | |
464 std::string* output) { | |
465 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
466 | |
467 output->append("<html><head><title>About DNS</title>" | |
468 // We'd like the following no-cache... but it doesn't work. | |
469 // "<META HTTP-EQUIV=\"Pragma\" CONTENT=\"no-cache\">" | |
470 "</head><body>"); | |
471 if (predictor && predictor->predictor_enabled()) { | |
472 predictor->GetHtmlInfo(output); | |
473 } else { | |
474 output->append("DNS pre-resolution and TCP pre-connection is disabled."); | |
475 } | |
476 output->append("</body></html>"); | |
477 } | 307 } |
478 | 308 |
479 // Provide sort order so all .com's are together, etc. | 309 // Provide sort order so all .com's are together, etc. |
480 struct RightToLeftStringSorter { | 310 struct RightToLeftStringSorter { |
481 bool operator()(const GURL& left, | 311 bool operator()(const GURL& left, |
482 const GURL& right) const { | 312 const GURL& right) const { |
483 return string_compare(left.host(), right.host()); | 313 return string_compare(left.host(), right.host()); |
484 } | 314 } |
485 | 315 |
486 static bool string_compare(const std::string& left_host, | 316 static bool string_compare(const std::string& left_host, |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
581 static_cast<int>(future_url->second.preresolution_count()), | 411 static_cast<int>(future_url->second.preresolution_count()), |
582 static_cast<double>(future_url->second.subresource_use_rate()), | 412 static_cast<double>(future_url->second.subresource_use_rate()), |
583 future_url->first.spec().c_str()); | 413 future_url->first.spec().c_str()); |
584 } | 414 } |
585 } | 415 } |
586 output->append("</table>"); | 416 output->append("</table>"); |
587 } | 417 } |
588 | 418 |
589 void Predictor::GetHtmlInfo(std::string* output) { | 419 void Predictor::GetHtmlInfo(std::string* output) { |
590 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 420 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
591 if (initial_observer_.get()) | |
592 initial_observer_->GetFirstResolutionsHtml(output); | |
593 // Show list of subresource predictions and stats. | |
594 GetHtmlReferrerLists(output); | |
595 | |
596 // Local lists for calling UrlInfo | 421 // Local lists for calling UrlInfo |
597 UrlInfo::UrlInfoTable name_not_found; | 422 UrlInfo::UrlInfoTable name_not_found; |
598 UrlInfo::UrlInfoTable name_preresolved; | 423 UrlInfo::UrlInfoTable name_preresolved; |
599 | 424 |
600 // Get copies of all useful data. | 425 // Get copies of all useful data. |
601 typedef std::map<GURL, UrlInfo, RightToLeftStringSorter> SortedUrlInfo; | 426 typedef std::map<GURL, UrlInfo, RightToLeftStringSorter> SortedUrlInfo; |
602 SortedUrlInfo snapshot; | 427 SortedUrlInfo snapshot; |
603 // UrlInfo supports value semantics, so we can do a shallow copy. | 428 // UrlInfo supports value semantics, so we can do a shallow copy. |
604 for (Results::iterator it(results_.begin()); it != results_.end(); it++) | 429 for (Results::iterator it(results_.begin()); it != results_.end(); it++) |
605 snapshot[it->first] = it->second; | 430 snapshot[it->first] = it->second; |
(...skipping 15 matching lines...) Expand all Loading... |
621 brief = true; | 446 brief = true; |
622 #endif // NDEBUG | 447 #endif // NDEBUG |
623 | 448 |
624 // Call for display of each table, along with title. | 449 // Call for display of each table, along with title. |
625 UrlInfo::GetHtmlTable(name_preresolved, | 450 UrlInfo::GetHtmlTable(name_preresolved, |
626 "Preresolution DNS records performed for ", brief, output); | 451 "Preresolution DNS records performed for ", brief, output); |
627 UrlInfo::GetHtmlTable(name_not_found, | 452 UrlInfo::GetHtmlTable(name_not_found, |
628 "Preresolving DNS records revealed non-existence for ", brief, output); | 453 "Preresolving DNS records revealed non-existence for ", brief, output); |
629 } | 454 } |
630 | 455 |
631 void Predictor::TrimReferrersNow() { | 456 UrlInfo* Predictor::AppendToResolutionQueue( |
| 457 const GURL& url, |
| 458 UrlInfo::ResolutionMotivation motivation) { |
632 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 459 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
633 // Just finish up work if an incremental trim is in progress. | 460 DCHECK(url.has_host()); |
634 if (urls_being_trimmed_.empty()) | 461 |
635 LoadUrlsForTrimming(); | 462 if (shutdown_) |
636 IncrementalTrimReferrers(true); // Do everything now. | 463 return NULL; |
| 464 |
| 465 UrlInfo* info = &results_[url]; |
| 466 info->SetUrl(url); // Initialize or DCHECK. |
| 467 // TODO(jar): I need to discard names that have long since expired. |
| 468 // Currently we only add to the domain map :-/ |
| 469 |
| 470 DCHECK(info->HasUrl(url)); |
| 471 |
| 472 if (!info->NeedsDnsUpdate()) { |
| 473 info->DLogResultsStats("DNS PrefetchNotUpdated"); |
| 474 return NULL; |
| 475 } |
| 476 |
| 477 info->SetQueuedState(motivation); |
| 478 work_queue_.Push(url, motivation); |
| 479 StartSomeQueuedResolutions(); |
| 480 return info; |
637 } | 481 } |
638 | 482 |
639 void Predictor::SerializeReferrers(base::ListValue* referral_list) { | 483 void Predictor::StartSomeQueuedResolutions() { |
640 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 484 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
641 referral_list->Clear(); | |
642 referral_list->Append(new base::FundamentalValue(PREDICTOR_REFERRER_VERSION)); | |
643 for (Referrers::const_iterator it = referrers_.begin(); | |
644 it != referrers_.end(); ++it) { | |
645 // Serialize the list of subresource names. | |
646 Value* subresource_list(it->second.Serialize()); | |
647 | 485 |
648 // Create a list for each referer. | 486 while (!work_queue_.IsEmpty() && |
649 ListValue* motivator(new ListValue); | 487 pending_lookups_.size() < max_concurrent_dns_lookups_) { |
650 motivator->Append(new StringValue(it->first.spec())); | 488 const GURL url(work_queue_.Pop()); |
651 motivator->Append(subresource_list); | 489 UrlInfo* info = &results_[url]; |
| 490 DCHECK(info->HasUrl(url)); |
| 491 info->SetAssignedState(); |
652 | 492 |
653 referral_list->Append(motivator); | 493 if (CongestionControlPerformed(info)) { |
654 } | 494 DCHECK(work_queue_.IsEmpty()); |
655 } | 495 return; |
| 496 } |
656 | 497 |
657 void Predictor::DeserializeReferrers(const base::ListValue& referral_list) { | 498 LookupRequest* request = new LookupRequest(this, host_resolver_, url); |
658 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 499 int status = request->Start(); |
659 int format_version = -1; | 500 if (status == net::ERR_IO_PENDING) { |
660 if (referral_list.GetSize() > 0 && | 501 // Will complete asynchronously. |
661 referral_list.GetInteger(0, &format_version) && | 502 pending_lookups_.insert(request); |
662 format_version == PREDICTOR_REFERRER_VERSION) { | 503 peak_pending_lookups_ = std::max(peak_pending_lookups_, |
663 for (size_t i = 1; i < referral_list.GetSize(); ++i) { | 504 pending_lookups_.size()); |
664 base::ListValue* motivator; | 505 } else { |
665 if (!referral_list.GetList(i, &motivator)) { | 506 // Completed synchronously (was already cached by HostResolver), or else |
666 NOTREACHED(); | 507 // there was (equivalently) some network error that prevents us from |
667 return; | 508 // finding the name. Status net::OK means it was "found." |
668 } | 509 LookupFinished(request, url, status == net::OK); |
669 std::string motivating_url_spec; | 510 delete request; |
670 if (!motivator->GetString(0, &motivating_url_spec)) { | |
671 NOTREACHED(); | |
672 return; | |
673 } | |
674 | |
675 Value* subresource_list; | |
676 if (!motivator->Get(1, &subresource_list)) { | |
677 NOTREACHED(); | |
678 return; | |
679 } | |
680 | |
681 referrers_[GURL(motivating_url_spec)].Deserialize(*subresource_list); | |
682 } | 511 } |
683 } | 512 } |
684 } | 513 } |
685 | 514 |
686 void Predictor::DeserializeReferrersThenDelete( | 515 bool Predictor::CongestionControlPerformed(UrlInfo* info) { |
687 base::ListValue* referral_list) { | |
688 DeserializeReferrers(*referral_list); | |
689 delete referral_list; | |
690 } | |
691 | |
692 void Predictor::DiscardInitialNavigationHistory() { | |
693 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 516 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
694 if (initial_observer_.get()) | 517 // Note: queue_duration is ONLY valid after we go to assigned state. |
695 initial_observer_->DiscardInitialNavigationHistory(); | 518 if (info->queue_duration() < max_dns_queue_delay_) |
696 } | 519 return false; |
697 | 520 // We need to discard all entries in our queue, as we're keeping them waiting |
698 void Predictor::FinalizeInitializationOnIOThread( | 521 // too long. By doing this, we'll have a chance to quickly service urgent |
699 const UrlList& startup_urls, | 522 // resolutions, and not have a bogged down system. |
700 base::ListValue* referral_list, | 523 while (true) { |
701 IOThread* io_thread) { | 524 info->RemoveFromQueue(); |
702 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 525 if (work_queue_.IsEmpty()) |
703 | 526 break; |
704 initial_observer_.reset(new InitialObserver()); | 527 info = &results_[work_queue_.Pop()]; |
705 host_resolver_ = io_thread->globals()->host_resolver.get(); | 528 info->SetAssignedState(); |
706 | |
707 // ScopedRunnableMethodFactory instances need to be created and destroyed | |
708 // on the same thread. The predictor lives on the IO thread and will die | |
709 // from there so now that we're on the IO thread we need to properly | |
710 // initialize the ScopedrunnableMethodFactory. | |
711 trim_task_factory_.reset(new ScopedRunnableMethodFactory<Predictor>(this)); | |
712 | |
713 // Prefetch these hostnames on startup. | |
714 DnsPrefetchMotivatedList(startup_urls, UrlInfo::STARTUP_LIST_MOTIVATED); | |
715 DeserializeReferrersThenDelete(referral_list); | |
716 } | |
717 | |
718 //----------------------------------------------------------------------------- | |
719 // This section intermingles prefetch results with actual browser HTTP | |
720 // network activity. It supports calculating of the benefit of a prefetch, as | |
721 // well as recording what prefetched hostname resolutions might be potentially | |
722 // helpful during the next chrome-startup. | |
723 //----------------------------------------------------------------------------- | |
724 | |
725 void Predictor::LearnAboutInitialNavigation(const GURL& url) { | |
726 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
727 if (!predictor_enabled_ || NULL == initial_observer_.get() ) | |
728 return; | |
729 initial_observer_->Append(url, this); | |
730 } | |
731 | |
732 // This API is only used in the browser process. | |
733 // It is called from an IPC message originating in the renderer. It currently | |
734 // includes both Page-Scan, and Link-Hover prefetching. | |
735 // TODO(jar): Separate out link-hover prefetching, and page-scan results. | |
736 void Predictor::DnsPrefetchList(const NameList& hostnames) { | |
737 // TODO(jar): Push GURL transport further back into renderer, but this will | |
738 // require a Webkit change in the observer :-/. | |
739 UrlList urls; | |
740 for (NameList::const_iterator it = hostnames.begin(); | |
741 it < hostnames.end(); | |
742 ++it) { | |
743 urls.push_back(GURL("http://" + *it + ":80")); | |
744 } | 529 } |
745 | 530 return true; |
746 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
747 DnsPrefetchMotivatedList(urls, UrlInfo::PAGE_SCAN_MOTIVATED); | |
748 } | |
749 | |
750 void Predictor::DnsPrefetchMotivatedList( | |
751 const UrlList& urls, | |
752 UrlInfo::ResolutionMotivation motivation) { | |
753 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI) || | |
754 BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
755 if (!predictor_enabled_) | |
756 return; | |
757 | |
758 if (BrowserThread::CurrentlyOn(BrowserThread::IO)) { | |
759 ResolveList(urls, motivation); | |
760 } else { | |
761 BrowserThread::PostTask( | |
762 BrowserThread::IO, | |
763 FROM_HERE, | |
764 base::Bind(&Predictor::ResolveList, base::Unretained(this), | |
765 urls, motivation)); | |
766 } | |
767 } | |
768 | |
769 //----------------------------------------------------------------------------- | |
770 // Functions to handle saving of hostnames from one session to the next, to | |
771 // expedite startup times. | |
772 | |
773 static void SaveDnsPrefetchStateForNextStartupAndTrimOnIOThread( | |
774 base::ListValue* startup_list, | |
775 base::ListValue* referral_list, | |
776 base::WaitableEvent* completion, | |
777 Predictor* predictor) { | |
778 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
779 | |
780 if (NULL == predictor) { | |
781 completion->Signal(); | |
782 return; | |
783 } | |
784 predictor->SaveDnsPrefetchStateForNextStartupAndTrim( | |
785 startup_list, referral_list, completion); | |
786 } | |
787 | |
788 void Predictor::SaveStateForNextStartupAndTrim(PrefService* prefs) { | |
789 if (!predictor_enabled_) | |
790 return; | |
791 | |
792 base::WaitableEvent completion(true, false); | |
793 | |
794 ListPrefUpdate update_startup_list(prefs, prefs::kDnsPrefetchingStartupList); | |
795 ListPrefUpdate update_referral_list(prefs, | |
796 prefs::kDnsPrefetchingHostReferralList); | |
797 if (BrowserThread::CurrentlyOn(BrowserThread::IO)) { | |
798 SaveDnsPrefetchStateForNextStartupAndTrimOnIOThread( | |
799 update_startup_list.Get(), | |
800 update_referral_list.Get(), | |
801 &completion, | |
802 this); | |
803 } else { | |
804 bool posted = BrowserThread::PostTask( | |
805 BrowserThread::IO, | |
806 FROM_HERE, | |
807 base::Bind( | |
808 SaveDnsPrefetchStateForNextStartupAndTrimOnIOThread, | |
809 update_startup_list.Get(), | |
810 update_referral_list.Get(), | |
811 &completion, | |
812 this)); | |
813 | |
814 // TODO(jar): Synchronous waiting for the IO thread is a potential source | |
815 // to deadlocks and should be investigated. See http://crbug.com/78451. | |
816 DCHECK(posted); | |
817 if (posted) | |
818 completion.Wait(); | |
819 } | |
820 } | |
821 | |
822 void Predictor::SaveDnsPrefetchStateForNextStartupAndTrim( | |
823 base::ListValue* startup_list, | |
824 base::ListValue* referral_list, | |
825 base::WaitableEvent* completion) { | |
826 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
827 if (initial_observer_.get()) | |
828 initial_observer_->GetInitialDnsResolutionList(startup_list); | |
829 | |
830 // Do at least one trim at shutdown, in case the user wasn't running long | |
831 // enough to do any regular trimming of referrers. | |
832 TrimReferrersNow(); | |
833 SerializeReferrers(referral_list); | |
834 | |
835 completion->Signal(); | |
836 } | |
837 | |
838 void Predictor::EnablePredictor(bool enable) { | |
839 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI) || | |
840 BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
841 | |
842 if (BrowserThread::CurrentlyOn(BrowserThread::IO)) { | |
843 EnablePredictorOnIOThread(enable); | |
844 } else { | |
845 BrowserThread::PostTask( | |
846 BrowserThread::IO, | |
847 FROM_HERE, | |
848 base::Bind(&Predictor::EnablePredictorOnIOThread, | |
849 base::Unretained(this), enable)); | |
850 } | |
851 } | |
852 | |
853 void Predictor::EnablePredictorOnIOThread(bool enable) { | |
854 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
855 predictor_enabled_ = enable; | |
856 } | |
857 | |
858 void Predictor::PredictFrameSubresources(const GURL& url) { | |
859 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI) || | |
860 BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
861 if (!predictor_enabled_) | |
862 return; | |
863 DCHECK_EQ(url.GetWithEmptyPath(), url); | |
864 // Add one pass through the message loop to allow current navigation to | |
865 // proceed. | |
866 if (BrowserThread::CurrentlyOn(BrowserThread::IO)) { | |
867 PrepareFrameSubresources(url); | |
868 } else { | |
869 BrowserThread::PostTask( | |
870 BrowserThread::IO, | |
871 FROM_HERE, | |
872 base::Bind(&Predictor::PrepareFrameSubresources, | |
873 base::Unretained(this), url)); | |
874 } | |
875 } | |
876 | |
877 enum SubresourceValue { | |
878 PRECONNECTION, | |
879 PRERESOLUTION, | |
880 TOO_NEW, | |
881 SUBRESOURCE_VALUE_MAX | |
882 }; | |
883 | |
884 void Predictor::PrepareFrameSubresources(const GURL& url) { | |
885 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
886 DCHECK_EQ(url.GetWithEmptyPath(), url); | |
887 Referrers::iterator it = referrers_.find(url); | |
888 if (referrers_.end() == it) { | |
889 // Only when we don't know anything about this url, make 2 connections | |
890 // available. We could do this completely via learning (by prepopulating | |
891 // the referrer_ list with this expected value), but it would swell the | |
892 // size of the list with all the "Leaf" nodes in the tree (nodes that don't | |
893 // load any subresources). If we learn about this resource, we will instead | |
894 // provide a more carefully estimated preconnection count. | |
895 if (preconnect_enabled_) | |
896 PreconnectOnIOThread(url, UrlInfo::SELF_REFERAL_MOTIVATED, 2); | |
897 return; | |
898 } | |
899 | |
900 Referrer* referrer = &(it->second); | |
901 referrer->IncrementUseCount(); | |
902 const UrlInfo::ResolutionMotivation motivation = | |
903 UrlInfo::LEARNED_REFERAL_MOTIVATED; | |
904 for (Referrer::iterator future_url = referrer->begin(); | |
905 future_url != referrer->end(); ++future_url) { | |
906 SubresourceValue evalution(TOO_NEW); | |
907 double connection_expectation = future_url->second.subresource_use_rate(); | |
908 UMA_HISTOGRAM_CUSTOM_COUNTS("Net.PreconnectSubresourceExpectation", | |
909 static_cast<int>(connection_expectation * 100), | |
910 10, 5000, 50); | |
911 future_url->second.ReferrerWasObserved(); | |
912 if (preconnect_enabled_ && | |
913 connection_expectation > kPreconnectWorthyExpectedValue) { | |
914 evalution = PRECONNECTION; | |
915 future_url->second.IncrementPreconnectionCount(); | |
916 int count = static_cast<int>(std::ceil(connection_expectation)); | |
917 if (url.host() == future_url->first.host()) | |
918 ++count; | |
919 PreconnectOnIOThread(future_url->first, motivation, count); | |
920 } else if (connection_expectation > kDNSPreresolutionWorthyExpectedValue) { | |
921 evalution = PRERESOLUTION; | |
922 future_url->second.preresolution_increment(); | |
923 UrlInfo* queued_info = AppendToResolutionQueue(future_url->first, | |
924 motivation); | |
925 if (queued_info) | |
926 queued_info->SetReferringHostname(url); | |
927 } | |
928 UMA_HISTOGRAM_ENUMERATION("Net.PreconnectSubresourceEval", evalution, | |
929 SUBRESOURCE_VALUE_MAX); | |
930 } | |
931 } | 531 } |
932 | 532 |
933 void Predictor::OnLookupFinished(LookupRequest* request, const GURL& url, | 533 void Predictor::OnLookupFinished(LookupRequest* request, const GURL& url, |
934 bool found) { | 534 bool found) { |
935 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 535 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
936 | 536 |
937 LookupFinished(request, url, found); | 537 LookupFinished(request, url, found); |
938 pending_lookups_.erase(request); | 538 pending_lookups_.erase(request); |
939 delete request; | 539 delete request; |
940 | 540 |
941 StartSomeQueuedResolutions(); | 541 StartSomeQueuedResolutions(); |
942 } | 542 } |
943 | 543 |
944 void Predictor::LookupFinished(LookupRequest* request, const GURL& url, | 544 void Predictor::LookupFinished(LookupRequest* request, const GURL& url, |
945 bool found) { | 545 bool found) { |
946 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 546 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
947 UrlInfo* info = &results_[url]; | 547 UrlInfo* info = &results_[url]; |
948 DCHECK(info->HasUrl(url)); | 548 DCHECK(info->HasUrl(url)); |
949 if (info->is_marked_to_delete()) { | 549 if (info->is_marked_to_delete()) { |
950 results_.erase(url); | 550 results_.erase(url); |
951 } else { | 551 } else { |
952 if (found) | 552 if (found) |
953 info->SetFoundState(); | 553 info->SetFoundState(); |
954 else | 554 else |
955 info->SetNoSuchNameState(); | 555 info->SetNoSuchNameState(); |
956 } | 556 } |
957 } | 557 } |
958 | 558 |
959 UrlInfo* Predictor::AppendToResolutionQueue( | 559 void Predictor::DiscardAllResults() { |
960 const GURL& url, | |
961 UrlInfo::ResolutionMotivation motivation) { | |
962 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 560 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
963 DCHECK(url.has_host()); | 561 // Delete anything listed so far in this session that shows in about:dns. |
| 562 referrers_.clear(); |
964 | 563 |
965 if (shutdown_) | |
966 return NULL; | |
967 | 564 |
968 UrlInfo* info = &results_[url]; | 565 // Try to delete anything in our work queue. |
969 info->SetUrl(url); // Initialize or DCHECK. | 566 while (!work_queue_.IsEmpty()) { |
970 // TODO(jar): I need to discard names that have long since expired. | 567 // Emulate processing cycle as though host was not found. |
971 // Currently we only add to the domain map :-/ | 568 GURL url = work_queue_.Pop(); |
972 | |
973 DCHECK(info->HasUrl(url)); | |
974 | |
975 if (!info->NeedsDnsUpdate()) { | |
976 info->DLogResultsStats("DNS PrefetchNotUpdated"); | |
977 return NULL; | |
978 } | |
979 | |
980 info->SetQueuedState(motivation); | |
981 work_queue_.Push(url, motivation); | |
982 StartSomeQueuedResolutions(); | |
983 return info; | |
984 } | |
985 | |
986 bool Predictor::CongestionControlPerformed(UrlInfo* info) { | |
987 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
988 // Note: queue_duration is ONLY valid after we go to assigned state. | |
989 if (info->queue_duration() < max_dns_queue_delay_) | |
990 return false; | |
991 // We need to discard all entries in our queue, as we're keeping them waiting | |
992 // too long. By doing this, we'll have a chance to quickly service urgent | |
993 // resolutions, and not have a bogged down system. | |
994 while (true) { | |
995 info->RemoveFromQueue(); | |
996 if (work_queue_.IsEmpty()) | |
997 break; | |
998 info = &results_[work_queue_.Pop()]; | |
999 info->SetAssignedState(); | |
1000 } | |
1001 return true; | |
1002 } | |
1003 | |
1004 void Predictor::StartSomeQueuedResolutions() { | |
1005 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
1006 | |
1007 while (!work_queue_.IsEmpty() && | |
1008 pending_lookups_.size() < max_concurrent_dns_lookups_) { | |
1009 const GURL url(work_queue_.Pop()); | |
1010 UrlInfo* info = &results_[url]; | 569 UrlInfo* info = &results_[url]; |
1011 DCHECK(info->HasUrl(url)); | 570 DCHECK(info->HasUrl(url)); |
1012 info->SetAssignedState(); | 571 info->SetAssignedState(); |
| 572 info->SetNoSuchNameState(); |
| 573 } |
| 574 // Now every result_ is either resolved, or is being resolved |
| 575 // (see LookupRequest). |
1013 | 576 |
1014 if (CongestionControlPerformed(info)) { | 577 // Step through result_, recording names of all hosts that can't be erased. |
1015 DCHECK(work_queue_.IsEmpty()); | 578 // We can't erase anything being worked on. |
1016 return; | 579 Results assignees; |
| 580 for (Results::iterator it = results_.begin(); results_.end() != it; ++it) { |
| 581 GURL url(it->first); |
| 582 UrlInfo* info = &it->second; |
| 583 DCHECK(info->HasUrl(url)); |
| 584 if (info->is_assigned()) { |
| 585 info->SetPendingDeleteState(); |
| 586 assignees[url] = *info; |
1017 } | 587 } |
| 588 } |
| 589 DCHECK(assignees.size() <= max_concurrent_dns_lookups_); |
| 590 results_.clear(); |
| 591 // Put back in the names being worked on. |
| 592 for (Results::iterator it = assignees.begin(); assignees.end() != it; ++it) { |
| 593 DCHECK(it->second.is_marked_to_delete()); |
| 594 results_[it->first] = it->second; |
| 595 } |
| 596 } |
1018 | 597 |
1019 LookupRequest* request = new LookupRequest(this, host_resolver_, url); | 598 void Predictor::TrimReferrersNow() { |
1020 int status = request->Start(); | 599 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
1021 if (status == net::ERR_IO_PENDING) { | 600 // Just finish up work if an incremental trim is in progress. |
1022 // Will complete asynchronously. | 601 if (urls_being_trimmed_.empty()) |
1023 pending_lookups_.insert(request); | 602 LoadUrlsForTrimming(); |
1024 peak_pending_lookups_ = std::max(peak_pending_lookups_, | 603 IncrementalTrimReferrers(true); // Do everything now. |
1025 pending_lookups_.size()); | 604 } |
1026 } else { | 605 |
1027 // Completed synchronously (was already cached by HostResolver), or else | 606 void Predictor::SerializeReferrers(ListValue* referral_list) { |
1028 // there was (equivalently) some network error that prevents us from | 607 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
1029 // finding the name. Status net::OK means it was "found." | 608 referral_list->Clear(); |
1030 LookupFinished(request, url, status == net::OK); | 609 referral_list->Append(new base::FundamentalValue(PREDICTOR_REFERRER_VERSION)); |
1031 delete request; | 610 for (Referrers::const_iterator it = referrers_.begin(); |
| 611 it != referrers_.end(); ++it) { |
| 612 // Serialize the list of subresource names. |
| 613 Value* subresource_list(it->second.Serialize()); |
| 614 |
| 615 // Create a list for each referer. |
| 616 ListValue* motivator(new ListValue); |
| 617 motivator->Append(new StringValue(it->first.spec())); |
| 618 motivator->Append(subresource_list); |
| 619 |
| 620 referral_list->Append(motivator); |
| 621 } |
| 622 } |
| 623 |
| 624 void Predictor::DeserializeReferrers(const ListValue& referral_list) { |
| 625 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
| 626 int format_version = -1; |
| 627 if (referral_list.GetSize() > 0 && |
| 628 referral_list.GetInteger(0, &format_version) && |
| 629 format_version == PREDICTOR_REFERRER_VERSION) { |
| 630 for (size_t i = 1; i < referral_list.GetSize(); ++i) { |
| 631 ListValue* motivator; |
| 632 if (!referral_list.GetList(i, &motivator)) { |
| 633 NOTREACHED(); |
| 634 return; |
| 635 } |
| 636 std::string motivating_url_spec; |
| 637 if (!motivator->GetString(0, &motivating_url_spec)) { |
| 638 NOTREACHED(); |
| 639 return; |
| 640 } |
| 641 |
| 642 Value* subresource_list; |
| 643 if (!motivator->Get(1, &subresource_list)) { |
| 644 NOTREACHED(); |
| 645 return; |
| 646 } |
| 647 |
| 648 referrers_[GURL(motivating_url_spec)].Deserialize(*subresource_list); |
1032 } | 649 } |
1033 } | 650 } |
1034 } | 651 } |
1035 | 652 |
1036 void Predictor::TrimReferrers() { | 653 void Predictor::TrimReferrers() { |
1037 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 654 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
1038 if (!urls_being_trimmed_.empty()) | 655 if (!urls_being_trimmed_.empty()) |
1039 return; // There is incremental trimming in progress already. | 656 return; // There is incremental trimming in progress already. |
1040 | 657 |
1041 // Check to see if it is time to trim yet. | 658 // Check to see if it is time to trim yet. |
(...skipping 12 matching lines...) Expand all Loading... |
1054 it != referrers_.end(); ++it) | 671 it != referrers_.end(); ++it) |
1055 urls_being_trimmed_.push_back(it->first); | 672 urls_being_trimmed_.push_back(it->first); |
1056 UMA_HISTOGRAM_COUNTS("Net.PredictionTrimSize", urls_being_trimmed_.size()); | 673 UMA_HISTOGRAM_COUNTS("Net.PredictionTrimSize", urls_being_trimmed_.size()); |
1057 } | 674 } |
1058 | 675 |
1059 void Predictor::PostIncrementalTrimTask() { | 676 void Predictor::PostIncrementalTrimTask() { |
1060 if (urls_being_trimmed_.empty()) | 677 if (urls_being_trimmed_.empty()) |
1061 return; | 678 return; |
1062 MessageLoop::current()->PostDelayedTask( | 679 MessageLoop::current()->PostDelayedTask( |
1063 FROM_HERE, | 680 FROM_HERE, |
1064 trim_task_factory_->NewRunnableMethod( | 681 trim_task_factory_.NewRunnableMethod(&Predictor::IncrementalTrimReferrers, |
1065 &Predictor::IncrementalTrimReferrers, false), | 682 false), |
1066 kDurationBetweenTrimmingIncrements.InMilliseconds()); | 683 kDurationBetweenTrimmingIncrements.InMilliseconds()); |
1067 } | 684 } |
1068 | 685 |
1069 void Predictor::IncrementalTrimReferrers(bool trim_all_now) { | 686 void Predictor::IncrementalTrimReferrers(bool trim_all_now) { |
1070 size_t trim_count = urls_being_trimmed_.size(); | 687 size_t trim_count = urls_being_trimmed_.size(); |
1071 if (!trim_all_now) | 688 if (!trim_all_now) |
1072 trim_count = std::min(trim_count, kUrlsTrimmedPerIncrement); | 689 trim_count = std::min(trim_count, kUrlsTrimmedPerIncrement); |
1073 while (trim_count-- != 0) { | 690 while (trim_count-- != 0) { |
1074 Referrers::iterator it = referrers_.find(urls_being_trimmed_.back()); | 691 Referrers::iterator it = referrers_.find(urls_being_trimmed_.back()); |
1075 urls_being_trimmed_.pop_back(); | 692 urls_being_trimmed_.pop_back(); |
1076 if (it == referrers_.end()) | 693 if (it == referrers_.end()) |
1077 continue; // Defensive code: It got trimmed away already. | 694 continue; // Defensive code: It got trimmed away already. |
1078 if (!it->second.Trim(kReferrerTrimRatio, kDiscardableExpectedValue)) | 695 if (!it->second.Trim(kReferrerTrimRatio, kDiscardableExpectedValue)) |
1079 referrers_.erase(it); | 696 referrers_.erase(it); |
1080 } | 697 } |
1081 PostIncrementalTrimTask(); | 698 PostIncrementalTrimTask(); |
1082 } | 699 } |
1083 | 700 |
1084 // ---------------------- End IO methods. ------------------------------------- | 701 //------------------------------------------------------------------------------ |
1085 | |
1086 //----------------------------------------------------------------------------- | |
1087 | 702 |
1088 Predictor::HostNameQueue::HostNameQueue() { | 703 Predictor::HostNameQueue::HostNameQueue() { |
1089 } | 704 } |
1090 | 705 |
1091 Predictor::HostNameQueue::~HostNameQueue() { | 706 Predictor::HostNameQueue::~HostNameQueue() { |
1092 } | 707 } |
1093 | 708 |
1094 void Predictor::HostNameQueue::Push(const GURL& url, | 709 void Predictor::HostNameQueue::Push(const GURL& url, |
1095 UrlInfo::ResolutionMotivation motivation) { | 710 UrlInfo::ResolutionMotivation motivation) { |
1096 switch (motivation) { | 711 switch (motivation) { |
(...skipping 15 matching lines...) Expand all Loading... |
1112 | 727 |
1113 GURL Predictor::HostNameQueue::Pop() { | 728 GURL Predictor::HostNameQueue::Pop() { |
1114 DCHECK(!IsEmpty()); | 729 DCHECK(!IsEmpty()); |
1115 std::queue<GURL> *queue(rush_queue_.empty() ? &background_queue_ | 730 std::queue<GURL> *queue(rush_queue_.empty() ? &background_queue_ |
1116 : &rush_queue_); | 731 : &rush_queue_); |
1117 GURL url(queue->front()); | 732 GURL url(queue->front()); |
1118 queue->pop(); | 733 queue->pop(); |
1119 return url; | 734 return url; |
1120 } | 735 } |
1121 | 736 |
1122 //----------------------------------------------------------------------------- | 737 void Predictor::DeserializeReferrersThenDelete(ListValue* referral_list) { |
1123 // Member definitions for InitialObserver class. | 738 DeserializeReferrers(*referral_list); |
1124 | 739 delete referral_list; |
1125 Predictor::InitialObserver::InitialObserver() { | |
1126 } | 740 } |
1127 | 741 |
1128 Predictor::InitialObserver::~InitialObserver() { | |
1129 } | |
1130 | 742 |
1131 void Predictor::InitialObserver::Append(const GURL& url, | 743 //------------------------------------------------------------------------------ |
1132 Predictor* predictor) { | |
1133 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
1134 | |
1135 // TODO(rlp): Do we really need the predictor check here? | |
1136 if (NULL == predictor) | |
1137 return; | |
1138 if (kStartupResolutionCount <= first_navigations_.size()) | |
1139 return; | |
1140 | |
1141 DCHECK(url.SchemeIs("http") || url.SchemeIs("https")); | |
1142 DCHECK_EQ(url, Predictor::CanonicalizeUrl(url)); | |
1143 if (first_navigations_.find(url) == first_navigations_.end()) | |
1144 first_navigations_[url] = base::TimeTicks::Now(); | |
1145 } | |
1146 | |
1147 void Predictor::InitialObserver::GetInitialDnsResolutionList( | |
1148 base::ListValue* startup_list) { | |
1149 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
1150 DCHECK(startup_list); | |
1151 startup_list->Clear(); | |
1152 DCHECK_EQ(0u, startup_list->GetSize()); | |
1153 startup_list->Append( | |
1154 new base::FundamentalValue(kPredictorStartupFormatVersion)); | |
1155 for (FirstNavigations::iterator it = first_navigations_.begin(); | |
1156 it != first_navigations_.end(); | |
1157 ++it) { | |
1158 DCHECK(it->first == Predictor::CanonicalizeUrl(it->first)); | |
1159 startup_list->Append(new StringValue(it->first.spec())); | |
1160 } | |
1161 } | |
1162 | |
1163 void Predictor::InitialObserver::GetFirstResolutionsHtml( | |
1164 std::string* output) { | |
1165 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | |
1166 | |
1167 UrlInfo::UrlInfoTable resolution_list; | |
1168 { | |
1169 for (FirstNavigations::iterator it(first_navigations_.begin()); | |
1170 it != first_navigations_.end(); | |
1171 it++) { | |
1172 UrlInfo info; | |
1173 info.SetUrl(it->first); | |
1174 info.set_time(it->second); | |
1175 resolution_list.push_back(info); | |
1176 } | |
1177 } | |
1178 UrlInfo::GetHtmlTable(resolution_list, | |
1179 "Future startups will prefetch DNS records for ", false, output); | |
1180 } | |
1181 | |
1182 //----------------------------------------------------------------------------- | |
1183 // Helper functions | 744 // Helper functions |
1184 //----------------------------------------------------------------------------- | 745 //------------------------------------------------------------------------------ |
1185 | 746 |
1186 // static | 747 // static |
1187 GURL Predictor::CanonicalizeUrl(const GURL& url) { | 748 GURL Predictor::CanonicalizeUrl(const GURL& url) { |
1188 if (!url.has_host()) | 749 if (!url.has_host()) |
1189 return GURL::EmptyGURL(); | 750 return GURL::EmptyGURL(); |
1190 | 751 |
1191 std::string scheme; | 752 std::string scheme; |
1192 if (url.has_scheme()) { | 753 if (url.has_scheme()) { |
1193 scheme = url.scheme(); | 754 scheme = url.scheme(); |
1194 if (scheme != "http" && scheme != "https") | 755 if (scheme != "http" && scheme != "https") |
1195 return GURL::EmptyGURL(); | 756 return GURL::EmptyGURL(); |
1196 if (url.has_port()) | 757 if (url.has_port()) |
1197 return url.GetWithEmptyPath(); | 758 return url.GetWithEmptyPath(); |
1198 } else { | 759 } else { |
1199 scheme = "http"; | 760 scheme = "http"; |
1200 } | 761 } |
1201 | 762 |
1202 // If we omit a port, it will default to 80 or 443 as appropriate. | 763 // If we omit a port, it will default to 80 or 443 as appropriate. |
1203 std::string colon_plus_port; | 764 std::string colon_plus_port; |
1204 if (url.has_port()) | 765 if (url.has_port()) |
1205 colon_plus_port = ":" + url.port(); | 766 colon_plus_port = ":" + url.port(); |
1206 | 767 |
1207 return GURL(scheme + "://" + url.host() + colon_plus_port); | 768 return GURL(scheme + "://" + url.host() + colon_plus_port); |
1208 } | 769 } |
1209 | 770 |
1210 void SimpleShutdownPredictor::ShutdownOnUIThread(PrefService* user_prefs) { | |
1211 SetShutdown(true); | |
1212 } | |
1213 | 771 |
1214 } // namespace chrome_browser_net | 772 } // namespace chrome_browser_net |
OLD | NEW |