Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(186)

Side by Side Diff: chrome/browser/predictors/resource_prefetch_predictor.cc

Issue 2355273002: Redirect handling in the resource_prefetch_predictor. (Closed)
Patch Set: Reusable DeleteDataHelper Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2014 The Chromium Authors. All rights reserved. 1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "chrome/browser/predictors/resource_prefetch_predictor.h" 5 #include "chrome/browser/predictors/resource_prefetch_predictor.h"
6 6
7 #include <map> 7 #include <map>
8 #include <set> 8 #include <set>
9 #include <utility> 9 #include <utility>
10 10
11 #include "base/command_line.h" 11 #include "base/command_line.h"
12 #include "base/macros.h" 12 #include "base/macros.h"
13 #include "base/memory/ptr_util.h"
13 #include "base/metrics/histogram_macros.h" 14 #include "base/metrics/histogram_macros.h"
14 #include "base/metrics/sparse_histogram.h" 15 #include "base/metrics/sparse_histogram.h"
15 #include "base/strings/string_number_conversions.h" 16 #include "base/strings/string_number_conversions.h"
16 #include "base/strings/stringprintf.h" 17 #include "base/strings/stringprintf.h"
17 #include "base/time/time.h" 18 #include "base/time/time.h"
18 #include "chrome/browser/history/history_service_factory.h" 19 #include "chrome/browser/history/history_service_factory.h"
19 #include "chrome/browser/predictors/predictor_database.h" 20 #include "chrome/browser/predictors/predictor_database.h"
20 #include "chrome/browser/predictors/predictor_database_factory.h" 21 #include "chrome/browser/predictors/predictor_database_factory.h"
21 #include "chrome/browser/predictors/resource_prefetcher_manager.h" 22 #include "chrome/browser/predictors/resource_prefetcher_manager.h"
22 #include "chrome/browser/profiles/profile.h" 23 #include "chrome/browser/profiles/profile.h"
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
58 enum ReportingEvent { 59 enum ReportingEvent {
59 REPORTING_EVENT_ALL_HISTORY_CLEARED = 0, 60 REPORTING_EVENT_ALL_HISTORY_CLEARED = 0,
60 REPORTING_EVENT_PARTIAL_HISTORY_CLEARED = 1, 61 REPORTING_EVENT_PARTIAL_HISTORY_CLEARED = 1,
61 REPORTING_EVENT_COUNT = 2 62 REPORTING_EVENT_COUNT = 2
62 }; 63 };
63 64
64 } // namespace 65 } // namespace
65 66
66 namespace predictors { 67 namespace predictors {
67 68
69 const int kMaxRedirectsPerEntry = 20;
70
68 //////////////////////////////////////////////////////////////////////////////// 71 ////////////////////////////////////////////////////////////////////////////////
69 // History lookup task. 72 // History lookup task.
70 73
71 // Used to fetch the visit count for a URL from the History database. 74 // Used to fetch the visit count for a URL from the History database.
72 class GetUrlVisitCountTask : public history::HistoryDBTask { 75 class GetUrlVisitCountTask : public history::HistoryDBTask {
73 public: 76 public:
74 typedef ResourcePrefetchPredictor::URLRequestSummary URLRequestSummary; 77 typedef ResourcePrefetchPredictor::URLRequestSummary URLRequestSummary;
75 typedef base::Callback<void( 78 typedef ResourcePrefetchPredictor::Navigation Navigation;
76 size_t, // Visit count. 79 typedef base::Callback<void(size_t, // Visit count.
77 const NavigationID&, 80 const NavigationID&,
78 const std::vector<URLRequestSummary>&)> VisitInfoCallback; 81 const Navigation&)>
82 VisitInfoCallback;
79 83
80 GetUrlVisitCountTask( 84 GetUrlVisitCountTask(const NavigationID& navigation_id,
81 const NavigationID& navigation_id, 85 std::unique_ptr<Navigation> navigation,
82 std::vector<URLRequestSummary>* requests, 86 VisitInfoCallback callback)
83 VisitInfoCallback callback)
84 : visit_count_(0), 87 : visit_count_(0),
85 navigation_id_(navigation_id), 88 navigation_id_(navigation_id),
86 requests_(requests), 89 navigation_(std::move(navigation)),
87 callback_(callback) { 90 callback_(callback) {
88 DCHECK(requests_.get()); 91 DCHECK(navigation_.get());
89 } 92 }
90 93
91 bool RunOnDBThread(history::HistoryBackend* backend, 94 bool RunOnDBThread(history::HistoryBackend* backend,
92 history::HistoryDatabase* db) override { 95 history::HistoryDatabase* db) override {
93 history::URLRow url_row; 96 history::URLRow url_row;
94 if (db->GetRowForURL(navigation_id_.main_frame_url, &url_row)) 97 if (db->GetRowForURL(navigation_id_.main_frame_url, &url_row))
95 visit_count_ = url_row.visit_count(); 98 visit_count_ = url_row.visit_count();
96 return true; 99 return true;
97 } 100 }
98 101
99 void DoneRunOnMainThread() override { 102 void DoneRunOnMainThread() override {
100 callback_.Run(visit_count_, navigation_id_, *requests_); 103 callback_.Run(visit_count_, navigation_id_, *navigation_);
101 } 104 }
102 105
103 private: 106 private:
104 ~GetUrlVisitCountTask() override {} 107 ~GetUrlVisitCountTask() override {}
105 108
106 int visit_count_; 109 int visit_count_;
107 NavigationID navigation_id_; 110 NavigationID navigation_id_;
108 std::unique_ptr<std::vector<URLRequestSummary>> requests_; 111 std::unique_ptr<Navigation> navigation_;
109 VisitInfoCallback callback_; 112 VisitInfoCallback callback_;
110 113
111 DISALLOW_COPY_AND_ASSIGN(GetUrlVisitCountTask); 114 DISALLOW_COPY_AND_ASSIGN(GetUrlVisitCountTask);
112 }; 115 };
113 116
114 //////////////////////////////////////////////////////////////////////////////// 117 ////////////////////////////////////////////////////////////////////////////////
115 // ResourcePrefetchPredictor static functions. 118 // ResourcePrefetchPredictor static functions.
116 119
117 // static 120 // static
118 bool ResourcePrefetchPredictor::ShouldRecordRequest( 121 bool ResourcePrefetchPredictor::ShouldRecordRequest(
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after
274 priority(other.priority), 277 priority(other.priority),
275 mime_type(other.mime_type), 278 mime_type(other.mime_type),
276 was_cached(other.was_cached), 279 was_cached(other.was_cached),
277 redirect_url(other.redirect_url), 280 redirect_url(other.redirect_url),
278 has_validators(other.has_validators), 281 has_validators(other.has_validators),
279 always_revalidate(other.always_revalidate) {} 282 always_revalidate(other.always_revalidate) {}
280 283
281 ResourcePrefetchPredictor::URLRequestSummary::~URLRequestSummary() { 284 ResourcePrefetchPredictor::URLRequestSummary::~URLRequestSummary() {
282 } 285 }
283 286
287 ResourcePrefetchPredictor::Navigation::Navigation(const GURL& i_initial_url)
288 : initial_url(i_initial_url) {}
289
290 ResourcePrefetchPredictor::Navigation::~Navigation() {}
291
284 // static 292 // static
285 bool ResourcePrefetchPredictor::URLRequestSummary::SummarizeResponse( 293 bool ResourcePrefetchPredictor::URLRequestSummary::SummarizeResponse(
286 const net::URLRequest& request, 294 const net::URLRequest& request,
287 URLRequestSummary* summary) { 295 URLRequestSummary* summary) {
288 const content::ResourceRequestInfo* info = 296 const content::ResourceRequestInfo* info =
289 content::ResourceRequestInfo::ForRequest(&request); 297 content::ResourceRequestInfo::ForRequest(&request);
290 if (!info) 298 if (!info)
291 return false; 299 return false;
292 300
293 int render_process_id, render_frame_id; 301 int render_process_id, render_frame_id;
(...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after
407 const URLRequestSummary& request) { 415 const URLRequestSummary& request) {
408 DCHECK_CURRENTLY_ON(BrowserThread::UI); 416 DCHECK_CURRENTLY_ON(BrowserThread::UI);
409 DCHECK_EQ(INITIALIZED, initialization_state_); 417 DCHECK_EQ(INITIALIZED, initialization_state_);
410 418
411 StartPrefetching(request.navigation_id); 419 StartPrefetching(request.navigation_id);
412 420
413 // Cleanup older navigations. 421 // Cleanup older navigations.
414 CleanupAbandonedNavigations(request.navigation_id); 422 CleanupAbandonedNavigations(request.navigation_id);
415 423
416 // New empty navigation entry. 424 // New empty navigation entry.
425 const GURL& initial_url = request.navigation_id.main_frame_url;
417 inflight_navigations_.insert(std::make_pair( 426 inflight_navigations_.insert(std::make_pair(
418 request.navigation_id, 427 request.navigation_id, base::MakeUnique<Navigation>(initial_url)));
419 make_linked_ptr(new std::vector<URLRequestSummary>())));
420 } 428 }
421 429
422 void ResourcePrefetchPredictor::OnMainFrameResponse( 430 void ResourcePrefetchPredictor::OnMainFrameResponse(
423 const URLRequestSummary& response) { 431 const URLRequestSummary& response) {
424 DCHECK_CURRENTLY_ON(BrowserThread::UI); 432 DCHECK_CURRENTLY_ON(BrowserThread::UI);
425 if (initialization_state_ != INITIALIZED) 433 if (initialization_state_ != INITIALIZED)
426 return; 434 return;
427 435
428 StopPrefetching(response.navigation_id); 436 StopPrefetching(response.navigation_id);
429 } 437 }
430 438
431 void ResourcePrefetchPredictor::OnMainFrameRedirect( 439 void ResourcePrefetchPredictor::OnMainFrameRedirect(
432 const URLRequestSummary& response) { 440 const URLRequestSummary& response) {
433 DCHECK_CURRENTLY_ON(BrowserThread::UI); 441 DCHECK_CURRENTLY_ON(BrowserThread::UI);
434 442
435 // TODO(shishir): There are significant gains to be had here if we can use the 443 // TODO(shishir): There are significant gains to be had here if we can use the
436 // start URL in a redirect chain as the key to start prefetching. We can save 444 // start URL in a redirect chain as the key to start prefetching. We can save
437 // of redirect times considerably assuming that the redirect chains do not 445 // of redirect times considerably assuming that the redirect chains do not
438 // change. 446 // change.
439 447
440 // Stop any inflight prefetching. Remove the older navigation. 448 // Stop any inflight prefetching. Remove the older navigation.
441 StopPrefetching(response.navigation_id); 449 StopPrefetching(response.navigation_id);
442 inflight_navigations_.erase(response.navigation_id); 450
451 std::unique_ptr<Navigation> navigation;
452 NavigationMap::iterator nav_it =
453 inflight_navigations_.find(response.navigation_id);
454 if (nav_it != inflight_navigations_.end()) {
455 navigation.reset(nav_it->second.release());
456 inflight_navigations_.erase(nav_it);
457 }
443 458
444 // A redirect will not lead to another OnMainFrameRequest call, so record the 459 // A redirect will not lead to another OnMainFrameRequest call, so record the
Benoit L 2016/09/26 12:51:14 Please update this comment.
alexilin 2016/09/26 15:38:28 Done.
445 // redirect url as a new navigation. 460 // redirect url as a new navigation.
446 461
447 // The redirect url may be empty if the URL was invalid. 462 // The redirect url may be empty if the URL was invalid.
448 if (response.redirect_url.is_empty()) 463 if (response.redirect_url.is_empty())
449 return; 464 return;
450 465
466 // If we lost information about first hop for some reason
467 if (!navigation)
468 navigation.reset(new Navigation(response.navigation_id.main_frame_url));
469
451 NavigationID navigation_id(response.navigation_id); 470 NavigationID navigation_id(response.navigation_id);
452 navigation_id.main_frame_url = response.redirect_url; 471 navigation_id.main_frame_url = response.redirect_url;
453 inflight_navigations_.insert(std::make_pair( 472 inflight_navigations_.insert(
454 navigation_id, 473 std::make_pair(navigation_id, std::move(navigation)));
455 make_linked_ptr(new std::vector<URLRequestSummary>())));
456 } 474 }
457 475
458 void ResourcePrefetchPredictor::OnSubresourceResponse( 476 void ResourcePrefetchPredictor::OnSubresourceResponse(
459 const URLRequestSummary& response) { 477 const URLRequestSummary& response) {
460 DCHECK_CURRENTLY_ON(BrowserThread::UI); 478 DCHECK_CURRENTLY_ON(BrowserThread::UI);
461 479
462 NavigationMap::const_iterator nav_it = 480 NavigationMap::const_iterator nav_it =
463 inflight_navigations_.find(response.navigation_id); 481 inflight_navigations_.find(response.navigation_id);
464 if (nav_it == inflight_navigations_.end()) { 482 if (nav_it == inflight_navigations_.end()) {
465 return; 483 return;
466 } 484 }
467 485
468 nav_it->second->push_back(response); 486 nav_it->second->subresource_requests.push_back(response);
469 } 487 }
470 488
471 void ResourcePrefetchPredictor::OnNavigationComplete( 489 void ResourcePrefetchPredictor::OnNavigationComplete(
472 const NavigationID& nav_id_without_timing_info) { 490 const NavigationID& nav_id_without_timing_info) {
473 DCHECK_CURRENTLY_ON(BrowserThread::UI); 491 DCHECK_CURRENTLY_ON(BrowserThread::UI);
474 492
475 NavigationMap::iterator nav_it = 493 NavigationMap::iterator nav_it =
476 inflight_navigations_.find(nav_id_without_timing_info); 494 inflight_navigations_.find(nav_id_without_timing_info);
477 if (nav_it == inflight_navigations_.end()) 495 if (nav_it == inflight_navigations_.end())
478 return; 496 return;
479 497
480 const NavigationID navigation_id(nav_it->first); 498 const NavigationID navigation_id(nav_it->first);
481 499
482 // Remove the navigation from the inflight navigations. 500 // Remove the navigation from the inflight navigations.
483 std::vector<URLRequestSummary>* requests = (nav_it->second).release(); 501 std::unique_ptr<Navigation> navigation = std::move(nav_it->second);
484 inflight_navigations_.erase(nav_it); 502 inflight_navigations_.erase(nav_it);
485 503
486 // Kick off history lookup to determine if we should record the URL. 504 // Kick off history lookup to determine if we should record the URL.
487 history::HistoryService* history_service = 505 history::HistoryService* history_service =
488 HistoryServiceFactory::GetForProfile(profile_, 506 HistoryServiceFactory::GetForProfile(profile_,
489 ServiceAccessType::EXPLICIT_ACCESS); 507 ServiceAccessType::EXPLICIT_ACCESS);
490 DCHECK(history_service); 508 DCHECK(history_service);
491 history_service->ScheduleDBTask( 509 history_service->ScheduleDBTask(
492 std::unique_ptr<history::HistoryDBTask>(new GetUrlVisitCountTask( 510 std::unique_ptr<history::HistoryDBTask>(new GetUrlVisitCountTask(
493 navigation_id, requests, 511 navigation_id, std::move(navigation),
494 base::Bind(&ResourcePrefetchPredictor::OnVisitCountLookup, 512 base::Bind(&ResourcePrefetchPredictor::OnVisitCountLookup,
495 AsWeakPtr()))), 513 AsWeakPtr()))),
496 &history_lookup_consumer_); 514 &history_lookup_consumer_);
497 } 515 }
498 516
499 bool ResourcePrefetchPredictor::GetPrefetchData( 517 bool ResourcePrefetchPredictor::GetPrefetchData(
500 const NavigationID& navigation_id, 518 const NavigationID& navigation_id,
501 std::vector<GURL>* urls, 519 std::vector<GURL>* urls,
502 PrefetchKeyType* key_type) { 520 PrefetchKeyType* key_type) {
503 DCHECK(urls); 521 DCHECK(urls);
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
581 599
582 void ResourcePrefetchPredictor::StartInitialization() { 600 void ResourcePrefetchPredictor::StartInitialization() {
583 DCHECK_CURRENTLY_ON(BrowserThread::UI); 601 DCHECK_CURRENTLY_ON(BrowserThread::UI);
584 602
585 DCHECK_EQ(NOT_INITIALIZED, initialization_state_); 603 DCHECK_EQ(NOT_INITIALIZED, initialization_state_);
586 initialization_state_ = INITIALIZING; 604 initialization_state_ = INITIALIZING;
587 605
588 // Create local caches using the database as loaded. 606 // Create local caches using the database as loaded.
589 std::unique_ptr<PrefetchDataMap> url_data_map(new PrefetchDataMap()); 607 std::unique_ptr<PrefetchDataMap> url_data_map(new PrefetchDataMap());
590 std::unique_ptr<PrefetchDataMap> host_data_map(new PrefetchDataMap()); 608 std::unique_ptr<PrefetchDataMap> host_data_map(new PrefetchDataMap());
609 std::unique_ptr<RedirectDataMap> url_redirect_data_map(new RedirectDataMap());
610 std::unique_ptr<RedirectDataMap> host_redirect_data_map(
611 new RedirectDataMap());
591 PrefetchDataMap* url_data_ptr = url_data_map.get(); 612 PrefetchDataMap* url_data_ptr = url_data_map.get();
592 PrefetchDataMap* host_data_ptr = host_data_map.get(); 613 PrefetchDataMap* host_data_ptr = host_data_map.get();
614 RedirectDataMap* url_redirect_data_ptr = url_redirect_data_map.get();
615 RedirectDataMap* host_redirect_data_ptr = host_redirect_data_map.get();
593 616
594 BrowserThread::PostTaskAndReply( 617 BrowserThread::PostTaskAndReply(
595 BrowserThread::DB, FROM_HERE, 618 BrowserThread::DB, FROM_HERE,
596 base::Bind(&ResourcePrefetchPredictorTables::GetAllData, 619 base::Bind(&ResourcePrefetchPredictorTables::GetAllData, tables_,
597 tables_, url_data_ptr, host_data_ptr), 620 url_data_ptr, host_data_ptr, url_redirect_data_ptr,
621 host_redirect_data_ptr),
598 base::Bind(&ResourcePrefetchPredictor::CreateCaches, AsWeakPtr(), 622 base::Bind(&ResourcePrefetchPredictor::CreateCaches, AsWeakPtr(),
599 base::Passed(&url_data_map), base::Passed(&host_data_map))); 623 base::Passed(&url_data_map), base::Passed(&host_data_map),
624 base::Passed(&url_redirect_data_map),
625 base::Passed(&host_redirect_data_map)));
600 } 626 }
601 627
602 void ResourcePrefetchPredictor::CreateCaches( 628 void ResourcePrefetchPredictor::CreateCaches(
603 std::unique_ptr<PrefetchDataMap> url_data_map, 629 std::unique_ptr<PrefetchDataMap> url_data_map,
604 std::unique_ptr<PrefetchDataMap> host_data_map) { 630 std::unique_ptr<PrefetchDataMap> host_data_map,
631 std::unique_ptr<RedirectDataMap> url_redirect_data_map,
632 std::unique_ptr<RedirectDataMap> host_redirect_data_map) {
605 DCHECK_CURRENTLY_ON(BrowserThread::UI); 633 DCHECK_CURRENTLY_ON(BrowserThread::UI);
606 634
607 DCHECK_EQ(INITIALIZING, initialization_state_); 635 DCHECK_EQ(INITIALIZING, initialization_state_);
608 DCHECK(!url_table_cache_); 636 DCHECK(!url_table_cache_);
609 DCHECK(!host_table_cache_); 637 DCHECK(!host_table_cache_);
638 DCHECK(!url_redirect_table_cache_);
639 DCHECK(!host_redirect_table_cache_);
610 DCHECK(inflight_navigations_.empty()); 640 DCHECK(inflight_navigations_.empty());
611 641
612 url_table_cache_.reset(url_data_map.release()); 642 url_table_cache_.reset(url_data_map.release());
613 host_table_cache_.reset(host_data_map.release()); 643 host_table_cache_.reset(host_data_map.release());
644 url_redirect_table_cache_.reset(url_redirect_data_map.release());
645 host_redirect_table_cache_.reset(host_redirect_data_map.release());
614 646
615 UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.UrlTableMainFrameUrlCount", 647 UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.UrlTableMainFrameUrlCount",
616 url_table_cache_->size()); 648 url_table_cache_->size());
617 UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HostTableHostCount", 649 UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HostTableHostCount",
618 host_table_cache_->size()); 650 host_table_cache_->size());
619 651
620 ConnectToHistoryService(); 652 ConnectToHistoryService();
621 } 653 }
622 654
623 void ResourcePrefetchPredictor::OnHistoryAndCacheLoaded() { 655 void ResourcePrefetchPredictor::OnHistoryAndCacheLoaded() {
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after
671 } 703 }
672 704
673 const std::string& host = it.url().host(); 705 const std::string& host = it.url().host();
674 if (host_table_cache_->find(host) != host_table_cache_->end()) { 706 if (host_table_cache_->find(host) != host_table_cache_->end()) {
675 hosts_to_delete.push_back(host); 707 hosts_to_delete.push_back(host);
676 host_table_cache_->erase(host); 708 host_table_cache_->erase(host);
677 } 709 }
678 } 710 }
679 711
680 if (!urls_to_delete.empty() || !hosts_to_delete.empty()) { 712 if (!urls_to_delete.empty() || !hosts_to_delete.empty()) {
681 BrowserThread::PostTask(BrowserThread::DB, FROM_HERE, 713 BrowserThread::PostTask(
682 base::Bind(&ResourcePrefetchPredictorTables::DeleteData, 714 BrowserThread::DB, FROM_HERE,
683 tables_, 715 base::Bind(&ResourcePrefetchPredictorTables::DeleteResourceData,
684 urls_to_delete, 716 tables_, urls_to_delete, hosts_to_delete));
685 hosts_to_delete));
686 } 717 }
687 } 718 }
688 719
689 void ResourcePrefetchPredictor::RemoveOldestEntryInPrefetchDataMap( 720 void ResourcePrefetchPredictor::RemoveOldestEntryInPrefetchDataMap(
690 PrefetchKeyType key_type, 721 PrefetchKeyType key_type,
691 PrefetchDataMap* data_map) { 722 PrefetchDataMap* data_map) {
692 if (data_map->empty()) 723 if (data_map->empty())
693 return; 724 return;
694 725
695 base::Time oldest_time; 726 base::Time oldest_time;
696 std::string key_to_delete; 727 std::string key_to_delete;
697 for (PrefetchDataMap::iterator it = data_map->begin(); 728 for (PrefetchDataMap::iterator it = data_map->begin();
698 it != data_map->end(); ++it) { 729 it != data_map->end(); ++it) {
699 if (key_to_delete.empty() || it->second.last_visit < oldest_time) { 730 if (key_to_delete.empty() || it->second.last_visit < oldest_time) {
700 key_to_delete = it->first; 731 key_to_delete = it->first;
701 oldest_time = it->second.last_visit; 732 oldest_time = it->second.last_visit;
702 } 733 }
703 } 734 }
704 735
705 data_map->erase(key_to_delete); 736 data_map->erase(key_to_delete);
706 BrowserThread::PostTask(BrowserThread::DB, FROM_HERE, 737 BrowserThread::PostTask(
707 base::Bind(&ResourcePrefetchPredictorTables::DeleteSingleDataPoint, 738 BrowserThread::DB, FROM_HERE,
708 tables_, 739 base::Bind(
709 key_to_delete, 740 &ResourcePrefetchPredictorTables::DeleteSingleResourceDataPoint,
710 key_type)); 741 tables_, key_to_delete, key_type));
742 }
743
744 void ResourcePrefetchPredictor::RemoveOldestEntryInRedirectDataMap(
745 PrefetchKeyType key_type,
746 RedirectDataMap* data_map) {
747 if (data_map->empty())
748 return;
749
750 uint64_t oldest_time;
751 std::string key_to_delete;
752 for (const auto& kv : *data_map) {
753 const RedirectData& data = kv.second;
754 if (key_to_delete.empty() || data.last_visit_time() < oldest_time) {
755 key_to_delete = data.primary_key();
756 oldest_time = data.last_visit_time();
757 }
758 }
759
760 data_map->erase(key_to_delete);
761 BrowserThread::PostTask(
762 BrowserThread::DB, FROM_HERE,
763 base::Bind(
764 &ResourcePrefetchPredictorTables::DeleteSingleRedirectDataPoint,
765 tables_, key_to_delete, key_type));
711 } 766 }
712 767
713 void ResourcePrefetchPredictor::OnVisitCountLookup( 768 void ResourcePrefetchPredictor::OnVisitCountLookup(
714 size_t visit_count, 769 size_t visit_count,
715 const NavigationID& navigation_id, 770 const NavigationID& navigation_id,
716 const std::vector<URLRequestSummary>& requests) { 771 const Navigation& navigation) {
717 DCHECK_CURRENTLY_ON(BrowserThread::UI); 772 DCHECK_CURRENTLY_ON(BrowserThread::UI);
718 773
719 UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HistoryVisitCountForUrl", 774 UMA_HISTOGRAM_COUNTS("ResourcePrefetchPredictor.HistoryVisitCountForUrl",
720 visit_count); 775 visit_count);
721 776
777 // TODO(alexilin) make only one request to DB thread
778
722 // URL level data - merge only if we are already saving the data, or we it 779 // URL level data - merge only if we are already saving the data, or we it
723 // meets the cutoff requirement. 780 // meets the cutoff requirement.
724 const std::string url_spec = navigation_id.main_frame_url.spec(); 781 const std::string url_spec = navigation_id.main_frame_url.spec();
725 bool already_tracking = url_table_cache_->find(url_spec) != 782 bool already_tracking = url_table_cache_->find(url_spec) !=
726 url_table_cache_->end(); 783 url_table_cache_->end();
727 bool should_track_url = already_tracking || 784 bool should_track_url = already_tracking ||
728 (visit_count >= config_.min_url_visit_count); 785 (visit_count >= config_.min_url_visit_count);
729 786
730 if (should_track_url && config_.IsURLLearningEnabled()) { 787 if (should_track_url && config_.IsURLLearningEnabled()) {
731 LearnNavigation(url_spec, PREFETCH_KEY_TYPE_URL, requests, 788 LearnNavigation(url_spec, PREFETCH_KEY_TYPE_URL,
732 config_.max_urls_to_track, url_table_cache_.get()); 789 navigation.subresource_requests, config_.max_urls_to_track,
790 url_table_cache_.get(), navigation.initial_url.spec(),
791 url_redirect_table_cache_.get());
733 } 792 }
734 793
735 // Host level data - no cutoff, always learn the navigation if enabled. 794 // Host level data - no cutoff, always learn the navigation if enabled.
736 if (config_.IsHostLearningEnabled()) { 795 if (config_.IsHostLearningEnabled()) {
737 LearnNavigation(navigation_id.main_frame_url.host(), 796 const std::string host = navigation_id.main_frame_url.host();
738 PREFETCH_KEY_TYPE_HOST, 797 LearnNavigation(host, PREFETCH_KEY_TYPE_HOST,
Benoit L 2016/09/26 12:51:14 nit: Wouldn't the code be a bit simpler by passing
alexilin 2016/09/26 15:38:28 It looks like no. We should still pass keys (|key|
Benoit L 2016/09/27 09:55:06 Ok, thanks for the explanation.
alexilin 2016/09/27 14:52:46 Done.
739 requests, 798 navigation.subresource_requests, config_.max_hosts_to_track,
740 config_.max_hosts_to_track, 799 host_table_cache_.get(), navigation.initial_url.host(),
741 host_table_cache_.get()); 800 host_redirect_table_cache_.get());
742 } 801 }
743 } 802 }
744 803
745 void ResourcePrefetchPredictor::LearnNavigation( 804 void ResourcePrefetchPredictor::LearnNavigation(
746 const std::string& key, 805 const std::string& key,
747 PrefetchKeyType key_type, 806 PrefetchKeyType key_type,
748 const std::vector<URLRequestSummary>& new_resources, 807 const std::vector<URLRequestSummary>& new_resources,
749 size_t max_data_map_size, 808 size_t max_data_map_size,
750 PrefetchDataMap* data_map) { 809 PrefetchDataMap* data_map,
810 const std::string& redirect_origin_key,
811 RedirectDataMap* redirect_map) {
751 DCHECK_CURRENTLY_ON(BrowserThread::UI); 812 DCHECK_CURRENTLY_ON(BrowserThread::UI);
752 813
753 // If the primary key is too long reject it. 814 // If the primary key is too long reject it.
754 if (key.length() > ResourcePrefetchPredictorTables::kMaxStringLength) 815 if (key.length() > ResourcePrefetchPredictorTables::kMaxStringLength)
755 return; 816 return;
756 817
757 PrefetchDataMap::iterator cache_entry = data_map->find(key); 818 PrefetchDataMap::iterator cache_entry = data_map->find(key);
758 if (cache_entry == data_map->end()) { 819 if (cache_entry == data_map->end()) {
759 if (data_map->size() >= max_data_map_size) { 820 // If the table is full, delete an entry.
760 // The table is full, delete an entry. 821 if (data_map->size() >= max_data_map_size)
761 RemoveOldestEntryInPrefetchDataMap(key_type, data_map); 822 RemoveOldestEntryInPrefetchDataMap(key_type, data_map);
762 }
763 823
764 cache_entry = data_map->insert(std::make_pair( 824 cache_entry = data_map->insert(std::make_pair(
765 key, PrefetchData(key_type, key))).first; 825 key, PrefetchData(key_type, key))).first;
766 cache_entry->second.last_visit = base::Time::Now(); 826 cache_entry->second.last_visit = base::Time::Now();
767 size_t new_resources_size = new_resources.size(); 827 size_t new_resources_size = new_resources.size();
768 std::set<GURL> resources_seen; 828 std::set<GURL> resources_seen;
769 for (size_t i = 0; i < new_resources_size; ++i) { 829 for (size_t i = 0; i < new_resources_size; ++i) {
770 if (resources_seen.find(new_resources[i].resource_url) != 830 if (resources_seen.find(new_resources[i].resource_url) !=
771 resources_seen.end()) { 831 resources_seen.end()) {
772 continue; 832 continue;
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after
872 ResourcePrefetchPredictorTables::SortResources(&resources); 932 ResourcePrefetchPredictorTables::SortResources(&resources);
873 if (resources.size() > config_.max_resources_per_entry) 933 if (resources.size() > config_.max_resources_per_entry)
874 resources.resize(config_.max_resources_per_entry); 934 resources.resize(config_.max_resources_per_entry);
875 935
876 // If the row has no resources, remove it from the cache and delete the 936 // If the row has no resources, remove it from the cache and delete the
877 // entry in the database. Else update the database. 937 // entry in the database. Else update the database.
878 if (resources.empty()) { 938 if (resources.empty()) {
879 data_map->erase(key); 939 data_map->erase(key);
880 BrowserThread::PostTask( 940 BrowserThread::PostTask(
881 BrowserThread::DB, FROM_HERE, 941 BrowserThread::DB, FROM_HERE,
882 base::Bind(&ResourcePrefetchPredictorTables::DeleteSingleDataPoint, 942 base::Bind(
883 tables_, 943 &ResourcePrefetchPredictorTables::DeleteSingleResourceDataPoint,
884 key, 944 tables_, key, key_type));
885 key_type));
886 } else { 945 } else {
887 bool is_host = key_type == PREFETCH_KEY_TYPE_HOST; 946 bool is_host = key_type == PREFETCH_KEY_TYPE_HOST;
888 PrefetchData empty_data( 947 PrefetchData empty_data(
889 !is_host ? PREFETCH_KEY_TYPE_HOST : PREFETCH_KEY_TYPE_URL, 948 !is_host ? PREFETCH_KEY_TYPE_HOST : PREFETCH_KEY_TYPE_URL,
890 std::string()); 949 std::string());
950 RedirectData empty_redirect_data;
891 const PrefetchData& host_data = is_host ? cache_entry->second : empty_data; 951 const PrefetchData& host_data = is_host ? cache_entry->second : empty_data;
892 const PrefetchData& url_data = is_host ? empty_data : cache_entry->second; 952 const PrefetchData& url_data = is_host ? empty_data : cache_entry->second;
893 BrowserThread::PostTask( 953 BrowserThread::PostTask(
894 BrowserThread::DB, FROM_HERE, 954 BrowserThread::DB, FROM_HERE,
895 base::Bind(&ResourcePrefetchPredictorTables::UpdateData, 955 base::Bind(&ResourcePrefetchPredictorTables::UpdateData, tables_,
896 tables_, 956 url_data, host_data, empty_redirect_data,
897 url_data, 957 empty_redirect_data));
898 host_data)); 958 }
959
960 if (key != redirect_origin_key) {
961 LearnRedirect(redirect_origin_key, key_type, key, max_data_map_size,
962 redirect_map);
899 } 963 }
900 } 964 }
901 965
966 void ResourcePrefetchPredictor::LearnRedirect(const std::string& key,
967 PrefetchKeyType key_type,
968 const std::string& final_redirect,
969 size_t max_redirect_map_size,
970 RedirectDataMap* redirect_map) {
971 DCHECK_CURRENTLY_ON(BrowserThread::UI);
972
973 RedirectDataMap::iterator cache_entry = redirect_map->find(key);
974 if (cache_entry == redirect_map->end()) {
975 if (redirect_map->size() >= max_redirect_map_size)
976 RemoveOldestEntryInRedirectDataMap(key_type, redirect_map);
977
978 RedirectData new_data;
979 new_data.set_primary_key(key);
980 cache_entry = redirect_map->insert(std::make_pair(key, new_data)).first;
981 cache_entry->second.set_last_visit_time(
982 base::Time::Now().ToInternalValue());
983 RedirectStat* redirect_to_add = cache_entry->second.add_redirects();
984 redirect_to_add->set_url(final_redirect);
985 redirect_to_add->set_number_of_hits(1);
986 } else {
987 bool need_to_add = true;
988 cache_entry->second.set_last_visit_time(
989 base::Time::Now().ToInternalValue());
990
991 for (RedirectStat& redirect : *(cache_entry->second.mutable_redirects())) {
992 if (redirect.url() == final_redirect) {
993 need_to_add = false;
994 redirect.set_number_of_hits(redirect.number_of_hits() + 1);
995 redirect.set_consecutive_misses(0);
996 } else {
997 redirect.set_number_of_misses(redirect.number_of_misses() + 1);
998 redirect.set_consecutive_misses(redirect.consecutive_misses() + 1);
999 }
1000 }
1001
1002 if (need_to_add) {
1003 RedirectStat* redirect_to_add = cache_entry->second.add_redirects();
1004 redirect_to_add->set_url(final_redirect);
1005 redirect_to_add->set_number_of_hits(1);
1006 }
1007 }
1008
1009 // Trim and sort redirects after update.
1010 std::vector<RedirectStat> redirects;
1011 redirects.reserve(cache_entry->second.redirects_size());
1012 for (const RedirectStat& redirect : cache_entry->second.redirects()) {
1013 if (redirect.consecutive_misses() < config_.max_consecutive_misses)
1014 redirects.push_back(redirect);
1015 }
1016 ResourcePrefetchPredictorTables::SortRedirects(&redirects);
1017
1018 if (redirects.size() > kMaxRedirectsPerEntry)
1019 redirects.resize(kMaxRedirectsPerEntry);
1020
1021 cache_entry->second.clear_redirects();
1022 for (const RedirectStat& redirect : redirects)
1023 cache_entry->second.add_redirects()->CopyFrom(redirect);
1024
1025 if (redirects.empty()) {
1026 redirect_map->erase(cache_entry);
1027 BrowserThread::PostTask(
1028 BrowserThread::DB, FROM_HERE,
1029 base::Bind(
1030 &ResourcePrefetchPredictorTables::DeleteSingleRedirectDataPoint,
1031 tables_, key, key_type));
1032 } else {
1033 bool is_host = key_type == PREFETCH_KEY_TYPE_HOST;
1034 RedirectData empty_redirect_data;
1035 PrefetchData empty_url_data(PREFETCH_KEY_TYPE_URL, std::string());
1036 PrefetchData empty_host_data(PREFETCH_KEY_TYPE_HOST, std::string());
1037 const RedirectData& host_redirect_data =
1038 is_host ? cache_entry->second : empty_redirect_data;
1039 const RedirectData& url_redirect_data =
1040 is_host ? empty_redirect_data : cache_entry->second;
1041 BrowserThread::PostTask(
1042 BrowserThread::DB, FROM_HERE,
1043 base::Bind(&ResourcePrefetchPredictorTables::UpdateData, tables_,
1044 empty_url_data, empty_host_data, url_redirect_data,
1045 host_redirect_data));
1046 }
1047 }
1048
902 void ResourcePrefetchPredictor::OnURLsDeleted( 1049 void ResourcePrefetchPredictor::OnURLsDeleted(
903 history::HistoryService* history_service, 1050 history::HistoryService* history_service,
904 bool all_history, 1051 bool all_history,
905 bool expired, 1052 bool expired,
906 const history::URLRows& deleted_rows, 1053 const history::URLRows& deleted_rows,
907 const std::set<GURL>& favicon_urls) { 1054 const std::set<GURL>& favicon_urls) {
908 DCHECK_CURRENTLY_ON(BrowserThread::UI); 1055 DCHECK_CURRENTLY_ON(BrowserThread::UI);
909 if (INITIALIZED != initialization_state_) 1056 if (INITIALIZED != initialization_state_)
910 return; 1057 return;
911 1058
(...skipping 27 matching lines...) Expand all
939 // HistoryService is already loaded. Continue with Initialization. 1086 // HistoryService is already loaded. Continue with Initialization.
940 OnHistoryAndCacheLoaded(); 1087 OnHistoryAndCacheLoaded();
941 return; 1088 return;
942 } 1089 }
943 DCHECK(!history_service_observer_.IsObserving(history_service)); 1090 DCHECK(!history_service_observer_.IsObserving(history_service));
944 history_service_observer_.Add(history_service); 1091 history_service_observer_.Add(history_service);
945 return; 1092 return;
946 } 1093 }
947 1094
948 } // namespace predictors 1095 } // namespace predictors
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698