Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(872)

Side by Side Diff: chrome/browser/predictors/resource_prefetch_predictor.cc

Issue 2388783002: predictors: Refactor resource_prefetch_predictor_tables. (Closed)
Patch Set: Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2014 The Chromium Authors. All rights reserved. 1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "chrome/browser/predictors/resource_prefetch_predictor.h" 5 #include "chrome/browser/predictors/resource_prefetch_predictor.h"
6 6
7 #include <map> 7 #include <map>
8 #include <set> 8 #include <set>
9 #include <utility> 9 #include <utility>
10 10
(...skipping 509 matching lines...) Expand 10 before | Expand all | Expand 10 after
520 PopulatePrefetcherRequest(iterator->second, urls); 520 PopulatePrefetcherRequest(iterator->second, urls);
521 } 521 }
522 } 522 }
523 523
524 return !urls->empty(); 524 return !urls->empty();
525 } 525 }
526 526
527 void ResourcePrefetchPredictor::PopulatePrefetcherRequest( 527 void ResourcePrefetchPredictor::PopulatePrefetcherRequest(
528 const PrefetchData& data, 528 const PrefetchData& data,
529 std::vector<GURL>* urls) { 529 std::vector<GURL>* urls) {
530 for (const ResourceData& resource : data.resources) { 530 for (const ResourceData& resource : data.resources()) {
531 float confidence = 531 float confidence =
532 static_cast<float>(resource.number_of_hits()) / 532 static_cast<float>(resource.number_of_hits()) /
533 (resource.number_of_hits() + resource.number_of_misses()); 533 (resource.number_of_hits() + resource.number_of_misses());
534 if (confidence < config_.min_resource_confidence_to_trigger_prefetch || 534 if (confidence < config_.min_resource_confidence_to_trigger_prefetch ||
535 resource.number_of_hits() < 535 resource.number_of_hits() <
536 config_.min_resource_hits_to_trigger_prefetch) { 536 config_.min_resource_hits_to_trigger_prefetch) {
537 continue; 537 continue;
538 } 538 }
539 539
540 urls->push_back(GURL(resource.resource_url())); 540 urls->push_back(GURL(resource.resource_url()));
(...skipping 167 matching lines...) Expand 10 before | Expand all | Expand 10 after
708 tables_, url_redirects_to_delete, host_redirects_to_delete)); 708 tables_, url_redirects_to_delete, host_redirects_to_delete));
709 } 709 }
710 } 710 }
711 711
712 void ResourcePrefetchPredictor::RemoveOldestEntryInPrefetchDataMap( 712 void ResourcePrefetchPredictor::RemoveOldestEntryInPrefetchDataMap(
713 PrefetchKeyType key_type, 713 PrefetchKeyType key_type,
714 PrefetchDataMap* data_map) { 714 PrefetchDataMap* data_map) {
715 if (data_map->empty()) 715 if (data_map->empty())
716 return; 716 return;
717 717
718 base::Time oldest_time; 718 uint64_t oldest_time = UINT64_MAX;
719 std::string key_to_delete; 719 std::string key_to_delete;
720 for (PrefetchDataMap::iterator it = data_map->begin(); 720 for (const auto& kv : *data_map) {
721 it != data_map->end(); ++it) { 721 const PrefetchData& data = kv.second;
722 if (key_to_delete.empty() || it->second.last_visit < oldest_time) { 722 if (key_to_delete.empty() || data.last_visit_time() < oldest_time) {
723 key_to_delete = it->first; 723 key_to_delete = data.primary_key();
724 oldest_time = it->second.last_visit; 724 oldest_time = data.last_visit_time();
725 } 725 }
726 } 726 }
727 727
728 data_map->erase(key_to_delete); 728 data_map->erase(key_to_delete);
729 BrowserThread::PostTask( 729 BrowserThread::PostTask(
730 BrowserThread::DB, FROM_HERE, 730 BrowserThread::DB, FROM_HERE,
731 base::Bind( 731 base::Bind(
732 &ResourcePrefetchPredictorTables::DeleteSingleResourceDataPoint, 732 &ResourcePrefetchPredictorTables::DeleteSingleResourceDataPoint,
733 tables_, key_to_delete, key_type)); 733 tables_, key_to_delete, key_type));
734 } 734 }
(...skipping 71 matching lines...) Expand 10 before | Expand all | Expand 10 after
806 // If the primary key is too long reject it. 806 // If the primary key is too long reject it.
807 if (key.length() > ResourcePrefetchPredictorTables::kMaxStringLength) 807 if (key.length() > ResourcePrefetchPredictorTables::kMaxStringLength)
808 return; 808 return;
809 809
810 PrefetchDataMap::iterator cache_entry = data_map->find(key); 810 PrefetchDataMap::iterator cache_entry = data_map->find(key);
811 if (cache_entry == data_map->end()) { 811 if (cache_entry == data_map->end()) {
812 // If the table is full, delete an entry. 812 // If the table is full, delete an entry.
813 if (data_map->size() >= max_data_map_size) 813 if (data_map->size() >= max_data_map_size)
814 RemoveOldestEntryInPrefetchDataMap(key_type, data_map); 814 RemoveOldestEntryInPrefetchDataMap(key_type, data_map);
815 815
816 cache_entry = data_map->insert(std::make_pair( 816 cache_entry = data_map->insert(std::make_pair(key, PrefetchData())).first;
817 key, PrefetchData(key_type, key))).first; 817 cache_entry->second.set_primary_key(key);
818 cache_entry->second.last_visit = base::Time::Now(); 818 cache_entry->second.set_last_visit_time(
819 base::Time::Now().ToInternalValue());
819 size_t new_resources_size = new_resources.size(); 820 size_t new_resources_size = new_resources.size();
820 std::set<GURL> resources_seen; 821 std::set<GURL> resources_seen;
821 for (size_t i = 0; i < new_resources_size; ++i) { 822 for (size_t i = 0; i < new_resources_size; ++i) {
822 if (resources_seen.find(new_resources[i].resource_url) != 823 const URLRequestSummary& summary = new_resources[i];
823 resources_seen.end()) { 824 if (resources_seen.find(summary.resource_url) != resources_seen.end())
824 continue; 825 continue;
825 } 826
826 ResourceData resource_to_add; 827 ResourceData* resource_to_add = cache_entry->second.add_resources();
827 resource_to_add.set_resource_url(new_resources[i].resource_url.spec()); 828 resource_to_add->set_resource_url(summary.resource_url.spec());
828 resource_to_add.set_resource_type(static_cast<ResourceData::ResourceType>( 829 resource_to_add->set_resource_type(
829 new_resources[i].resource_type)); 830 static_cast<ResourceData::ResourceType>(summary.resource_type));
830 resource_to_add.set_number_of_hits(1); 831 resource_to_add->set_number_of_hits(1);
831 resource_to_add.set_average_position(i + 1); 832 resource_to_add->set_average_position(i + 1);
832 resource_to_add.set_priority( 833 resource_to_add->set_priority(
833 static_cast<ResourceData::Priority>(new_resources[i].priority)); 834 static_cast<ResourceData::Priority>(summary.priority));
834 resource_to_add.set_has_validators(new_resources[i].has_validators); 835 resource_to_add->set_has_validators(summary.has_validators);
835 resource_to_add.set_always_revalidate(new_resources[i].always_revalidate); 836 resource_to_add->set_always_revalidate(summary.always_revalidate);
836 cache_entry->second.resources.push_back(resource_to_add); 837
837 resources_seen.insert(new_resources[i].resource_url); 838 resources_seen.insert(summary.resource_url);
838 } 839 }
839 } else { 840 } else {
840 std::vector<ResourceData>& old_resources = cache_entry->second.resources; 841 cache_entry->second.set_last_visit_time(
841 cache_entry->second.last_visit = base::Time::Now(); 842 base::Time::Now().ToInternalValue());
842 843
843 // Build indices over the data. 844 // Build indices over the data.
844 std::map<GURL, int> new_index, old_index; 845 std::map<GURL, int> new_index, old_index;
845 int new_resources_size = static_cast<int>(new_resources.size()); 846 int new_resources_size = static_cast<int>(new_resources.size());
846 for (int i = 0; i < new_resources_size; ++i) { 847 for (int i = 0; i < new_resources_size; ++i) {
847 const URLRequestSummary& summary = new_resources[i]; 848 const URLRequestSummary& summary = new_resources[i];
848 // Take the first occurence of every url. 849 // Take the first occurence of every url.
849 if (new_index.find(summary.resource_url) == new_index.end()) 850 if (new_index.find(summary.resource_url) == new_index.end())
850 new_index[summary.resource_url] = i; 851 new_index[summary.resource_url] = i;
851 } 852 }
852 int old_resources_size = static_cast<int>(old_resources.size()); 853 int old_resources_size =
854 static_cast<int>(cache_entry->second.resources_size());
853 for (int i = 0; i < old_resources_size; ++i) { 855 for (int i = 0; i < old_resources_size; ++i) {
854 const ResourceData& resource = old_resources[i]; 856 const ResourceData& resource = cache_entry->second.resources(i);
855 GURL resource_url(resource.resource_url()); 857 GURL resource_url(resource.resource_url());
856 DCHECK(old_index.find(resource_url) == old_index.end()); 858 DCHECK(old_index.find(resource_url) == old_index.end());
857 old_index[resource_url] = i; 859 old_index[resource_url] = i;
858 } 860 }
859 861
860 // Go through the old urls and update their hit/miss counts. 862 // Go through the old urls and update their hit/miss counts.
861 for (int i = 0; i < old_resources_size; ++i) { 863 for (int i = 0; i < old_resources_size; ++i) {
862 ResourceData& old_resource = old_resources[i]; 864 ResourceData* old_resource = cache_entry->second.mutable_resources(i);
863 GURL resource_url(old_resource.resource_url()); 865 GURL resource_url(old_resource->resource_url());
864 if (new_index.find(resource_url) == new_index.end()) { 866 if (new_index.find(resource_url) == new_index.end()) {
865 old_resource.set_number_of_misses(old_resource.number_of_misses() + 1); 867 old_resource->set_number_of_misses(old_resource->number_of_misses() +
866 old_resource.set_consecutive_misses(old_resource.consecutive_misses() + 868 1);
867 1); 869 old_resource->set_consecutive_misses(
870 old_resource->consecutive_misses() + 1);
868 } else { 871 } else {
869 const URLRequestSummary& new_summary = 872 const URLRequestSummary& new_summary =
870 new_resources[new_index[resource_url]]; 873 new_resources[new_index[resource_url]];
871 874
872 // Update the resource type since it could have changed. 875 // Update the resource type since it could have changed.
873 if (new_summary.resource_type != content::RESOURCE_TYPE_LAST_TYPE) 876 if (new_summary.resource_type != content::RESOURCE_TYPE_LAST_TYPE) {
874 old_resource.set_resource_type( 877 old_resource->set_resource_type(
875 static_cast<ResourceData::ResourceType>( 878 static_cast<ResourceData::ResourceType>(
876 new_summary.resource_type)); 879 new_summary.resource_type));
880 }
877 881
878 old_resource.set_priority( 882 old_resource->set_priority(
879 static_cast<ResourceData::Priority>(new_summary.priority)); 883 static_cast<ResourceData::Priority>(new_summary.priority));
880 884
881 int position = new_index[resource_url] + 1; 885 int position = new_index[resource_url] + 1;
882 int total = 886 int total =
883 old_resource.number_of_hits() + old_resource.number_of_misses(); 887 old_resource->number_of_hits() + old_resource->number_of_misses();
884 old_resource.set_average_position( 888 old_resource->set_average_position(
885 ((old_resource.average_position() * total) + position) / 889 ((old_resource->average_position() * total) + position) /
886 (total + 1)); 890 (total + 1));
887 old_resource.set_number_of_hits(old_resource.number_of_hits() + 1); 891 old_resource->set_number_of_hits(old_resource->number_of_hits() + 1);
888 old_resource.set_consecutive_misses(0); 892 old_resource->set_consecutive_misses(0);
889 } 893 }
890 } 894 }
891 895
892 // Add the new ones that we have not seen before. 896 // Add the new ones that we have not seen before.
893 for (int i = 0; i < new_resources_size; ++i) { 897 for (int i = 0; i < new_resources_size; ++i) {
894 const URLRequestSummary& summary = new_resources[i]; 898 const URLRequestSummary& summary = new_resources[i];
895 if (old_index.find(summary.resource_url) != old_index.end()) 899 if (old_index.find(summary.resource_url) != old_index.end())
896 continue; 900 continue;
897 901
898 // Only need to add new stuff. 902 // Only need to add new stuff.
899 ResourceData resource_to_add; 903 ResourceData* resource_to_add = cache_entry->second.add_resources();
900 resource_to_add.set_resource_url(summary.resource_url.spec()); 904 resource_to_add->set_resource_url(summary.resource_url.spec());
901 resource_to_add.set_resource_type( 905 resource_to_add->set_resource_type(
902 static_cast<ResourceData::ResourceType>(summary.resource_type)); 906 static_cast<ResourceData::ResourceType>(summary.resource_type));
903 resource_to_add.set_number_of_hits(1); 907 resource_to_add->set_number_of_hits(1);
904 resource_to_add.set_average_position(i + 1); 908 resource_to_add->set_average_position(i + 1);
905 resource_to_add.set_priority( 909 resource_to_add->set_priority(
906 static_cast<ResourceData::Priority>(summary.priority)); 910 static_cast<ResourceData::Priority>(summary.priority));
907 resource_to_add.set_has_validators(new_resources[i].has_validators); 911 resource_to_add->set_has_validators(new_resources[i].has_validators);
908 resource_to_add.set_always_revalidate(new_resources[i].always_revalidate); 912 resource_to_add->set_always_revalidate(
909 old_resources.push_back(resource_to_add); 913 new_resources[i].always_revalidate);
910 914
911 // To ensure we dont add the same url twice. 915 // To ensure we dont add the same url twice.
912 old_index[summary.resource_url] = 0; 916 old_index[summary.resource_url] = 0;
913 } 917 }
914 } 918 }
915 919
916 // Trim and sort the resources after the update. 920 // Trim and sort the resources after the update.
917 std::vector<ResourceData>& resources = cache_entry->second.resources; 921 std::vector<ResourceData> resources;
918 for (auto it = resources.begin(); it != resources.end();) { 922 resources.reserve(cache_entry->second.resources_size());
919 if (it->consecutive_misses() >= config_.max_consecutive_misses) 923 for (const ResourceData& resource : cache_entry->second.resources()) {
920 it = resources.erase(it); 924 if (resource.consecutive_misses() < config_.max_consecutive_misses)
921 else 925 resources.push_back(resource);
922 ++it;
923 } 926 }
924 ResourcePrefetchPredictorTables::SortResources(&resources); 927 ResourcePrefetchPredictorTables::SortResources(&resources);
925 if (resources.size() > config_.max_resources_per_entry) 928 if (resources.size() > config_.max_resources_per_entry)
926 resources.resize(config_.max_resources_per_entry); 929 resources.resize(config_.max_resources_per_entry);
927 930
931 cache_entry->second.clear_resources();
932 for (const ResourceData& resource : resources)
933 cache_entry->second.add_resources()->CopyFrom(resource);
934
928 // If the row has no resources, remove it from the cache and delete the 935 // If the row has no resources, remove it from the cache and delete the
929 // entry in the database. Else update the database. 936 // entry in the database. Else update the database.
930 if (resources.empty()) { 937 if (resources.empty()) {
931 data_map->erase(key); 938 data_map->erase(key);
932 BrowserThread::PostTask( 939 BrowserThread::PostTask(
933 BrowserThread::DB, FROM_HERE, 940 BrowserThread::DB, FROM_HERE,
934 base::Bind( 941 base::Bind(
935 &ResourcePrefetchPredictorTables::DeleteSingleResourceDataPoint, 942 &ResourcePrefetchPredictorTables::DeleteSingleResourceDataPoint,
936 tables_, key, key_type)); 943 tables_, key, key_type));
937 } else { 944 } else {
945 PrefetchData empty_data;
946 RedirectData empty_redirect_data;
938 bool is_host = key_type == PREFETCH_KEY_TYPE_HOST; 947 bool is_host = key_type == PREFETCH_KEY_TYPE_HOST;
939 PrefetchData empty_data(
940 !is_host ? PREFETCH_KEY_TYPE_HOST : PREFETCH_KEY_TYPE_URL,
941 std::string());
942 RedirectData empty_redirect_data;
943 const PrefetchData& host_data = is_host ? cache_entry->second : empty_data; 948 const PrefetchData& host_data = is_host ? cache_entry->second : empty_data;
944 const PrefetchData& url_data = is_host ? empty_data : cache_entry->second; 949 const PrefetchData& url_data = is_host ? empty_data : cache_entry->second;
945 BrowserThread::PostTask( 950 BrowserThread::PostTask(
946 BrowserThread::DB, FROM_HERE, 951 BrowserThread::DB, FROM_HERE,
947 base::Bind(&ResourcePrefetchPredictorTables::UpdateData, tables_, 952 base::Bind(&ResourcePrefetchPredictorTables::UpdateData, tables_,
948 url_data, host_data, empty_redirect_data, 953 url_data, host_data, empty_redirect_data,
949 empty_redirect_data)); 954 empty_redirect_data));
950 } 955 }
951 956
952 if (key != key_before_redirects) { 957 if (key != key_before_redirects) {
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after
1016 cache_entry->second.add_redirect_endpoints()->CopyFrom(redirect); 1021 cache_entry->second.add_redirect_endpoints()->CopyFrom(redirect);
1017 1022
1018 if (redirects.empty()) { 1023 if (redirects.empty()) {
1019 redirect_map->erase(cache_entry); 1024 redirect_map->erase(cache_entry);
1020 BrowserThread::PostTask( 1025 BrowserThread::PostTask(
1021 BrowserThread::DB, FROM_HERE, 1026 BrowserThread::DB, FROM_HERE,
1022 base::Bind( 1027 base::Bind(
1023 &ResourcePrefetchPredictorTables::DeleteSingleRedirectDataPoint, 1028 &ResourcePrefetchPredictorTables::DeleteSingleRedirectDataPoint,
1024 tables_, key, key_type)); 1029 tables_, key, key_type));
1025 } else { 1030 } else {
1031 RedirectData empty_redirect_data;
1032 PrefetchData empty_data;
1026 bool is_host = key_type == PREFETCH_KEY_TYPE_HOST; 1033 bool is_host = key_type == PREFETCH_KEY_TYPE_HOST;
1027 RedirectData empty_redirect_data;
1028 PrefetchData empty_url_data(PREFETCH_KEY_TYPE_URL, std::string());
1029 PrefetchData empty_host_data(PREFETCH_KEY_TYPE_HOST, std::string());
1030 const RedirectData& host_redirect_data = 1034 const RedirectData& host_redirect_data =
1031 is_host ? cache_entry->second : empty_redirect_data; 1035 is_host ? cache_entry->second : empty_redirect_data;
1032 const RedirectData& url_redirect_data = 1036 const RedirectData& url_redirect_data =
1033 is_host ? empty_redirect_data : cache_entry->second; 1037 is_host ? empty_redirect_data : cache_entry->second;
1034 BrowserThread::PostTask( 1038 BrowserThread::PostTask(
1035 BrowserThread::DB, FROM_HERE, 1039 BrowserThread::DB, FROM_HERE,
1036 base::Bind(&ResourcePrefetchPredictorTables::UpdateData, tables_, 1040 base::Bind(&ResourcePrefetchPredictorTables::UpdateData, tables_,
1037 empty_url_data, empty_host_data, url_redirect_data, 1041 empty_data, empty_data, url_redirect_data,
1038 host_redirect_data)); 1042 host_redirect_data));
1039 } 1043 }
1040 } 1044 }
1041 1045
1042 void ResourcePrefetchPredictor::OnURLsDeleted( 1046 void ResourcePrefetchPredictor::OnURLsDeleted(
1043 history::HistoryService* history_service, 1047 history::HistoryService* history_service,
1044 bool all_history, 1048 bool all_history,
1045 bool expired, 1049 bool expired,
1046 const history::URLRows& deleted_rows, 1050 const history::URLRows& deleted_rows,
1047 const std::set<GURL>& favicon_urls) { 1051 const std::set<GURL>& favicon_urls) {
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
1079 // HistoryService is already loaded. Continue with Initialization. 1083 // HistoryService is already loaded. Continue with Initialization.
1080 OnHistoryAndCacheLoaded(); 1084 OnHistoryAndCacheLoaded();
1081 return; 1085 return;
1082 } 1086 }
1083 DCHECK(!history_service_observer_.IsObserving(history_service)); 1087 DCHECK(!history_service_observer_.IsObserving(history_service));
1084 history_service_observer_.Add(history_service); 1088 history_service_observer_.Add(history_service);
1085 return; 1089 return;
1086 } 1090 }
1087 1091
1088 } // namespace predictors 1092 } // namespace predictors
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698