predictors: Add origin learning.

chrome/browser/predictors/resource_prefetch_predictor_tables.cc

Index: chrome/browser/predictors/resource_prefetch_predictor_tables.cc
diff --git a/chrome/browser/predictors/resource_prefetch_predictor_tables.cc b/chrome/browser/predictors/resource_prefetch_predictor_tables.cc
index ae833342805bffb4fa49147a24b892918843308e..841d99a353e672acc14d2f36a939d37f3b23d7c5 100644
--- a/chrome/browser/predictors/resource_prefetch_predictor_tables.cc
+++ b/chrome/browser/predictors/resource_prefetch_predictor_tables.cc
@@ -25,6 +25,7 @@ const char kHostResourceTableName[] = "resource_prefetch_predictor_host";
 const char kHostRedirectTableName[] =
     "resource_prefetch_predictor_host_redirect";
 const char kManifestTableName[] = "resource_prefetch_predictor_manifest";
+const char kOriginTableName[] = "resource_prefetch_predictor_origin";
 
 const char kCreateGlobalMetadataStatementTemplate[] =
     "CREATE TABLE %s ( "
@@ -35,9 +36,10 @@ const char kCreateProtoTableStatementTemplate[] =
     "key TEXT, "
     "proto BLOB, "
     "PRIMARY KEY(key))";
-const char kInsertProtoTableStatementTemplate[] =
+const char kInsertProtoStatementTemplate[] =
     "INSERT INTO %s (key, proto) VALUES (?,?)";
-const char kDeleteProtoTableStatementTemplate[] = "DELETE FROM %s WHERE key=?";
+const char kDeleteProtoStatementTemplate[] = "DELETE FROM %s WHERE key=?";
+const char kSelectAllStatementTemplate[] = "SELECT * FROM %s";
 
 void BindProtoDataToStatement(const std::string& key,
                               const MessageLite& data,
@@ -109,12 +111,34 @@ void ResourcePrefetchPredictorTables::TrimRedirects(
       new_end, data->mutable_redirect_endpoints()->end());
 }
 
+// static
+void ResourcePrefetchPredictorTables::TrimOrigins(
+    OriginData* data,
+    size_t max_consecutive_misses) {
+  auto* origins = data->mutable_origins();
+  auto new_end = std::remove_if(
+      origins->begin(), origins->end(), [=](const OriginStat& x) {
+        return x.consecutive_misses() >= max_consecutive_misses;
+      });
+  origins->erase(new_end, origins->end());
+}
+
+// static
+void ResourcePrefetchPredictorTables::SortOrigins(OriginData* data) {
+  std::sort(data->mutable_origins()->begin(), data->mutable_origins()->end(),
+            [](const OriginStat& x, const OriginStat& y) {
+              // Decreasing score ordering.
+              return ComputeOriginScore(x) > ComputeOriginScore(y);
+            });
+}
+
 void ResourcePrefetchPredictorTables::GetAllData(
     PrefetchDataMap* url_data_map,
     PrefetchDataMap* host_data_map,
     RedirectDataMap* url_redirect_data_map,
     RedirectDataMap* host_redirect_data_map,
-    ManifestDataMap* manifest_map) {
+    ManifestDataMap* manifest_map,
+    OriginDataMap* origin_data_map) {
   TRACE_EVENT0("browser", "ResourcePrefetchPredictor::GetAllData");
   DCHECK_CURRENTLY_ON(BrowserThread::DB);
   if (CantAccessDatabase())
@@ -124,17 +148,21 @@ void ResourcePrefetchPredictorTables::GetAllData(
   DCHECK(host_data_map);
   DCHECK(url_redirect_data_map);
   DCHECK(host_redirect_data_map);
+  DCHECK(manifest_map);
+  DCHECK(origin_data_map);
   url_data_map->clear();
   host_data_map->clear();
   url_redirect_data_map->clear();
   host_redirect_data_map->clear();
   manifest_map->clear();
+  origin_data_map->clear();
 
   GetAllResourceDataHelper(PREFETCH_KEY_TYPE_URL, url_data_map);
   GetAllResourceDataHelper(PREFETCH_KEY_TYPE_HOST, host_data_map);
   GetAllRedirectDataHelper(PREFETCH_KEY_TYPE_URL, url_redirect_data_map);
   GetAllRedirectDataHelper(PREFETCH_KEY_TYPE_HOST, host_redirect_data_map);
   GetAllManifestDataHelper(manifest_map);
+  GetAllOriginDataHelper(origin_data_map);
 }
 
 void ResourcePrefetchPredictorTables::UpdateData(
@@ -189,6 +217,23 @@ void ResourcePrefetchPredictorTables::UpdateManifestData(
     DB()->CommitTransaction();
 }
 
+void ResourcePrefetchPredictorTables::UpdateOriginData(
+    const std::string& host,

[alexilin, 2017/04/10 14:58:28]

nit: 
You could get rid of the host parameter because the OriginData contains a host.
This is not true for a PrecacheManifest.

[Benoit L, 2017/04/11 09:04:05]

I retained it for symmetry with the rest of the code, but that indeed a bit
heavy-handed.
Done.

+    const OriginData& origin_data) {
+  DCHECK_CURRENTLY_ON(BrowserThread::DB);
+  if (CantAccessDatabase())
+    return;
+
+  DB()->BeginTransaction();
+  bool success = UpdateDataHelper(PREFETCH_KEY_TYPE_HOST,
+                                  PrefetchDataType::ORIGIN, host, origin_data);
+
+  if (!success)
+    DB()->RollbackTransaction();
+  else
+    DB()->CommitTransaction();
+}
+
 void ResourcePrefetchPredictorTables::DeleteResourceData(
     const std::vector<std::string>& urls,
     const std::vector<std::string>& hosts) {
@@ -248,6 +293,15 @@ void ResourcePrefetchPredictorTables::DeleteManifestData(
   DeleteDataHelper(PREFETCH_KEY_TYPE_HOST, PrefetchDataType::MANIFEST, hosts);
 }
 
+void ResourcePrefetchPredictorTables::DeleteOriginData(
+    const std::vector<std::string>& hosts) {
+  DCHECK_CURRENTLY_ON(BrowserThread::DB);
+  if (CantAccessDatabase())
+    return;
+
+  DeleteDataHelper(PREFETCH_KEY_TYPE_HOST, PrefetchDataType::ORIGIN, hosts);
+}
+
 void ResourcePrefetchPredictorTables::DeleteAllData() {
   DCHECK_CURRENTLY_ON(BrowserThread::DB);
   if (CantAccessDatabase())
@@ -273,7 +327,7 @@ void ResourcePrefetchPredictorTables::GetAllResourceDataHelper(
   // Read the resources table and organize it per primary key.
   const char* table_name = GetTableName(key_type, PrefetchDataType::RESOURCE);
   sql::Statement resource_reader(DB()->GetUniqueStatement(
-      base::StringPrintf("SELECT * FROM %s", table_name).c_str()));
+      base::StringPrintf(kSelectAllStatementTemplate, table_name).c_str()));
 
   PrefetchData data;
   std::string key;
@@ -294,7 +348,7 @@ void ResourcePrefetchPredictorTables::GetAllRedirectDataHelper(
   // Read the redirects table and organize it per primary key.
   const char* table_name = GetTableName(key_type, PrefetchDataType::REDIRECT);
   sql::Statement redirect_reader(DB()->GetUniqueStatement(
-      base::StringPrintf("SELECT * FROM %s", table_name).c_str()));
+      base::StringPrintf(kSelectAllStatementTemplate, table_name).c_str()));
 
   RedirectData data;
   std::string key;
@@ -307,7 +361,8 @@ void ResourcePrefetchPredictorTables::GetAllRedirectDataHelper(
 void ResourcePrefetchPredictorTables::GetAllManifestDataHelper(
     ManifestDataMap* manifest_map) {
   sql::Statement manifest_reader(DB()->GetUniqueStatement(
-      base::StringPrintf("SELECT * FROM %s", kManifestTableName).c_str()));
+      base::StringPrintf(kSelectAllStatementTemplate, kManifestTableName)
+          .c_str()));
 
   precache::PrecacheManifest data;
   std::string key;
@@ -316,6 +371,18 @@ void ResourcePrefetchPredictorTables::GetAllManifestDataHelper(
   }
 }
 
+void ResourcePrefetchPredictorTables::GetAllOriginDataHelper(
+    OriginDataMap* origin_map) {
+  sql::Statement reader(DB()->GetUniqueStatement(
+      base::StringPrintf(kSelectAllStatementTemplate, kOriginTableName)
+          .c_str()));
+
+  OriginData data;
+  std::string key;
+  while (StepAndInitializeProtoData(&reader, &key, &data))
+    origin_map->insert({key, data});

[alexilin, 2017/04/10 14:58:28]

Add `DCHECK_EQ(data.host(), key)`

[Benoit L, 2017/04/11 09:04:05]

Done.

+}
+
 bool ResourcePrefetchPredictorTables::UpdateDataHelper(
     PrefetchKeyType key_type,
     PrefetchDataType data_type,
@@ -392,6 +459,27 @@ float ResourcePrefetchPredictorTables::ComputeResourceScore(
          data.average_position();
 }
 
+float ResourcePrefetchPredictorTables::ComputeOriginScore(
+    const OriginStat& origin) {
+  // The ranking is done by considering, in this order:
+  // 1. High confidence resources (>75% and more than 10 hits)
+  // 2. Mandatory network access
+  // 3. Network accessed
+  // 4. Average position (decreasing)
+  float score = 0;
+  float confidence = static_cast<float>(origin.number_of_hits()) /
+                     (origin.number_of_hits() + origin.number_of_misses());
+
+  bool is_high_confidence = confidence > .75 && origin.number_of_hits() > 10;
+  score += is_high_confidence * 1e6;
+
+  score += origin.always_access_network() * 1e4;
+  score += origin.accessed_network() * 1e2;
+  score += 1e2 - origin.average_position();
+
+  return score;
+}
+
 // static
 bool ResourcePrefetchPredictorTables::DropTablesIfOutdated(
     sql::Connection* db) {
@@ -410,7 +498,7 @@ bool ResourcePrefetchPredictorTables::DropTablesIfOutdated(
     for (const char* table_name :
          {kMetadataTableName, kUrlResourceTableName, kHostResourceTableName,
           kUrlRedirectTableName, kHostRedirectTableName, kManifestTableName,
-          kUrlMetadataTableName, kHostMetadataTableName}) {
+          kUrlMetadataTableName, kHostMetadataTableName, kOriginTableName}) {
       success =
           success &&
           db->Execute(base::StringPrintf("DROP TABLE IF EXISTS %s", table_name)
@@ -467,7 +555,7 @@ void ResourcePrefetchPredictorTables::CreateTableIfNonExistent() {
 
   for (const char* table_name :
        {kUrlResourceTableName, kHostResourceTableName, kUrlRedirectTableName,
-        kHostRedirectTableName, kManifestTableName}) {
+        kHostRedirectTableName, kManifestTableName, kOriginTableName}) {
     success = success &&
               (db->DoesTableExist(table_name) ||
                db->Execute(base::StringPrintf(
@@ -511,9 +599,9 @@ ResourcePrefetchPredictorTables::GetTableUpdateStatement(
     TableOperationType op_type) {
   sql::StatementID id(__FILE__, key_type | (static_cast<int>(data_type) << 1) |
                                     (static_cast<int>(op_type) << 3));
-  const char* statement_template = (op_type == TableOperationType::REMOVE
-                                        ? kDeleteProtoTableStatementTemplate
-                                        : kInsertProtoTableStatementTemplate);
+  const char* statement_template =
+      (op_type == TableOperationType::REMOVE ? kDeleteProtoStatementTemplate
+                                             : kInsertProtoStatementTemplate);
   const char* table_name = GetTableName(key_type, data_type);
   return base::MakeUnique<sql::Statement>(DB()->GetCachedStatement(
       id, base::StringPrintf(statement_template, table_name).c_str()));
@@ -531,6 +619,8 @@ const char* ResourcePrefetchPredictorTables::GetTableName(
       return is_host ? kHostRedirectTableName : kUrlRedirectTableName;
     case PrefetchDataType::MANIFEST:
       return kManifestTableName;
+    case PrefetchDataType::ORIGIN:
+      return kOriginTableName;
   }
 
   NOTREACHED();