Add the ability to check the CSD Whitelist asynchronously, for PhishGuard.

components/safe_browsing_db/v4_local_database_manager.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // This file should not be build on Android but is currently getting built.
 // TODO(vakh): Fix that: http://crbug.com/621647
 
 #include "components/safe_browsing_db/v4_local_database_manager.h"
 
 #include <vector>
@@ skipping 41 matching lines @@
   return ListInfos({
       ListInfo(kSyncOnlyOnChromeBuilds, "CertCsdDownloadWhitelist.store",
                GetCertCsdDownloadWhitelistId(), SB_THREAT_TYPE_UNUSED),
       ListInfo(kSyncOnlyOnChromeBuilds, "ChromeFilenameClientIncident.store",
                GetChromeFilenameClientIncidentId(), SB_THREAT_TYPE_UNUSED),
       ListInfo(kSyncAlways, "IpMalware.store", GetIpMalwareId(),
                SB_THREAT_TYPE_UNUSED),
       ListInfo(kSyncOnlyOnChromeBuilds, "UrlCsdDownloadWhitelist.store",
                GetUrlCsdDownloadWhitelistId(), SB_THREAT_TYPE_UNUSED),
       ListInfo(kSyncOnlyOnChromeBuilds, "UrlCsdWhitelist.store",
-               GetUrlCsdWhitelistId(), SB_THREAT_TYPE_UNUSED),
+               GetUrlCsdWhitelistId(), SB_THREAT_TYPE_CSD_WHITELIST),
       ListInfo(kSyncAlways, "UrlSoceng.store", GetUrlSocEngId(),
                SB_THREAT_TYPE_URL_PHISHING),
       ListInfo(kSyncAlways, "UrlMalware.store", GetUrlMalwareId(),
                SB_THREAT_TYPE_URL_MALWARE),
       ListInfo(kSyncAlways, "UrlUws.store", GetUrlUwsId(),
                SB_THREAT_TYPE_URL_UNWANTED),
       ListInfo(kSyncAlways, "UrlMalBin.store", GetUrlMalBinId(),
                SB_THREAT_TYPE_BINARY_MALWARE_URL),
       ListInfo(kSyncAlways, "ChromeExtMalware.store", GetChromeExtMalwareId(),
                SB_THREAT_TYPE_EXTENSION),
@@ skipping 16 matching lines @@
     case MALWARE_THREAT:
     case SOCIAL_ENGINEERING_PUBLIC:
     case MALICIOUS_BINARY:
       return 0;
     case UNWANTED_SOFTWARE:
       return 1;
     case API_ABUSE:
     case CLIENT_INCIDENT:
     case SUBRESOURCE_FILTER:
       return 2;
+    case CSD_WHITELIST:
+      return 3;
     default:
       NOTREACHED() << "Unexpected ThreatType encountered: "
                    << list_id.threat_type();
       return kLeastSeverity;
   }
 }
 
 }  // namespace
 
 V4LocalDatabaseManager::PendingCheck::PendingCheck(
@@ skipping 173 matching lines @@
     return true;
   }
 
   std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
       client, ClientCallbackType::CHECK_URL_FOR_SUBRESOURCE_FILTER,
       stores_to_check, std::vector<GURL>(1, url));
 
   return HandleCheck(std::move(check));
 }
 
+AsyncMatch V4LocalDatabaseManager::CheckCsdWhitelistUrl(const GURL& url,
+                                                        Client* client) {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  StoresToCheck stores_to_check({GetUrlCsdWhitelistId()});
+  if (!AreAllStoresAvailableNow(stores_to_check) || !CanCheckUrl(url)) {
+    // Fail open: Whitelist everything. Otherwise we may run the
+    // CSD phishing/malware detector on popular domains and generate
+    // undue load on the client and server, or send Password Reputation
+    // requests on popular sites. This has the effect of disabling
+    // CSD phishing/malware detection and password reputation service
+    // until the store is first synced and/or loaded from disk.
+    return AsyncMatch::MATCH;
+  }
+
+  std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
+      client, ClientCallbackType::CHECK_CSD_WHITELIST, stores_to_check,
+      std::vector<GURL>(1, url));
+
+  return HandleWhitelistCheck(std::move(check));
+}
+
 bool V4LocalDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   StoresToCheck stores_to_check({GetUrlCsdWhitelistId()});
   if (!AreAllStoresAvailableNow(stores_to_check)) {
-    // Fail open: Whitelist everything. Otherwise we may run the
-    // CSD phishing/malware detector on popular domains and generate
-    // undue load on the client and server. This has the effect of disabling
-    // CSD phishing/malware detection until the store is first synced.
+    // Fail open: Whitelist everything. See CheckCsdWhitelistUrl.
     return true;
   }
 
   return HandleUrlSynchronously(url, stores_to_check);
 }
 
 bool V4LocalDatabaseManager::MatchDownloadWhitelistString(
     const std::string& str) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
@@ skipping 236 matching lines @@
     const ListIdentifier& list_id) {
   auto it = std::find_if(
       std::begin(list_infos_), std::end(list_infos_),
       [&list_id](ListInfo const& li) { return li.list_id() == list_id; });
   DCHECK(list_infos_.end() != it);
   DCHECK_NE(SB_THREAT_TYPE_SAFE, it->sb_threat_type());
   DCHECK_NE(SB_THREAT_TYPE_UNUSED, it->sb_threat_type());
   return it->sb_threat_type();
 }
 
+AsyncMatch V4LocalDatabaseManager::HandleWhitelistCheck(
+    std::unique_ptr<PendingCheck> check) {
+  // We don't bother queuing whitelist checks since the DB will
+  // normally be available already -- whitelists are used after page load,
+  // and navigations are blocked until the DB is ready and dequeues checks.
+  // The caller should have already checked that the DB is ready.
+  DCHECK(v4_database_);
+
+  FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
+  if (!GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes)) {
+    return AsyncMatch::NO_MATCH;
+  }
+
+  // Look for any full-length hash in the matches. If there is one,
+  // there's no need for a full-hash check. This saves bandwidth for
+  // very popular sites since they'll have full-length hashes locally.
+  // These loops will have exactly 1 entry most of the time.
+  for (const auto& entry : full_hash_to_store_and_hash_prefixes) {
+    for (const auto& store_and_prefix : entry.second) {
+      if (store_and_prefix.hash_prefix.size() == kMaxHashPrefixLength)
+        return AsyncMatch::MATCH;
+    }
+  }
+
+  ScheduleFullHashCheck(std::move(check), full_hash_to_store_and_hash_prefixes);
+  return AsyncMatch::ASYNC;
+}
+
 bool V4LocalDatabaseManager::HandleCheck(std::unique_ptr<PendingCheck> check) {
   if (!v4_database_) {
     queued_checks_.push_back(std::move(check));
     return false;
   }
 
   FullHashToStoreAndHashPrefixesMap full_hash_to_store_and_hash_prefixes;
   if (!GetPrefixMatches(check, &full_hash_to_store_and_hash_prefixes)) {
     return true;
   }
 
+  ScheduleFullHashCheck(std::move(check), full_hash_to_store_and_hash_prefixes);
+  return false;
+}
+
+void V4LocalDatabaseManager::ScheduleFullHashCheck(
+    std::unique_ptr<PendingCheck> check,
+    const FullHashToStoreAndHashPrefixesMap&
+        full_hash_to_store_and_hash_prefixes) {
   // Add check to pending_checks_ before scheduling PerformFullHashCheck so that
   // even if the client calls CancelCheck before PerformFullHashCheck gets
   // called, the check can be found in pending_checks_.
   pending_checks_.insert(check.get());
 
   // Post on the IO thread to enforce async behavior.
   BrowserThread::PostTask(
       BrowserThread::IO, FROM_HERE,
       base::Bind(&V4LocalDatabaseManager::PerformFullHashCheck,
                  weak_factory_.GetWeakPtr(), base::Passed(std::move(check)),
                  full_hash_to_store_and_hash_prefixes));
-
-  return false;
 }
 
 bool V4LocalDatabaseManager::HandleHashSynchronously(
     const FullHash& hash,
     const StoresToCheck& stores_to_check) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   std::set<FullHash> hashes{hash};
   std::unique_ptr<PendingCheck> check = base::MakeUnique<PendingCheck>(
       nullptr, ClientCallbackType::CHECK_OTHER, stores_to_check, hashes);
@@ skipping 102 matching lines @@
                                               check->most_severe_threat_type);
       break;
 
     case ClientCallbackType::CHECK_RESOURCE_URL:
       DCHECK_EQ(1u, check->urls.size());
       check->client->OnCheckResourceUrlResult(check->urls[0],
                                               check->most_severe_threat_type,
                                               check->matching_full_hash);
       break;
 
+    case ClientCallbackType::CHECK_CSD_WHITELIST: {
+      DCHECK_EQ(1u, check->urls.size());
+      DCHECK(check->most_severe_threat_type == SB_THREAT_TYPE_SAFE ||
+             check->most_severe_threat_type == SB_THREAT_TYPE_CSD_WHITELIST);
+      bool did_match_whitelist =

[vakh (use Gerrit instead), 2017/05/24 16:01:17]

would it be clearer to have it the other way:

bool did_match_whitelist = check->most_severe_threat_type ==
SB_THREAT_TYPE_CSD_WHITELIST;
DCHECK(did_match_whitelist || check->most_severe_threat_type ==
SB_THREAT_TYPE_SAFE);

?

[Nathan Parker, 2017/05/31 23:40:11]

yea, I think it is clearer, but I had it the other way to be more conservative
since then any non-safe verdict would be considered "whitelisted".  Actually
that seems really unlikley.  I'll do it your way.

+          check->most_severe_threat_type != SB_THREAT_TYPE_SAFE;
+      check->client->OnCheckWhitelistUrlResult(did_match_whitelist);
+      break;
+    }
+
     case ClientCallbackType::CHECK_EXTENSION_IDS: {
       DCHECK_EQ(check->full_hash_threat_types.size(),
                 check->full_hashes.size());
       std::set<FullHash> unsafe_extension_ids;
       for (size_t i = 0; i < check->full_hash_threat_types.size(); i++) {
         if (check->full_hash_threat_types[i] == SB_THREAT_TYPE_EXTENSION) {
           unsafe_extension_ids.insert(check->full_hashes[i]);
         }
       }
       check->client->OnCheckExtensionsResult(unsafe_extension_ids);
@@ skipping 51 matching lines @@
          v4_database_->AreAllStoresAvailable(stores_to_check);
 }
 
 bool V4LocalDatabaseManager::AreAnyStoresAvailableNow(
     const StoresToCheck& stores_to_check) const {
   return enabled_ && v4_database_ &&
          v4_database_->AreAnyStoresAvailable(stores_to_check);
 }
 
 }  // namespace safe_browsing