Add support for goog-csdinclusionwhite-sha256 in SafeBrowsingDatabase.

chrome/browser/safe_browsing/safe_browsing_database.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
 
 #include <algorithm>
 #include <iterator>
 
 #include "base/bind.h"
@@ skipping 32 matching lines @@
     FILE_PATH_LITERAL(" Prefix Set");
 // Filename suffix for download store.
 const base::FilePath::CharType kDownloadDBFile[] =
     FILE_PATH_LITERAL(" Download");
 // Filename suffix for client-side phishing detection whitelist store.
 const base::FilePath::CharType kCsdWhitelistDBFile[] =
     FILE_PATH_LITERAL(" Csd Whitelist");
 // Filename suffix for the download whitelist store.
 const base::FilePath::CharType kDownloadWhitelistDBFile[] =
     FILE_PATH_LITERAL(" Download Whitelist");
+// Filename suffix for the off-domain inclusion whitelist store.
+const base::FilePath::CharType kInclusionWhitelistDBFile[] =
+    FILE_PATH_LITERAL(" Inclusion Whitelist");
 // Filename suffix for the extension blacklist store.
 const base::FilePath::CharType kExtensionBlacklistDBFile[] =
     FILE_PATH_LITERAL(" Extension Blacklist");
 // Filename suffix for the side-effect free whitelist store.
 const base::FilePath::CharType kSideEffectFreeWhitelistDBFile[] =
     FILE_PATH_LITERAL(" Side-Effect Free Whitelist");
 // Filename suffix for the csd malware IP blacklist store.
 const base::FilePath::CharType kIPBlacklistDBFile[] =
     FILE_PATH_LITERAL(" IP Blacklist");
 // Filename suffix for the unwanted software blacklist store.
@@ skipping 257 matching lines @@
  public:
   SafeBrowsingDatabase* CreateSafeBrowsingDatabase(
       bool enable_download_protection,
       bool enable_client_side_whitelist,
       bool enable_download_whitelist,
       bool enable_extension_blacklist,
       bool enable_side_effect_free_whitelist,
       bool enable_ip_blacklist,
       bool enable_unwanted_software_list) override {
     return new SafeBrowsingDatabaseNew(
-        new SafeBrowsingStoreFile,
+        new SafeBrowsingStoreFile,  // browse_store
         enable_download_protection ? new SafeBrowsingStoreFile : NULL,
         enable_client_side_whitelist ? new SafeBrowsingStoreFile : NULL,
         enable_download_whitelist ? new SafeBrowsingStoreFile : NULL,
+        new SafeBrowsingStoreFile,  // inclusion_whitelist_store
         enable_extension_blacklist ? new SafeBrowsingStoreFile : NULL,
         enable_side_effect_free_whitelist ? new SafeBrowsingStoreFile : NULL,
         enable_ip_blacklist ? new SafeBrowsingStoreFile : NULL,
         enable_unwanted_software_list ? new SafeBrowsingStoreFile : NULL);
   }
 
   SafeBrowsingDatabaseFactoryImpl() { }
 
  private:
   DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseFactoryImpl);
@@ skipping 59 matching lines @@
   return base::FilePath(db_filename.value() + kCsdWhitelistDBFile);
 }
 
 // static
 base::FilePath SafeBrowsingDatabase::DownloadWhitelistDBFilename(
     const base::FilePath& db_filename) {
   return base::FilePath(db_filename.value() + kDownloadWhitelistDBFile);
 }
 
 // static
+base::FilePath SafeBrowsingDatabase::InclusionWhitelistDBFilename(
+    const base::FilePath& db_filename) {
+  return base::FilePath(db_filename.value() + kInclusionWhitelistDBFile);
+}
+
+// static
 base::FilePath SafeBrowsingDatabase::ExtensionBlacklistDBFilename(
     const base::FilePath& db_filename) {
   return base::FilePath(db_filename.value() + kExtensionBlacklistDBFile);
 }
 
 // static
 base::FilePath SafeBrowsingDatabase::SideEffectFreeWhitelistDBFilename(
     const base::FilePath& db_filename) {
   return base::FilePath(db_filename.value() + kSideEffectFreeWhitelistDBFile);
 }
@@ skipping 16 matching lines @@
 
   if (list_id == safe_browsing_util::PHISH ||
       list_id == safe_browsing_util::MALWARE) {
     return browse_store_.get();
   } else if (list_id == safe_browsing_util::BINURL) {
     return download_store_.get();
   } else if (list_id == safe_browsing_util::CSDWHITELIST) {
     return csd_whitelist_store_.get();
   } else if (list_id == safe_browsing_util::DOWNLOADWHITELIST) {
     return download_whitelist_store_.get();
+  } else if (list_id == safe_browsing_util::INCLUSIONWHITELIST) {
+    return inclusion_whitelist_store_.get();
   } else if (list_id == safe_browsing_util::EXTENSIONBLACKLIST) {
     return extension_blacklist_store_.get();
   } else if (list_id == safe_browsing_util::SIDEEFFECTFREEWHITELIST) {
     return side_effect_free_whitelist_store_.get();
   } else if (list_id == safe_browsing_util::IPBLACKLIST) {
     return ip_blacklist_store_.get();
   } else if (list_id == safe_browsing_util::UNWANTEDURL) {
     return unwanted_software_store_.get();
   }
   return NULL;
 }
 
 // static
 void SafeBrowsingDatabase::RecordFailure(FailureType failure_type) {
   UMA_HISTOGRAM_ENUMERATION("SB2.DatabaseFailure", failure_type,
                             FAILURE_DATABASE_MAX);
 }
 
 class SafeBrowsingDatabaseNew::ThreadSafeStateManager::ReadTransaction {
  public:
   const SBWhitelist* GetSBWhitelist(SBWhitelistId id) {
     switch (id) {
       case SBWhitelistId::CSD:
         return &outer_->csd_whitelist_;
       case SBWhitelistId::DOWNLOAD:
         return &outer_->download_whitelist_;
+      case SBWhitelistId::INCLUSION:
+        return &outer_->inclusion_whitelist_;
     }
     NOTREACHED();
     return nullptr;
   }
 
   const IPBlacklist* ip_blacklist() { return &outer_->ip_blacklist_; }
 
   const PrefixSet* GetPrefixSet(PrefixSetId id) {
     switch (id) {
       case PrefixSetId::BROWSE:
@@ skipping 93 matching lines @@
     DCHECK(outer_);
     DCHECK(outer_->thread_checker_.CalledOnValidThread());
   }
 
   SBWhitelist* SBWhitelistForId(SBWhitelistId id) {
     switch (id) {
       case SBWhitelistId::CSD:
         return &outer_->csd_whitelist_;
       case SBWhitelistId::DOWNLOAD:
         return &outer_->download_whitelist_;
+      case SBWhitelistId::INCLUSION:
+        return &outer_->inclusion_whitelist_;
     }
     NOTREACHED();
     return nullptr;
   }
 
   ThreadSafeStateManager* outer_;
   base::AutoLock transaction_lock_;
 
   DISALLOW_COPY_AND_ASSIGN(WriteTransaction);
 };
@@ skipping 17 matching lines @@
   return make_scoped_ptr(new ReadTransaction(
       this, ReadTransaction::AutoLockRequirement::DONT_LOCK_ON_MAIN_THREAD));
 }
 
 scoped_ptr<SafeBrowsingDatabaseNew::WriteTransaction>
 SafeBrowsingDatabaseNew::ThreadSafeStateManager::BeginWriteTransaction() {
   return make_scoped_ptr(new WriteTransaction(this));
 }
 
 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew()
-    : SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile,
-                              NULL,
-                              NULL,
-                              NULL,
-                              NULL,
-                              NULL,
-                              NULL,
-                              NULL) {
+    : SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile,  // browse_store
+                              NULL,    // download_store
+                              NULL,    // csd_whitelist_store
+                              NULL,    // download_whitelist_store
+                              NULL,    // inclusion_whitelist_store
+                              NULL,    // extension_blacklist_store
+                              NULL,    // side_effect_free_whitelist_store
+                              NULL,    // ip_blacklist_store
+                              NULL) {  // unwanted_software_store
   DCHECK(browse_store_.get());
   DCHECK(!download_store_.get());
   DCHECK(!csd_whitelist_store_.get());
   DCHECK(!download_whitelist_store_.get());
+  DCHECK(!inclusion_whitelist_store_.get());
   DCHECK(!extension_blacklist_store_.get());
   DCHECK(!side_effect_free_whitelist_store_.get());
   DCHECK(!ip_blacklist_store_.get());
   DCHECK(!unwanted_software_store_.get());
 }
 
 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew(
     SafeBrowsingStore* browse_store,
     SafeBrowsingStore* download_store,
     SafeBrowsingStore* csd_whitelist_store,
     SafeBrowsingStore* download_whitelist_store,
+    SafeBrowsingStore* inclusion_whitelist_store,
     SafeBrowsingStore* extension_blacklist_store,
     SafeBrowsingStore* side_effect_free_whitelist_store,
     SafeBrowsingStore* ip_blacklist_store,
     SafeBrowsingStore* unwanted_software_store)
     : state_manager_(thread_checker_),
       browse_store_(browse_store),
       download_store_(download_store),
       csd_whitelist_store_(csd_whitelist_store),
       download_whitelist_store_(download_whitelist_store),
+      inclusion_whitelist_store_(inclusion_whitelist_store),
       extension_blacklist_store_(extension_blacklist_store),
       side_effect_free_whitelist_store_(side_effect_free_whitelist_store),
       ip_blacklist_store_(ip_blacklist_store),
       unwanted_software_store_(unwanted_software_store),
       corruption_detected_(false),
       change_detected_(false),
       reset_factory_(this) {
   DCHECK(browse_store_.get());
 }
 
@@ skipping 106 matching lines @@
       LoadWhitelist(full_hashes, SBWhitelistId::DOWNLOAD);
     } else {
       state_manager_.BeginWriteTransaction()->WhitelistEverything(
           SBWhitelistId::DOWNLOAD);
     }
   } else {
     state_manager_.BeginWriteTransaction()->WhitelistEverything(
         SBWhitelistId::DOWNLOAD);  // Just to be safe.
   }
 
+  if (inclusion_whitelist_store_.get()) {
+    inclusion_whitelist_store_->Init(
+        InclusionWhitelistDBFilename(filename_base_),
+        base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase,
+                   base::Unretained(this)));
+
+    std::vector<SBAddFullHash> full_hashes;
+    if (inclusion_whitelist_store_->GetAddFullHashes(&full_hashes)) {
+      LoadWhitelist(full_hashes, SBWhitelistId::INCLUSION);
+    } else {
+      state_manager_.BeginWriteTransaction()->WhitelistEverything(
+          SBWhitelistId::INCLUSION);
+    }
+  } else {
+    state_manager_.BeginWriteTransaction()->WhitelistEverything(
+        SBWhitelistId::INCLUSION);  // Just to be safe.
+  }
+
   if (extension_blacklist_store_.get()) {
     extension_blacklist_store_->Init(
         ExtensionBlacklistDBFilename(filename_base_),
         base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase,
                    base::Unretained(this)));
   }
 
   if (ip_blacklist_store_.get()) {
     ip_blacklist_store_->Init(
         IpBlacklistDBFilename(filename_base_),
@@ skipping 129 matching lines @@
   UrlToFullHashes(url, true, &full_hashes);
   return ContainsWhitelistedHashes(SBWhitelistId::CSD, full_hashes);
 }
 
 bool SafeBrowsingDatabaseNew::ContainsDownloadWhitelistedUrl(const GURL& url) {
   std::vector<SBFullHash> full_hashes;
   UrlToFullHashes(url, true, &full_hashes);
   return ContainsWhitelistedHashes(SBWhitelistId::DOWNLOAD, full_hashes);
 }
 
+bool SafeBrowsingDatabaseNew::ContainsInclusionWhitelistedUrl(const GURL& url) {
+  std::vector<SBFullHash> full_hashes;
+  UrlToFullHashes(url, true, &full_hashes);
+  return ContainsWhitelistedHashes(SBWhitelistId::INCLUSION, full_hashes);
+}
+
 bool SafeBrowsingDatabaseNew::ContainsExtensionPrefixes(
     const std::vector<SBPrefix>& prefixes,
     std::vector<SBPrefix>* prefix_hits) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (!extension_blacklist_store_)
     return false;
 
   return MatchAddPrefixes(extension_blacklist_store_.get(),
                           safe_browsing_util::EXTENSIONBLACKLIST % 2,
@@ skipping 261 matching lines @@
     return false;
   }
 
   if (download_whitelist_store_.get() &&
       !download_whitelist_store_->BeginUpdate()) {
     RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN);
     HandleCorruptDatabase();
     return false;
   }
 
+  if (inclusion_whitelist_store_.get() &&
+      !inclusion_whitelist_store_->BeginUpdate()) {
+    RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN);
+    HandleCorruptDatabase();
+    return false;
+  }
+
   if (extension_blacklist_store_ &&
       !extension_blacklist_store_->BeginUpdate()) {
     RecordFailure(FAILURE_EXTENSION_BLACKLIST_UPDATE_BEGIN);
     HandleCorruptDatabase();
     return false;
   }
 
   if (side_effect_free_whitelist_store_ &&
       !side_effect_free_whitelist_store_->BeginUpdate()) {
     RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_UPDATE_BEGIN);
@@ skipping 28 matching lines @@
   // extra data of that sort.
   UpdateChunkRangesForList(download_store_.get(),
                            safe_browsing_util::kBinUrlList, lists);
 
   UpdateChunkRangesForList(csd_whitelist_store_.get(),
                            safe_browsing_util::kCsdWhiteList, lists);
 
   UpdateChunkRangesForList(download_whitelist_store_.get(),
                            safe_browsing_util::kDownloadWhiteList, lists);
 
+  UpdateChunkRangesForList(inclusion_whitelist_store_.get(),
+                           safe_browsing_util::kInclusionWhitelist, lists);
+
   UpdateChunkRangesForList(extension_blacklist_store_.get(),
                            safe_browsing_util::kExtensionBlacklist, lists);
 
   UpdateChunkRangesForList(side_effect_free_whitelist_store_.get(),
                            safe_browsing_util::kSideEffectFreeWhitelist, lists);
 
   UpdateChunkRangesForList(ip_blacklist_store_.get(),
                            safe_browsing_util::kIPBlacklist, lists);
 
   UpdateChunkRangesForList(unwanted_software_store_.get(),
@@ skipping 22 matching lines @@
       DLOG(ERROR) << "Safe-browsing download database corrupt.";
 
     if (csd_whitelist_store_.get() && !csd_whitelist_store_->CheckValidity())
       DLOG(ERROR) << "Safe-browsing csd whitelist database corrupt.";
 
     if (download_whitelist_store_.get() &&
         !download_whitelist_store_->CheckValidity()) {
       DLOG(ERROR) << "Safe-browsing download whitelist database corrupt.";
     }
 
+    if (inclusion_whitelist_store_.get() &&
+        !inclusion_whitelist_store_->CheckValidity()) {
+      DLOG(ERROR) << "Safe-browsing inclusion whitelist database corrupt.";
+    }
+
     if (extension_blacklist_store_ &&
         !extension_blacklist_store_->CheckValidity()) {
       DLOG(ERROR) << "Safe-browsing extension blacklist database corrupt.";
     }
 
     if (side_effect_free_whitelist_store_ &&
         !side_effect_free_whitelist_store_->CheckValidity()) {
       DLOG(ERROR) << "Safe-browsing side-effect free whitelist database "
                   << "corrupt.";
     }
@@ skipping 18 matching lines @@
     // Track empty updates to answer questions at http://crbug.com/72216 .
     if (update_succeeded && !change_detected_)
       UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", 0);
     browse_store_->CancelUpdate();
     if (download_store_.get())
       download_store_->CancelUpdate();
     if (csd_whitelist_store_.get())
       csd_whitelist_store_->CancelUpdate();
     if (download_whitelist_store_.get())
       download_whitelist_store_->CancelUpdate();
+    if (inclusion_whitelist_store_.get())
+      inclusion_whitelist_store_->CancelUpdate();
     if (extension_blacklist_store_)
       extension_blacklist_store_->CancelUpdate();
     if (side_effect_free_whitelist_store_)
       side_effect_free_whitelist_store_->CancelUpdate();
     if (ip_blacklist_store_)
       ip_blacklist_store_->CancelUpdate();
     if (unwanted_software_store_)
       unwanted_software_store_->CancelUpdate();
     return;
   }
@@ skipping 13 matching lines @@
                           FAILURE_BROWSE_DATABASE_UPDATE_FINISH,
                           FAILURE_BROWSE_PREFIX_SET_WRITE,
                           true);
 
   UpdateWhitelistStore(CsdWhitelistDBFilename(filename_base_),
                        csd_whitelist_store_.get(),
                        SBWhitelistId::CSD);
   UpdateWhitelistStore(DownloadWhitelistDBFilename(filename_base_),
                        download_whitelist_store_.get(),
                        SBWhitelistId::DOWNLOAD);
+  UpdateWhitelistStore(InclusionWhitelistDBFilename(filename_base_),
+                       inclusion_whitelist_store_.get(),
+                       SBWhitelistId::INCLUSION);
 
   if (extension_blacklist_store_) {
     int64 size_bytes = UpdateHashPrefixStore(
         ExtensionBlacklistDBFilename(filename_base_),
         extension_blacklist_store_.get(),
         FAILURE_EXTENSION_BLACKLIST_UPDATE_FINISH);
     UMA_HISTOGRAM_COUNTS("SB2.ExtensionBlacklistKilobytes",
                          static_cast<int>(size_bytes / 1024));
   }
 
@@ skipping 264 matching lines @@
   const bool r3 = csd_whitelist_store_.get() ?
       csd_whitelist_store_->Delete() : true;
   if (!r3)
     RecordFailure(FAILURE_DATABASE_STORE_DELETE);
 
   const bool r4 = download_whitelist_store_.get() ?
       download_whitelist_store_->Delete() : true;
   if (!r4)
     RecordFailure(FAILURE_DATABASE_STORE_DELETE);
 
+  const bool r5 = inclusion_whitelist_store_.get() ?
+      inclusion_whitelist_store_->Delete() : true;
+  if (!r5)
+    RecordFailure(FAILURE_DATABASE_STORE_DELETE);

[mattm, 2015/01/08 22:07:27]

These don't seem to be sorted anyway, so why not just put the new one at the end
for a cleaner diff? 

Or maybe just change the implementation to avoid this problem..
maybe something like:


bool ok = true;

{
  if (!DeleteFile(...) {
    RecordFailure(..)
    ok = false;
  }
}

...

return ok;

[gab, 2015/01/09 13:51:29]

I inserted the |inclusion_whitelist_store_| code right after
|csd_whitelist_store_| and |download_whitelist_store_| everywhere since it's
exactly the same code as the other whitelists, having it together makes this
easier to read IMO (in this specific method all stores are handled the same way,
but I find it easier if all methods at least handle all the stores/lists in the
same order).
Yea, considered this but it would make for a bigger diff and cleaning up is not
really the point of this CL... I totally agree that the multiple bools are
overkill though (besides perhaps when debugging, but I don't think anyone would
have used this property in a while).

Happy to make a follow-up CL if you feel strongly.

[mattm, 2015/01/10 01:10:54]

Since new lists keep getting added, would be nice. Though a more extensive
refactoring would also be nice, dunno how likely that is to happen any time
soon.

[gab, 2015/01/10 06:34:12]

Agreed, I think the most likely refactoring would be one towards complementizing
safe_browsing under components/, but yea dunno how likely that is to happen any
time soon..

[gab, 2015/01/10 06:41:35]

Err... Swipe typo on phone.. s/complementizing/componentizing 

+
   const base::FilePath browse_filename = BrowseDBFilename(filename_base_);
   const base::FilePath bloom_filter_filename =
       BloomFilterForFilename(browse_filename);
-  const bool r5 = base::DeleteFile(bloom_filter_filename, false);
-  if (!r5)
+  const bool r6 = base::DeleteFile(bloom_filter_filename, false);
+  if (!r6)
     RecordFailure(FAILURE_DATABASE_FILTER_DELETE);
 
   const base::FilePath browse_prefix_set_filename =
       PrefixSetForFilename(browse_filename);
-  const bool r6 = base::DeleteFile(browse_prefix_set_filename, false);
-  if (!r6)
+  const bool r7 = base::DeleteFile(browse_prefix_set_filename, false);
+  if (!r7)
     RecordFailure(FAILURE_BROWSE_PREFIX_SET_DELETE);
 
   const base::FilePath extension_blacklist_filename =
       ExtensionBlacklistDBFilename(filename_base_);
-  const bool r7 = base::DeleteFile(extension_blacklist_filename, false);
-  if (!r7)
+  const bool r8 = base::DeleteFile(extension_blacklist_filename, false);
+  if (!r8)
     RecordFailure(FAILURE_EXTENSION_BLACKLIST_DELETE);
 
   const base::FilePath side_effect_free_whitelist_filename =
       SideEffectFreeWhitelistDBFilename(filename_base_);
-  const bool r8 = base::DeleteFile(side_effect_free_whitelist_filename,
+  const bool r9 = base::DeleteFile(side_effect_free_whitelist_filename,
                                    false);
-  if (!r8)
+  if (!r9)
     RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_DELETE);
 
   const base::FilePath side_effect_free_whitelist_prefix_set_filename =
       PrefixSetForFilename(side_effect_free_whitelist_filename);
-  const bool r9 = base::DeleteFile(
+  const bool r10 = base::DeleteFile(
       side_effect_free_whitelist_prefix_set_filename,
       false);
-  if (!r9)
+  if (!r10)
     RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_PREFIX_SET_DELETE);
 
-  const bool r10 = base::DeleteFile(IpBlacklistDBFilename(filename_base_),
+  const bool r11 = base::DeleteFile(IpBlacklistDBFilename(filename_base_),
                                     false);
-  if (!r10)
+  if (!r11)
     RecordFailure(FAILURE_IP_BLACKLIST_DELETE);
 
-  const bool r11 =
+  const bool r12 =
       base::DeleteFile(UnwantedSoftwareDBFilename(filename_base_), false);
-  if (!r11)
+  if (!r12)
     RecordFailure(FAILURE_UNWANTED_SOFTWARE_PREFIX_SET_DELETE);
 
-  return r1 && r2 && r3 && r4 && r5 && r6 && r7 && r8 && r9 && r10 && r11;
+  return r1 && r2 && r3 && r4 && r5 && r6 && r7 && r8 && r9 && r10 && r11 &&
+         r12;
 }
 
 void SafeBrowsingDatabaseNew::WritePrefixSet(const base::FilePath& db_filename,
                                              const PrefixSet* prefix_set,
                                              FailureType write_failure_type) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (!prefix_set)
     return;
 
@@ skipping 94 matching lines @@
 bool SafeBrowsingDatabaseNew::IsCsdWhitelistKillSwitchOn() {
   return state_manager_.BeginReadTransaction()
       ->GetSBWhitelist(SBWhitelistId::CSD)
       ->second;
 }
 
 SafeBrowsingDatabaseNew::PrefixGetHashCache*
 SafeBrowsingDatabaseNew::GetUnsynchronizedPrefixGetHashCacheForTesting() {
   return state_manager_.BeginReadTransaction()->prefix_gethash_cache();
 }