Add a ModuleLoadAnalyzer which checks modules against a whitelist

chrome/browser/safe_browsing/safe_browsing_database.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
 
 #include <stddef.h>
 #include <stdint.h>
 #include <algorithm>
 #include <iterator>
@@ skipping 52 matching lines @@
     FILE_PATH_LITERAL(" Extension Blacklist");
 // Filename suffix for the side-effect free whitelist store.
 const base::FilePath::CharType kSideEffectFreeWhitelistDBFile[] =
     FILE_PATH_LITERAL(" Side-Effect Free Whitelist");
 // Filename suffix for the csd malware IP blacklist store.
 const base::FilePath::CharType kIPBlacklistDBFile[] =
     FILE_PATH_LITERAL(" IP Blacklist");
 // Filename suffix for the unwanted software blacklist store.
 const base::FilePath::CharType kUnwantedSoftwareDBFile[] =
     FILE_PATH_LITERAL(" UwS List");
+const base::FilePath::CharType kModuleWhitelistDBFile[] =
+    FILE_PATH_LITERAL(" Module Whitelist");
 
 // Filename suffix for browse store.
 // TODO(shess): "Safe Browsing Bloom Prefix Set" is full of win.
 // Unfortunately, to change the name implies lots of transition code
 // for little benefit.  If/when file formats change (say to put all
 // the data in one file), that would be a convenient point to rectify
 // this.
 const base::FilePath::CharType kBrowseDBFile[] = FILE_PATH_LITERAL(" Bloom");
 
-// Maximum number of entries we allow in any of the whitelists.
+// Maximum number of entries we allow in any of the whitelists, excluding the
+// module whitelist.

[robertshield, 2016/02/05 21:20:32]

Just curious, have we tried a massive whitelist here to see what it does to
performance?

[proberge, 2016/02/05 22:26:10]

I have not. afaik there's no simple way to fake a part of the chunk server.

 // If a whitelist on disk contains more entries then all lookups to
 // the whitelist will be considered a match.
 const size_t kMaxWhitelistSize = 5000;
 
 // If the hash of this exact expression is on a whitelist then all
 // lookups to this whitelist will be considered a match.
 const char kWhitelistKillSwitchUrl[] =
     "sb-ssl.google.com/safebrowsing/csd/killswitch";  // Don't change this!
 
 // If the hash of this exact expression is on a whitelist then the
@@ skipping 179 matching lines @@
 // The default SafeBrowsingDatabaseFactory.
 class SafeBrowsingDatabaseFactoryImpl : public SafeBrowsingDatabaseFactory {
  public:
   SafeBrowsingDatabase* CreateSafeBrowsingDatabase(
       const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
       bool enable_download_protection,
       bool enable_client_side_whitelist,
       bool enable_download_whitelist,
       bool enable_extension_blacklist,
       bool enable_ip_blacklist,
-      bool enable_unwanted_software_list) override {
+      bool enable_unwanted_software_list,
+      bool enable_module_whitelist) override {
     return new SafeBrowsingDatabaseNew(
         db_task_runner, CreateStore(true, db_task_runner),  // browse_store
         CreateStore(enable_download_protection, db_task_runner),
         CreateStore(enable_client_side_whitelist, db_task_runner),
         CreateStore(enable_download_whitelist, db_task_runner),
         CreateStore(true, db_task_runner),  // inclusion_whitelist_store
         CreateStore(enable_extension_blacklist, db_task_runner),
         CreateStore(enable_ip_blacklist, db_task_runner),
-        CreateStore(enable_unwanted_software_list, db_task_runner));
+        CreateStore(enable_unwanted_software_list, db_task_runner),
+        CreateStore(enable_module_whitelist, db_task_runner));
   }
 
   SafeBrowsingDatabaseFactoryImpl() {}
 
  private:
   DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseFactoryImpl);
 };
 
 // static
 SafeBrowsingDatabaseFactory* SafeBrowsingDatabase::factory_ = NULL;
 
 // Factory method, should be called on the Safe Browsing sequenced task runner,
 // which is also passed to the function as |current_task_runner|.
 // TODO(shess): There's no need for a factory any longer.  Convert
 // SafeBrowsingDatabaseNew to SafeBrowsingDatabase, and have Create()
 // callers just construct things directly.
 SafeBrowsingDatabase* SafeBrowsingDatabase::Create(
     const scoped_refptr<base::SequencedTaskRunner>& current_task_runner,
     bool enable_download_protection,
     bool enable_client_side_whitelist,
     bool enable_download_whitelist,
     bool enable_extension_blacklist,
     bool enable_ip_blacklist,
-    bool enable_unwanted_software_list) {
+    bool enable_unwanted_software_list,
+    bool enable_module_whitelist) {
   DCHECK(current_task_runner->RunsTasksOnCurrentThread());
   if (!factory_)
     factory_ = new SafeBrowsingDatabaseFactoryImpl();
   return factory_->CreateSafeBrowsingDatabase(
       current_task_runner, enable_download_protection,
       enable_client_side_whitelist, enable_download_whitelist,
       enable_extension_blacklist, enable_ip_blacklist,
-      enable_unwanted_software_list);
+      enable_unwanted_software_list, enable_module_whitelist);
 }
 
 SafeBrowsingDatabase::~SafeBrowsingDatabase() {}
 
 // static
 base::FilePath SafeBrowsingDatabase::BrowseDBFilename(
     const base::FilePath& db_base_filename) {
   return base::FilePath(db_base_filename.value() + kBrowseDBFile);
 }
 
@@ skipping 51 matching lines @@
   return base::FilePath(db_filename.value() + kIPBlacklistDBFile);
 }
 
 // static
 base::FilePath SafeBrowsingDatabase::UnwantedSoftwareDBFilename(
     const base::FilePath& db_filename) {
   return base::FilePath(db_filename.value() + kUnwantedSoftwareDBFile);
 }
 
 // static
+base::FilePath SafeBrowsingDatabase::ModuleWhitelistDBFilename(
+    const base::FilePath& db_filename) {
+  return base::FilePath(db_filename.value() + kModuleWhitelistDBFile);
+}
+
+// static
 void SafeBrowsingDatabase::GetDownloadUrlPrefixes(
     const std::vector<GURL>& urls,
     std::vector<SBPrefix>* prefixes) {
   std::vector<SBFullHash> full_hashes;
   for (size_t i = 0; i < urls.size(); ++i)
     UrlToFullHashes(urls[i], false, &full_hashes);
 
   for (size_t i = 0; i < full_hashes.size(); ++i)
     prefixes->push_back(full_hashes[i].prefix);
 }
@@ skipping 11 matching lines @@
   } else if (list_id == DOWNLOADWHITELIST) {
     return download_whitelist_store_.get();
   } else if (list_id == INCLUSIONWHITELIST) {
     return inclusion_whitelist_store_.get();
   } else if (list_id == EXTENSIONBLACKLIST) {
     return extension_blacklist_store_.get();
   } else if (list_id == IPBLACKLIST) {
     return ip_blacklist_store_.get();
   } else if (list_id == UNWANTEDURL) {
     return unwanted_software_store_.get();
+  } else if (list_id == MODULEWHITELIST) {
+    return module_whitelist_store_.get();
   }
   return NULL;
 }
 
 // static
 void SafeBrowsingDatabase::RecordFailure(FailureType failure_type) {
   UMA_HISTOGRAM_ENUMERATION("SB2.DatabaseFailure", failure_type,
                             FAILURE_DATABASE_MAX);
 }
 
 class SafeBrowsingDatabaseNew::ThreadSafeStateManager::ReadTransaction {
  public:
   const SBWhitelist* GetSBWhitelist(SBWhitelistId id) {
     switch (id) {
       case SBWhitelistId::CSD:
         return &outer_->csd_whitelist_;
       case SBWhitelistId::DOWNLOAD:
         return &outer_->download_whitelist_;
       case SBWhitelistId::INCLUSION:
         return &outer_->inclusion_whitelist_;
+      case SBWhitelistId::MODULE:
+        return &outer_->module_whitelist_;
     }
     NOTREACHED();
     return nullptr;
   }
 
   const IPBlacklist* ip_blacklist() { return &outer_->ip_blacklist_; }
 
   const PrefixSet* GetPrefixSet(PrefixSetId id) {
     switch (id) {
       case PrefixSetId::BROWSE:
@@ skipping 90 matching lines @@
   }
 
   SBWhitelist* SBWhitelistForId(SBWhitelistId id) {
     switch (id) {
       case SBWhitelistId::CSD:
         return &outer_->csd_whitelist_;
       case SBWhitelistId::DOWNLOAD:
         return &outer_->download_whitelist_;
       case SBWhitelistId::INCLUSION:
         return &outer_->inclusion_whitelist_;
+      case SBWhitelistId::MODULE:
+        return &outer_->module_whitelist_;
     }
     NOTREACHED();
     return nullptr;
   }
 
   ThreadSafeStateManager* outer_;
   base::AutoLock transaction_lock_;
 
   DISALLOW_COPY_AND_ASSIGN(WriteTransaction);
 };
@@ skipping 32 matching lines @@
 
 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew(
     const scoped_refptr<base::SequencedTaskRunner>& db_task_runner,
     SafeBrowsingStore* browse_store,
     SafeBrowsingStore* download_store,
     SafeBrowsingStore* csd_whitelist_store,
     SafeBrowsingStore* download_whitelist_store,
     SafeBrowsingStore* inclusion_whitelist_store,
     SafeBrowsingStore* extension_blacklist_store,
     SafeBrowsingStore* ip_blacklist_store,
-    SafeBrowsingStore* unwanted_software_store)
+    SafeBrowsingStore* unwanted_software_store,
+    SafeBrowsingStore* module_whitelist_store)
     : db_task_runner_(db_task_runner),
       state_manager_(db_task_runner_),
       db_state_manager_(db_task_runner_),
       browse_store_(browse_store),
       download_store_(download_store),
       csd_whitelist_store_(csd_whitelist_store),
       download_whitelist_store_(download_whitelist_store),
       inclusion_whitelist_store_(inclusion_whitelist_store),
       extension_blacklist_store_(extension_blacklist_store),
       ip_blacklist_store_(ip_blacklist_store),
       unwanted_software_store_(unwanted_software_store),
+      module_whitelist_store_(module_whitelist_store),
       reset_factory_(this) {
   DCHECK(browse_store_.get());
 }
 
 SafeBrowsingDatabaseNew::~SafeBrowsingDatabaseNew() {
   // The DCHECK is disabled due to crbug.com/338486 .
   // DCHECK(db_task_runner_->RunsTasksOnCurrentThread());
 }
 
 void SafeBrowsingDatabaseNew::Init(const base::FilePath& filename_base) {
@@ skipping 126 matching lines @@
         base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase,
                    base::Unretained(this)));
 
     std::vector<SBAddFullHash> full_hashes;
     if (ip_blacklist_store_->GetAddFullHashes(&full_hashes)) {
       LoadIpBlacklist(full_hashes);
     } else {
       LoadIpBlacklist(std::vector<SBAddFullHash>());  // Clear the list.
     }
   }
+
+  if (module_whitelist_store_.get()) {
+    module_whitelist_store_->Init(
+        ModuleWhitelistDBFilename(db_state_manager_.filename_base()),
+        base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase,
+                   base::Unretained(this)));
+
+    std::vector<SBAddFullHash> full_hashes;
+    if (module_whitelist_store_->GetAddFullHashes(&full_hashes)) {
+      LoadWhitelist(full_hashes, SBWhitelistId::MODULE);
+    } else {
+      state_manager_.BeginWriteTransaction()->WhitelistEverything(
+          SBWhitelistId::MODULE);
+    }
+  } else {
+    state_manager_.BeginWriteTransaction()->WhitelistEverything(
+        SBWhitelistId::MODULE);  // Just to be safe.
+  }
 }
 
 bool SafeBrowsingDatabaseNew::ResetDatabase() {
   DCHECK(db_task_runner_->RunsTasksOnCurrentThread());
 
   // Delete files on disk.
   // TODO(shess): Hard to see where one might want to delete without a
   // reset.  Perhaps inline |Delete()|?
   if (!Delete())
     return false;
@@ skipping 169 matching lines @@
   return false;
 }
 
 bool SafeBrowsingDatabaseNew::ContainsDownloadWhitelistedString(
     const std::string& str) {
   std::vector<SBFullHash> hashes;
   hashes.push_back(SBFullHashForString(str));
   return ContainsWhitelistedHashes(SBWhitelistId::DOWNLOAD, hashes);
 }
 
+bool SafeBrowsingDatabaseNew::ContainsModuleWhitelistedString(
+    const std::string& str) {
+  std::vector<SBFullHash> hashes;
+  hashes.push_back(SBFullHashForString(str));
+  return ContainsWhitelistedHashes(SBWhitelistId::MODULE, hashes);
+}
+
 bool SafeBrowsingDatabaseNew::ContainsWhitelistedHashes(
     SBWhitelistId whitelist_id,
     const std::vector<SBFullHash>& hashes) {
   scoped_ptr<ReadTransaction> txn = state_manager_.BeginReadTransaction();
   const SBWhitelist* whitelist = txn->GetSBWhitelist(whitelist_id);
   if (whitelist->second)
     return true;
   for (std::vector<SBFullHash>::const_iterator it = hashes.begin();
        it != hashes.end(); ++it) {
     if (std::binary_search(whitelist->first.begin(), whitelist->first.end(),
@@ skipping 203 matching lines @@
     HandleCorruptDatabase();
     return false;
   }
 
   if (unwanted_software_store_ && !unwanted_software_store_->BeginUpdate()) {
     RecordFailure(FAILURE_UNWANTED_SOFTWARE_DATABASE_UPDATE_BEGIN);
     HandleCorruptDatabase();
     return false;
   }
 
+  if (module_whitelist_store_.get() &&
+      !module_whitelist_store_->BeginUpdate()) {
+    RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN);
+    HandleCorruptDatabase();
+    return false;
+  }
+
   // Cached fullhash results must be cleared on every database update (whether
   // successful or not).
   state_manager_.BeginWriteTransaction()->clear_prefix_gethash_cache();
 
   UpdateChunkRangesForLists(browse_store_.get(), kMalwareList, kPhishingList,
                             lists);
 
   // NOTE(shess): |download_store_| used to contain kBinHashList, which has been
   // deprecated.  Code to delete the list from the store shows ~15k hits/day as
   // of Feb 2014, so it has been removed.  Everything _should_ be resilient to
   // extra data of that sort.
   UpdateChunkRangesForList(download_store_.get(), kBinUrlList, lists);
 
   UpdateChunkRangesForList(csd_whitelist_store_.get(), kCsdWhiteList, lists);
 
   UpdateChunkRangesForList(download_whitelist_store_.get(), kDownloadWhiteList,
                            lists);
 
   UpdateChunkRangesForList(inclusion_whitelist_store_.get(),
                            kInclusionWhitelist, lists);
 
   UpdateChunkRangesForList(extension_blacklist_store_.get(),
                            kExtensionBlacklist, lists);
 
   UpdateChunkRangesForList(ip_blacklist_store_.get(), kIPBlacklist, lists);
 
   UpdateChunkRangesForList(unwanted_software_store_.get(), kUnwantedUrlList,
                            lists);
 
+  UpdateChunkRangesForList(module_whitelist_store_.get(), kModuleWhitelist,
+                           lists);
+
   db_state_manager_.reset_corruption_detected();
   db_state_manager_.reset_change_detected();
   return true;
 }
 
 void SafeBrowsingDatabaseNew::UpdateFinished(bool update_succeeded) {
   DCHECK(db_task_runner_->RunsTasksOnCurrentThread());
 
   // The update may have failed due to corrupt storage (for instance,
   // an excessive number of invalid add_chunks and sub_chunks).
@@ skipping 27 matching lines @@
     }
 
     if (ip_blacklist_store_ && !ip_blacklist_store_->CheckValidity()) {
       DLOG(ERROR) << "Safe-browsing IP blacklist database corrupt.";
     }
 
     if (unwanted_software_store_ &&
         !unwanted_software_store_->CheckValidity()) {
       DLOG(ERROR) << "Unwanted software url list database corrupt.";
     }
+
+    if (module_whitelist_store_ && !module_whitelist_store_->CheckValidity()) {
+      DLOG(ERROR) << "Module digest whitelist database corrupt.";
+    }
   }
 
   if (db_state_manager_.corruption_detected())
     return;
 
   // Unroll the transaction if there was a protocol error or if the
   // transaction was empty.  This will leave the prefix set, the
   // pending hashes, and the prefix miss cache in place.
   if (!update_succeeded || !db_state_manager_.change_detected()) {
     // Track empty updates to answer questions at http://crbug.com/72216 .
     if (update_succeeded && !db_state_manager_.change_detected())
       UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", 0);
     browse_store_->CancelUpdate();
     if (download_store_.get())
       download_store_->CancelUpdate();
     if (csd_whitelist_store_.get())
       csd_whitelist_store_->CancelUpdate();
     if (download_whitelist_store_.get())
       download_whitelist_store_->CancelUpdate();
     if (inclusion_whitelist_store_.get())
       inclusion_whitelist_store_->CancelUpdate();
     if (extension_blacklist_store_)
       extension_blacklist_store_->CancelUpdate();
     if (ip_blacklist_store_)
       ip_blacklist_store_->CancelUpdate();
     if (unwanted_software_store_)
       unwanted_software_store_->CancelUpdate();
+    if (module_whitelist_store_)
+      module_whitelist_store_->CancelUpdate();
     return;
   }
 
   if (download_store_) {
     UpdateHashPrefixStore(DownloadDBFilename(db_state_manager_.filename_base()),
                           download_store_.get(),
                           FAILURE_DOWNLOAD_DATABASE_UPDATE_FINISH);
   }
 
   UpdatePrefixSetUrlStore(BrowseDBFilename(db_state_manager_.filename_base()),
@@ skipping 21 matching lines @@
   if (ip_blacklist_store_)
     UpdateIpBlacklistStore();
 
   if (unwanted_software_store_) {
     UpdatePrefixSetUrlStore(
         UnwantedSoftwareDBFilename(db_state_manager_.filename_base()),
         unwanted_software_store_.get(), PrefixSetId::UNWANTED_SOFTWARE,
         FAILURE_UNWANTED_SOFTWARE_DATABASE_UPDATE_FINISH,
         FAILURE_UNWANTED_SOFTWARE_PREFIX_SET_WRITE, true);
   }
+
+  if (module_whitelist_store_) {
+    UpdateWhitelistStore(
+        ModuleWhitelistDBFilename(db_state_manager_.filename_base()),
+        module_whitelist_store_.get(), SBWhitelistId::MODULE);
+  }
 }
 
 void SafeBrowsingDatabaseNew::UpdateWhitelistStore(
     const base::FilePath& store_filename,
     SafeBrowsingStore* store,
     SBWhitelistId whitelist_id) {
   DCHECK(db_task_runner_->RunsTasksOnCurrentThread());
 
   if (!store)
     return;
@@ skipping 302 matching lines @@
 #if defined(OS_MACOSX)
   base::mac::SetFileBackupExclusion(prefix_set_filename);
 #endif
 }
 
 void SafeBrowsingDatabaseNew::LoadWhitelist(
     const std::vector<SBAddFullHash>& full_hashes,
     SBWhitelistId whitelist_id) {
   DCHECK(db_task_runner_->RunsTasksOnCurrentThread());
 
-  if (full_hashes.size() > kMaxWhitelistSize) {
+  if (full_hashes.size() > kMaxWhitelistSize &&
+      whitelist_id != SBWhitelistId::MODULE) {
     state_manager_.BeginWriteTransaction()->WhitelistEverything(whitelist_id);
     return;
   }
 
+  if (full_hashes.size() == 0 && whitelist_id == SBWhitelistId::MODULE) {
+    state_manager_.BeginWriteTransaction()->WhitelistEverything(
+        SBWhitelistId::MODULE);
+    return;
+  }
+
   std::vector<SBFullHash> new_whitelist;
   new_whitelist.reserve(full_hashes.size());
   for (std::vector<SBAddFullHash>::const_iterator it = full_hashes.begin();
        it != full_hashes.end(); ++it) {
     new_whitelist.push_back(it->full_hash);
   }
   std::sort(new_whitelist.begin(), new_whitelist.end(), SBFullHashLess);
 
   SBFullHash kill_switch = SBFullHashForString(kWhitelistKillSwitchUrl);
   if (std::binary_search(new_whitelist.begin(), new_whitelist.end(),
@@ skipping 102 matching lines @@
     histogram_name.append(".InclusionWhitelist");
   else if (base::EndsWith(filename, kExtensionBlacklistDBFile,
                           base::CompareCase::SENSITIVE))
     histogram_name.append(".ExtensionBlacklist");
   else if (base::EndsWith(filename, kIPBlacklistDBFile,
                           base::CompareCase::SENSITIVE))
     histogram_name.append(".IPBlacklist");
   else if (base::EndsWith(filename, kUnwantedSoftwareDBFile,
                           base::CompareCase::SENSITIVE))
     histogram_name.append(".UnwantedSoftware");
+  else if (base::EndsWith(filename, kModuleWhitelistDBFile,
+                          base::CompareCase::SENSITIVE))
+    histogram_name.append(".ModuleWhitelist");
   else
     NOTREACHED();  // Add support for new lists above.
 
   // Histogram properties as in UMA_HISTOGRAM_COUNTS macro.
   base::HistogramBase* histogram_pointer = base::Histogram::FactoryGet(
       histogram_name, 1, 1000000, 50,
       base::HistogramBase::kUmaTargetedHistogramFlag);
 
   histogram_pointer->Add(file_size_kilobytes);
 }
 
 }  // namespace safe_browsing