Add the goog-unwanted-shavar list to a new SafeBrowsing PrefixSet.

chrome/browser/safe_browsing/safe_browsing_database.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef CHROME_BROWSER_SAFE_BROWSING_SAFE_BROWSING_DATABASE_H_
 #define CHROME_BROWSER_SAFE_BROWSING_SAFE_BROWSING_DATABASE_H_
 
 #include <map>
 #include <set>
 #include <string>
@@ skipping 24 matching lines @@
 class SafeBrowsingDatabaseFactory {
  public:
   SafeBrowsingDatabaseFactory() { }
   virtual ~SafeBrowsingDatabaseFactory() { }
   virtual SafeBrowsingDatabase* CreateSafeBrowsingDatabase(
       bool enable_download_protection,
       bool enable_client_side_whitelist,
       bool enable_download_whitelist,
       bool enable_extension_blacklist,
       bool enable_side_effect_free_whitelist,
-      bool enable_ip_blacklist) = 0;
+      bool enable_ip_blacklist,
+      bool enable_unwanted_software_list) = 0;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseFactory);
 };
 
 // Encapsulates on-disk databases that for safebrowsing. There are
 // four databases: browse, download, download whitelist and
 // client-side detection (csd) whitelist databases. The browse database contains
 // information about phishing and malware urls. The download database contains
 // URLs for bad binaries (e.g: those containing virus) and hash of
 // these downloaded contents. The download whitelist contains whitelisted
 // download hosting sites as well as whitelisted binary signing certificates
 // etc.  The csd whitelist database contains URLs that will never be considered
 // as phishing by the client-side phishing detection. These on-disk databases
 // are shared among all profiles, as it doesn't contain user-specific data. This
 // object is not thread-safe, i.e. all its methods should be used on the same
 // thread that it was created on.
 class SafeBrowsingDatabase {
  public:
   // Factory method for obtaining a SafeBrowsingDatabase implementation.
   // It is not thread safe.
   // |enable_download_protection| is used to control the download database
   // feature.
   // |enable_client_side_whitelist| is used to control the csd whitelist
   // database feature.
   // |enable_download_whitelist| is used to control the download whitelist
   // database feature.
   // |enable_ip_blacklist| is used to control the csd malware IP blacklist
   // database feature.
+  // |enable_unwanted_software_list| is used to control the unwanted software
+  // list database feature.
   static SafeBrowsingDatabase* Create(bool enable_download_protection,
                                       bool enable_client_side_whitelist,
                                       bool enable_download_whitelist,
                                       bool enable_extension_blacklist,
                                       bool side_effect_free_whitelist,
-                                      bool enable_ip_blacklist);
+                                      bool enable_ip_blacklist,
+                                      bool enable_unwanted_software_list);
 
   // Makes the passed |factory| the factory used to instantiate
   // a SafeBrowsingDatabase. This is used for tests.
   static void RegisterFactory(SafeBrowsingDatabaseFactory* factory) {
     factory_ = factory;
   }
 
   virtual ~SafeBrowsingDatabase();
 
   // Initializes the database with the given filename.
   virtual void Init(const base::FilePath& filename) = 0;
 
   // Deletes the current database and creates a new one.
   virtual bool ResetDatabase() = 0;
 
   // Returns false if |url| is not in the browse database or already was cached
-  // as a miss.  If it returns true, |prefix_hits| contains matching hash
-  // prefixes which had no cached results and |cache_hits| contains any matching
-  // cached gethash results.  This function is safe to call from any thread.
+  // as a miss.  If it returns true, |prefix_hits| contains sorted unique
+  // matching hash prefixes which had no cached results and |cache_hits|
+  // contains any matching cached gethash results.  This function is safe to
+  // call from any thread.
   virtual bool ContainsBrowseUrl(
       const GURL& url,
       std::vector<SBPrefix>* prefix_hits,
       std::vector<SBFullHashResult>* cache_hits) = 0;
 
+  // Returns true iff the given url is on the unwanted software blacklist.
+  // Returns false if |url| is not in the browse database or already was cached
+  // as a miss.  If it returns true, |prefix_hits| contains sorted unique
+  // matching hash prefixes which had no cached results and |cache_hits|
+  // contains any matching cached gethash results.  This function is safe to
+  // call from any thread.
+  virtual bool ContainsUnwantedSoftwareUrl(
+      const GURL& url,
+      std::vector<SBPrefix>* prefix_hits,
+      std::vector<SBFullHashResult>* cache_hits) = 0;
+
   // Returns false if none of |urls| are in Download database. If it returns
   // true, |prefix_hits| should contain the prefixes for the URLs that were in
   // the database.  This function could ONLY be accessed from creation thread.
   virtual bool ContainsDownloadUrl(const std::vector<GURL>& urls,
                                    std::vector<SBPrefix>* prefix_hits) = 0;
 
   // Returns false if |url| is not on the client-side phishing detection
   // whitelist.  Otherwise, this function returns true.  Note: the whitelist
   // only contains full-length hashes so we don't return any prefix hit.
   // This function should only be called from the IO thread.
@@ skipping 97 matching lines @@
       const base::FilePath& extension_blacklist_base_filename);
 
   // Filename for side-effect free whitelist database.
   static base::FilePath SideEffectFreeWhitelistDBFilename(
       const base::FilePath& side_effect_free_whitelist_base_filename);
 
   // Filename for the csd malware IP blacklist database.
   static base::FilePath IpBlacklistDBFilename(
       const base::FilePath& ip_blacklist_base_filename);
 
+  // Filename for the unwanted software blacklist database.
+  static base::FilePath UnwantedSoftwareDBFilename(
+      const base::FilePath& db_filename);
+
   // Enumerate failures for histogramming purposes.  DO NOT CHANGE THE
   // ORDERING OF THESE VALUES.
   enum FailureType {
     FAILURE_DATABASE_CORRUPT,
     FAILURE_DATABASE_CORRUPT_HANDLER,
     FAILURE_BROWSE_DATABASE_UPDATE_BEGIN,
     FAILURE_BROWSE_DATABASE_UPDATE_FINISH,
     FAILURE_DATABASE_FILTER_MISSING_OBSOLETE,
     FAILURE_DATABASE_FILTER_READ_OBSOLETE,
     FAILURE_DATABASE_FILTER_WRITE_OBSOLETE,
     FAILURE_DATABASE_FILTER_DELETE,
     FAILURE_DATABASE_STORE_MISSING,
     FAILURE_DATABASE_STORE_DELETE,
     FAILURE_DOWNLOAD_DATABASE_UPDATE_BEGIN,
     FAILURE_DOWNLOAD_DATABASE_UPDATE_FINISH,
     FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN,
     FAILURE_WHITELIST_DATABASE_UPDATE_FINISH,
-    FAILURE_BROWSE_PREFIX_SET_MISSING,
     FAILURE_BROWSE_PREFIX_SET_READ,
     FAILURE_BROWSE_PREFIX_SET_WRITE,
     FAILURE_BROWSE_PREFIX_SET_DELETE,
     FAILURE_EXTENSION_BLACKLIST_UPDATE_BEGIN,
     FAILURE_EXTENSION_BLACKLIST_UPDATE_FINISH,
     FAILURE_EXTENSION_BLACKLIST_DELETE,
     FAILURE_SIDE_EFFECT_FREE_WHITELIST_UPDATE_BEGIN,
     FAILURE_SIDE_EFFECT_FREE_WHITELIST_UPDATE_FINISH,
     FAILURE_SIDE_EFFECT_FREE_WHITELIST_DELETE,
     FAILURE_SIDE_EFFECT_FREE_WHITELIST_PREFIX_SET_READ,
     FAILURE_SIDE_EFFECT_FREE_WHITELIST_PREFIX_SET_WRITE,
     FAILURE_SIDE_EFFECT_FREE_WHITELIST_PREFIX_SET_DELETE,
     FAILURE_IP_BLACKLIST_UPDATE_BEGIN,
     FAILURE_IP_BLACKLIST_UPDATE_FINISH,
     FAILURE_IP_BLACKLIST_UPDATE_INVALID,
     FAILURE_IP_BLACKLIST_DELETE,
+    FAILURE_UNWANTED_SOFTWARE_DATABASE_UPDATE_BEGIN,

[Alexei Svitkine (slow), 2015/02/27 21:37:54]

I think you also need to update histograms.xml with these.

[gab, 2015/03/02 17:13:36]

Indeed, thanks: https://codereview.chromium.org/968253002/

+    FAILURE_UNWANTED_SOFTWARE_DATABASE_UPDATE_FINISH,
+    FAILURE_UNWANTED_SOFTWARE_PREFIX_SET_READ,
+    FAILURE_UNWANTED_SOFTWARE_PREFIX_SET_WRITE,
+    FAILURE_UNWANTED_SOFTWARE_PREFIX_SET_DELETE,
 
     // Memory space for histograms is determined by the max.  ALWAYS
     // ADD NEW VALUES BEFORE THIS ONE.
     FAILURE_DATABASE_MAX
   };
 
   static void RecordFailure(FailureType failure_type);
 
  private:
   // The factory used to instantiate a SafeBrowsingDatabase object.
   // Useful for tests, so they can provide their own implementation of
   // SafeBrowsingDatabase.
   static SafeBrowsingDatabaseFactory* factory_;
 };
 
 class SafeBrowsingDatabaseNew : public SafeBrowsingDatabase {
  public:
   // Create a database with a browse, download, download whitelist and
   // csd whitelist store objects. Takes ownership of all the store objects.
   // When |download_store| is NULL, the database will ignore any operations
   // related download (url hashes and binary hashes).  The same is true for
   // the |csd_whitelist_store|, |download_whitelist_store| and
   // |ip_blacklist_store|.
   SafeBrowsingDatabaseNew(SafeBrowsingStore* browse_store,
                           SafeBrowsingStore* download_store,
                           SafeBrowsingStore* csd_whitelist_store,
                           SafeBrowsingStore* download_whitelist_store,
                           SafeBrowsingStore* extension_blacklist_store,
                           SafeBrowsingStore* side_effect_free_whitelist_store,
-                          SafeBrowsingStore* ip_blacklist_store);
+                          SafeBrowsingStore* ip_blacklist_store,
+                          SafeBrowsingStore* unwanted_software_store);
 
   // Create a database with a browse store. This is a legacy interface that
   // useds Sqlite.
   SafeBrowsingDatabaseNew();
 
   ~SafeBrowsingDatabaseNew() override;
 
   // Implement SafeBrowsingDatabase interface.
   void Init(const base::FilePath& filename) override;
   bool ResetDatabase() override;
   bool ContainsBrowseUrl(const GURL& url,
                          std::vector<SBPrefix>* prefix_hits,
                          std::vector<SBFullHashResult>* cache_hits) override;
+  bool ContainsUnwantedSoftwareUrl(
+      const GURL& url,
+      std::vector<SBPrefix>* prefix_hits,
+      std::vector<SBFullHashResult>* cache_hits) override;
   bool ContainsDownloadUrl(const std::vector<GURL>& urls,
                            std::vector<SBPrefix>* prefix_hits) override;
   bool ContainsCsdWhitelistedUrl(const GURL& url) override;
   bool ContainsDownloadWhitelistedUrl(const GURL& url) override;
   bool ContainsDownloadWhitelistedString(const std::string& str) override;
   bool ContainsExtensionPrefixes(const std::vector<SBPrefix>& prefixes,
                                  std::vector<SBPrefix>* prefix_hits) override;
   bool ContainsSideEffectFreeWhitelistUrl(const GURL& url) override;
   bool ContainsMalwareIP(const std::string& ip_address) override;
   bool UpdateStarted(std::vector<SBListChunkRanges>* lists) override;
@@ skipping 23 matching lines @@
   // A SafeBrowsing whitelist contains a list of whitelisted full-hashes (stored
   // in a sorted vector) as well as a boolean flag indicating whether all
   // lookups in the whitelist should be considered matches for safety.
   typedef std::pair<std::vector<SBFullHash>, bool> SBWhitelist;
 
   // This map holds a csd malware IP blacklist which maps a prefix mask
   // to a set of hashed blacklisted IP prefixes.  Each IP prefix is a hashed
   // IPv6 IP prefix using SHA-1.
   typedef std::map<std::string, base::hash_set<std::string> > IPBlacklist;
 
-  // Helper for ContainsBrowseUrl, exposed for testing.
-  bool ContainsBrowseUrlHashes(const std::vector<SBFullHash>& full_hashes,
-                               std::vector<SBPrefix>* prefix_hits,
-                               std::vector<SBFullHashResult>* cache_hits);
+  bool PrefixSetContainsUrl(
+      const GURL& url,
+      scoped_ptr<safe_browsing::PrefixSet>* prefix_set_getter,
+      std::vector<SBPrefix>* prefix_hits,
+      std::vector<SBFullHashResult>* cache_hits);
+
+  // Exposed for testing of PrefixSetContainsUrlHashes() on the
+  // PrefixSet backing kMalwareList.
+  bool ContainsBrowseUrlHashesForTesting(
+      const std::vector<SBFullHash>& full_hashes,
+      std::vector<SBPrefix>* prefix_hits,
+      std::vector<SBFullHashResult>* cache_hits);
+
+  bool PrefixSetContainsUrlHashes(
+      const std::vector<SBFullHash>& full_hashes,
+      scoped_ptr<safe_browsing::PrefixSet>* prefix_set_getter,
+      std::vector<SBPrefix>* prefix_hits,
+      std::vector<SBFullHashResult>* cache_hits);
 
   // Returns true if the whitelist is disabled or if any of the given hashes
   // matches the whitelist.
   bool ContainsWhitelistedHashes(const SBWhitelist& whitelist,
                                  const std::vector<SBFullHash>& hashes);
 
   // Return the browse_store_, download_store_, download_whitelist_store or
   // csd_whitelist_store_ based on list_id.
   SafeBrowsingStore* GetStore(int list_id);
 
   // Deletes the files on disk.
   bool Delete();
 
-  // Load the prefix set off disk, if available.
-  void LoadPrefixSet();
+  // Load the prefix set in "|db_filename| Prefix Set" off disk, if available,
+  // and stores it in |prefix_set|.  |read_failure_type| provides a
+  // caller-specific error code to be used on failure.
+  void LoadPrefixSet(const base::FilePath& db_filename,
+                     scoped_ptr<safe_browsing::PrefixSet>* prefix_set,
+                     FailureType read_failure_type);
 
-  // Writes the current prefix set to disk.
-  void WritePrefixSet();
+  // Writes the current prefix set "|db_filename| Prefix Set" on disk.
+  // |write_failure_type| provides a caller-specific error code to be used on
+  // failure.
+  void WritePrefixSet(const base::FilePath& db_filename,
+                      safe_browsing::PrefixSet* prefix_set,
+                      FailureType write_failure_type);
 
   // Loads the given full-length hashes to the given whitelist.  If the number
   // of hashes is too large or if the kill switch URL is on the whitelist
   // we will whitelist everything.
   void LoadWhitelist(const std::vector<SBAddFullHash>& full_hashes,
                      SBWhitelist* whitelist);
 
   // Call this method if an error occured with the given whitelist.  This will
   // result in all lookups to the whitelist to return true.
   void WhitelistEverything(SBWhitelist* whitelist);
@@ skipping 16 matching lines @@
                       safe_browsing_util::ListType list_id,
                       const SBChunkData& chunk);
   void InsertSubChunk(SafeBrowsingStore* store,
                       safe_browsing_util::ListType list_id,
                       const SBChunkData& chunk);
 
   // Returns the size in bytes of the store after the update.
   int64 UpdateHashPrefixStore(const base::FilePath& store_filename,
                                SafeBrowsingStore* store,
                                FailureType failure_type);
-  void UpdateBrowseStore();
+
+  // Updates a PrefixStore store for URLs (|url_store|) which is backed on disk
+  // by a "|db_filename| Prefix Set" file. Specific failure types are provided
+  // to highlight the specific store who made the initial request on failure.
+  void UpdatePrefixSetUrlStore(const base::FilePath& db_filename,
+                               SafeBrowsingStore* url_store,
+                               scoped_ptr<safe_browsing::PrefixSet>* prefix_set,
+                               FailureType finish_failure_type,
+                               FailureType write_failure_type);
+
+  void UpdateUrlStore(SafeBrowsingStore* url_store,
+                      scoped_ptr<safe_browsing::PrefixSet>* prefix_set,
+                      FailureType failure_type);
+
   void UpdateSideEffectFreeWhitelistStore();
   void UpdateWhitelistStore(const base::FilePath& store_filename,
                             SafeBrowsingStore* store,
                             SBWhitelist* whitelist);
   void UpdateIpBlacklistStore();
 
   // Used to verify that various calls are made from the thread the
   // object was created on.
   base::MessageLoop* creation_loop_;
 
-  // Lock for protecting access to variables that may be used on the IO thread.
-  // This includes |prefix_set_|, |browse_gethash_cache_|, |csd_whitelist_|.
-  base::Lock lookup_lock_;
-
   // The base filename passed to Init(), used to generate the store and prefix
   // set filenames used to store data on disk.
   base::FilePath filename_base_;
 
   // Underlying persistent store for chunk data.
   // For browsing related (phishing and malware URLs) chunks and prefixes.
   scoped_ptr<SafeBrowsingStore> browse_store_;
 
   // For download related (download URL and binary hash) chunks and prefixes.
   scoped_ptr<SafeBrowsingStore> download_store_;
 
   // For the client-side phishing detection whitelist chunks and full-length
   // hashes.  This list only contains 256 bit hashes.
   scoped_ptr<SafeBrowsingStore> csd_whitelist_store_;
 
   // For the download whitelist chunks and full-length hashes.  This list only
   // contains 256 bit hashes.
   scoped_ptr<SafeBrowsingStore> download_whitelist_store_;
 
   // For extension IDs.
   scoped_ptr<SafeBrowsingStore> extension_blacklist_store_;
 
   // For side-effect free whitelist.
   scoped_ptr<SafeBrowsingStore> side_effect_free_whitelist_store_;
 
   // For IP blacklist.
   scoped_ptr<SafeBrowsingStore> ip_blacklist_store_;
 
+  // For unwanted software list.
+  scoped_ptr<SafeBrowsingStore> unwanted_software_store_;
+
+  // Lock for protecting access to variables that may be used on the IO thread.
+  // This includes |(browse|unwanted_software)_prefix_set_|,
+  // |prefix_gethash_cache_|, |csd_whitelist_|.
+  base::Lock lookup_lock_;
+
   SBWhitelist csd_whitelist_;
   SBWhitelist download_whitelist_;
   SBWhitelist extension_blacklist_;
 
   // The IP blacklist should be small.  At most a couple hundred IPs.
   IPBlacklist ip_blacklist_;
 
-  // Cache of gethash results for browse store. Entries should not be used if
+  // Cache of gethash results for prefix stores. Entries should not be used if
   // they are older than their expire_after field.  Cached misses will have
   // empty full_hashes field.  Cleared on each update.
-  std::map<SBPrefix, SBCachedFullHashResult> browse_gethash_cache_;
+  std::map<SBPrefix, SBCachedFullHashResult> prefix_gethash_cache_;
 
   // Set if corruption is detected during the course of an update.
   // Causes the update functions to fail with no side effects, until
   // the next call to |UpdateStarted()|.
   bool corruption_detected_;
 
   // Set to true if any chunks are added or deleted during an update.
   // Used to optimize away database update.
   bool change_detected_;
 
   // Used to check if a prefix was in the browse database.
   scoped_ptr<safe_browsing::PrefixSet> browse_prefix_set_;
 
-  // Used to check if a prefix was in the browse database.
+  // Used to check if a prefix was in the side-effect free whitelist database.
   scoped_ptr<safe_browsing::PrefixSet> side_effect_free_whitelist_prefix_set_;
 
+  // Used to check if a prexfix was in the unwanted software database.
+  scoped_ptr<safe_browsing::PrefixSet> unwanted_software_prefix_set_;
+
   // Used to schedule resetting the database because of corruption.
   base::WeakPtrFactory<SafeBrowsingDatabaseNew> reset_factory_;
 };
 
 #endif  // CHROME_BROWSER_SAFE_BROWSING_SAFE_BROWSING_DATABASE_H_