Switch to the new client-side phishing model that uses Murmurhash for word hashes.

chrome/renderer/safe_browsing/phishing_term_feature_extractor.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/safe_browsing/phishing_term_feature_extractor.h"
 
 #include <list>
 #include <map>
 
 #include "base/compiler_specific.h"
 #include "base/i18n/case_conversion.h"
 #include "base/logging.h"
 #include "base/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/time.h"
 #include "base/utf_string_conversions.h"
 #include "crypto/sha2.h"
 #include "chrome/renderer/safe_browsing/feature_extractor_clock.h"
 #include "chrome/renderer/safe_browsing/features.h"
+#include "chrome/renderer/safe_browsing/murmurhash3_util.h"
 #include "ui/base/l10n/l10n_util.h"
 #include "unicode/ubrk.h"
 
 namespace safe_browsing {
 
 // This time should be short enough that it doesn't noticeably disrupt the
 // user's interaction with the page.
 const int PhishingTermFeatureExtractor::kMaxTimePerChunkMs = 20;
 
 // Experimenting shows that we get a reasonable gain in performance by
 // increasing this up to around 10, but there's not much benefit in
 // increasing it past that.
 const int PhishingTermFeatureExtractor::kClockCheckGranularity = 5;
 
 // This should be longer than we expect feature extraction to take on any
 // actual phishing page.
 const int PhishingTermFeatureExtractor::kMaxTotalTimeMs = 500;
 
 // The maximum size of the negative word cache.
 const int PhishingTermFeatureExtractor::kMaxNegativeWordCacheSize = 1000;
 
 // All of the state pertaining to the current feature extraction.
 struct PhishingTermFeatureExtractor::ExtractionState {
-  // Stores up to max_words_per_ngram_ previous words separated by spaces.
+  // Stores up to max_words_per_term_ previous words separated by spaces.
   std::string previous_words;
 
   // Stores the sizes of the words in previous_words.  Note: the size includes
   // the space after each word.  In other words, the sum of all sizes in this
   // list is equal to the length of previous_words.
   std::list<size_t> previous_word_sizes;
 
   // An iterator for word breaking.
   UBreakIterator* iterator;
 
@@ skipping 29 matching lines @@
 
   ~ExtractionState() {
     if (iterator) {
       ubrk_close(iterator);
     }
   }
 };
 
 PhishingTermFeatureExtractor::PhishingTermFeatureExtractor(
     const base::hash_set<std::string>* page_term_hashes,
-    const base::hash_set<std::string>* page_word_hashes,
+    const base::hash_set<uint32>* page_word_hashes,
     size_t max_words_per_term,
+    uint32 murmurhash3_seed,
     FeatureExtractorClock* clock)
     : page_term_hashes_(page_term_hashes),
       page_word_hashes_(page_word_hashes),
       max_words_per_term_(max_words_per_term),
+      murmurhash3_seed_(murmurhash3_seed),
       negative_word_cache_(kMaxNegativeWordCacheSize),
       clock_(clock),
       ALLOW_THIS_IN_INITIALIZER_LIST(method_factory_(this)) {
   Clear();
 }
 
 PhishingTermFeatureExtractor::~PhishingTermFeatureExtractor() {
   // The RenderView should have called CancelPendingExtraction() before
   // we are destroyed.
   CheckNoPendingExtraction();
@@ skipping 90 matching lines @@
       }
       // Otherwise, continue.
     }
   }
   RunCallback(true);
 }
 
 void PhishingTermFeatureExtractor::HandleWord(
     const base::StringPiece16& word) {
   // Quickest out if we have seen this word before and know that it's not
-  // part of any term. This avoids the SHA256, lowercasing, and UTF conversion,
-  // all of which are relatively expensive.
+  // part of any term. This avoids the lowercasing and UTF conversion, both of
+  // which are relatively expensive.
   if (negative_word_cache_.Get(word) != negative_word_cache_.end()) {
     // We know we're no longer in a possible n-gram, so clear the previous word
     // state.
     state_->previous_words.clear();
     state_->previous_word_sizes.clear();
     return;
   }
 
   std::string word_lower = UTF16ToUTF8(base::i18n::ToLower(word));
-  std::string word_hash = crypto::SHA256HashString(word_lower);
+  uint32 word_hash = MurmurHash3String(word_lower, murmurhash3_seed_);
 
   // Quick out if the word is not part of any term, which is the common case.
   if (page_word_hashes_->find(word_hash) == page_word_hashes_->end()) {
     // Word doesn't exist in our terms so we can clear the n-gram state.
     state_->previous_words.clear();
     state_->previous_word_sizes.clear();
     // Insert into negative cache so that we don't try this again.
     negative_word_cache_.Put(word, true);
     return;
   }
 
-  // Find all of the n-grams that we need to check and compute their hashes.
-  // We already have the hash for word_lower, so we don't compute that again.
+  // Find all of the n-grams that we need to check and compute their SHA-256
+  // hashes.
   std::map<std::string /* hash */, std::string /* plaintext */>
       hashes_to_check;
-  hashes_to_check[word_hash] = word_lower;
+  hashes_to_check[crypto::SHA256HashString(word_lower)] = word_lower;
 
   // Combine the new word with the previous words to find additional n-grams.
   // Note that we don't yet add the new word length to previous_word_sizes,
   // since we don't want to compute the hash for the word by itself again.
   //
   state_->previous_words.append(word_lower);
   std::string current_term = state_->previous_words;
   for (std::list<size_t>::iterator it = state_->previous_word_sizes.begin();
        it != state_->previous_word_sizes.end(); ++it) {
     hashes_to_check[crypto::SHA256HashString(current_term)] = current_term;
@@ skipping 49 matching lines @@
 
 void PhishingTermFeatureExtractor::Clear() {
   page_text_ = NULL;
   features_ = NULL;
   done_callback_.reset(NULL);
   state_.reset(NULL);
   negative_word_cache_.Clear();
 }
 
 }  // namespace safe_browsing