Extracting page shingle hashes for similarity detection.

chrome/renderer/safe_browsing/phishing_classifier.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/safe_browsing/phishing_classifier.h"
 
 #include <string>
 
 #include "base/bind.h"
 #include "base/callback.h"
@@ skipping 45 matching lines @@
   scorer_ = scorer;
   if (scorer_) {
     url_extractor_.reset(new PhishingUrlFeatureExtractor);
     dom_extractor_.reset(
         new PhishingDOMFeatureExtractor(render_view_, clock_.get()));
     term_extractor_.reset(new PhishingTermFeatureExtractor(
         &scorer_->page_terms(),
         &scorer_->page_words(),
         scorer_->max_words_per_term(),
         scorer_->murmurhash3_seed(),
+        scorer_->max_shingles_per_page(),
+        scorer_->shingle_size(),
         clock_.get()));
   } else {
     // We're disabling client-side phishing detection, so tear down all
     // of the relevant objects.
     url_extractor_.reset();
     dom_extractor_.reset();
     term_extractor_.reset();
   }
 }
 
@@ skipping 71 matching lines @@
   // Note that cancelling the feature extractors is simply a no-op if they
   // were not running.
   DCHECK(is_ready());
   dom_extractor_->CancelPendingExtraction();
   term_extractor_->CancelPendingExtraction();
   weak_factory_.InvalidateWeakPtrs();
   Clear();
 }
 
 void PhishingClassifier::DOMExtractionFinished(bool success) {
+  shingle_hashes_.reset(new std::set<uint32>);
   if (success) {
     // Term feature extraction can take awhile, so it runs asynchronously
     // in several chunks of work and invokes the callback when finished.
     term_extractor_->ExtractFeatures(
         page_text_,
         features_.get(),
+        shingle_hashes_.get(),
         base::Bind(&PhishingClassifier::TermExtractionFinished,
                    base::Unretained(this)));
   } else {
     RunFailureCallback();
   }
 }
 
 void PhishingClassifier::TermExtractionFinished(bool success) {
   if (success) {
     blink::WebView* web_view = render_view_->GetWebView();
@@ skipping 17 matching lines @@
              features_->features().begin();
          it != features_->features().end(); ++it) {
       VLOG(2) << "Feature: " << it->first << " = " << it->second;
       bool result = hashed_features.AddRealFeature(
           crypto::SHA256HashString(it->first), it->second);
       DCHECK(result);
       ClientPhishingRequest::Feature* feature = verdict.add_feature_map();
       feature->set_name(it->first);
       feature->set_value(it->second);
     }
+    for (std::set<uint32>::const_iterator it = shingle_hashes_->begin();
+         it != shingle_hashes_->end(); ++it) {
+      verdict.add_shingle_hashes(*it);
+    }
     float score = static_cast<float>(scorer_->ComputeScore(hashed_features));
     verdict.set_client_score(score);
     verdict.set_is_phishing(score >= kPhishyThreshold);
     RunCallback(verdict);
   } else {
     RunFailureCallback();
   }
 }
 
 void PhishingClassifier::CheckNoPendingClassification() {
@@ skipping 19 matching lines @@
   verdict.set_url("");
   verdict.set_client_score(kInvalidScore);
   verdict.set_is_phishing(false);
   RunCallback(verdict);
 }
 
 void PhishingClassifier::Clear() {
   page_text_ = NULL;
   done_callback_.Reset();
   features_.reset(NULL);
+  shingle_hashes_.reset(NULL);
 }
 
 }  // namespace safe_browsing