Add support for client-side phishing detection for non-UMA users.

chrome/browser/safe_browsing/client_side_detection_service_unittest.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <map>
 #include <queue>
 #include <string>
 
 #include "base/callback.h"
 #include "base/logging.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/message_loop.h"
+#include "base/stringprintf.h"
 #include "base/task.h"
 #include "base/time.h"
+#include "chrome/browser/safe_browsing/browser_features.h"
 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
 #include "chrome/common/safe_browsing/client_model.pb.h"
 #include "chrome/common/safe_browsing/csd.pb.h"
+#include "chrome/renderer/safe_browsing/features.h"
 #include "chrome/test/base/testing_browser_process_test.h"
 #include "content/browser/browser_thread.h"
 #include "content/common/url_fetcher.h"
 #include "content/test/test_url_fetcher_factory.h"
 #include "crypto/sha2.h"
 #include "googleurl/src/gurl.h"
 #include "net/url_request/url_request_status.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
@@ skipping 113 matching lines @@
     EXPECT_TRUE(csd_service_->GetValidCachedResult(
         GURL("http://first.url.com"), &is_phishing));
     EXPECT_FALSE(is_phishing);
     EXPECT_FALSE(csd_service_->GetValidCachedResult(
         GURL("http://third.url.com"), &is_phishing));
     EXPECT_TRUE(csd_service_->GetValidCachedResult(
         GURL("http://fourth.url.com"), &is_phishing));
     EXPECT_TRUE(is_phishing);
   }
 
+  void AddFeature(const std::string& name, double value,
+                  ClientPhishingRequest* request) {
+    ClientPhishingRequest_Feature* feature = request->add_feature_map();
+    feature->set_name(name);
+    feature->set_value(value);
+  }
+
+  void AddNonModelFeature(const std::string& name, double value,
+                          ClientPhishingRequest* request) {
+    ClientPhishingRequest_Feature* feature =
+        request->add_non_model_feature_map();
+    feature->set_name(name);
+    feature->set_value(value);
+  }
+
  protected:
   scoped_ptr<ClientSideDetectionService> csd_service_;
   scoped_ptr<FakeURLFetcherFactory> factory_;
   MessageLoop msg_loop_;
 
  private:
   void SendRequestDone(GURL phishing_url, bool is_phishing) {
     ASSERT_EQ(phishing_url, phishing_url_);
     is_phishing_ = is_phishing;
     msg_loop_.Quit();
@@ skipping 391 matching lines @@
   rule->add_feature(-1);
   rule->set_weight(0.2f);
   EXPECT_FALSE(ClientSideDetectionService::ModelHasValidHashIds(model));
 
   rule->set_feature(2, 2);
   EXPECT_FALSE(ClientSideDetectionService::ModelHasValidHashIds(model));
 
   rule->set_feature(2, 1);
   EXPECT_TRUE(ClientSideDetectionService::ModelHasValidHashIds(model));
 }
+
+TEST_F(ClientSideDetectionServiceTest, SanitizeRequestForPingback) {
+  ClientPhishingRequest request;
+  request.set_url("http://www.us.host.com/blah");
+  request.set_suffix_prefix_hash("hash");
+  request.set_client_score(0.8f);
+  request.set_is_phishing(true);
+  AddFeature(std::string(features::kUrlTldToken) + "com", 1.0, &request);
+  AddFeature(std::string(features::kUrlDomainToken) + "host", 1.0, &request);
+  AddFeature(std::string(features::kUrlOtherHostToken) + "us", 1.0, &request);
+  AddFeature(std::string(features::kUrlOtherHostToken) + "www", 1.0, &request);
+  AddFeature(features::kUrlNumOtherHostTokensGTOne, 1.0, &request);
+  AddFeature(std::string(features::kUrlPathToken) + "blah", 1.0, &request);
+  AddFeature(features::kPageHasForms, 1.0, &request);
+  AddFeature(std::string(features::kPageTerm) + "term", 1.0, &request);
+  AddFeature(features::kPageImgOtherDomainFreq, 0.5, &request);
+  request.set_model_version(3);
+  AddNonModelFeature(features::kUrlHistoryVisitCount, 5.0, &request);
+  AddNonModelFeature(StringPrintf("%s=http://referrer.com/",
+                                  features::kReferrer),
+                     1.0, &request);
+  AddNonModelFeature(StringPrintf("%s%s=http://redirreferrer.com/",
+                                  features::kRedirectPrefix,
+                                  features::kReferrer),
+                     1.0, &request);
+  AddNonModelFeature(StringPrintf("%s%s=http://hostreferrer.com/",
+                                  features::kHostPrefix, features::kReferrer),
+                     1.0, &request);
+  AddNonModelFeature(StringPrintf("%s%s%s=http://hostredirreferrer.com/",
+                                  features::kHostPrefix,
+                                  features::kRedirectPrefix,
+                                  features::kReferrer),
+                     1.0, &request);
+  AddNonModelFeature(std::string(features::kBadIpFetch) + "1.2.3.4",
+                     1.0, &request);
+  AddNonModelFeature(std::string(features::kSafeBrowsingMaliciousUrl) +
+                     "http://malicious.com/", 1.0, &request);
+  AddNonModelFeature(std::string(features::kSafeBrowsingOriginalUrl) +
+                     "http://original.com/", 1.0, &request);
+
+  ClientPhishingRequest sanitized_request;
+  ClientSideDetectionService::SanitizeRequestForPingback(request,
+                                                         &sanitized_request);
+
+  // For easier debugging, we'll check the output protobuf fields individually.
+  ClientPhishingRequest expected;
+  expected.set_suffix_prefix_hash(request.suffix_prefix_hash());
+  expected.set_client_score(request.client_score());
+  expected.set_is_phishing(request.is_phishing());
+  AddFeature(features::kUrlNumOtherHostTokensGTOne, 1.0, &expected);
+  AddFeature(features::kPageHasForms, 1.0, &expected);
+  AddFeature(features::kPageImgOtherDomainFreq, 0.5, &expected);
+  expected.set_model_version(3);
+  AddNonModelFeature(features::kUrlHistoryVisitCount, 5.0, &expected);
+
+  EXPECT_FALSE(sanitized_request.has_url());
+  EXPECT_EQ(expected.suffix_prefix_hash(),
+            sanitized_request.suffix_prefix_hash());
+  EXPECT_FLOAT_EQ(expected.client_score(), sanitized_request.client_score());
+  EXPECT_EQ(expected.is_phishing(), sanitized_request.is_phishing());
+
+  ASSERT_EQ(expected.feature_map_size(), sanitized_request.feature_map_size());
+  for (int i = 0; i < expected.feature_map_size(); ++i) {
+    EXPECT_EQ(expected.feature_map(i).name(),
+              sanitized_request.feature_map(i).name()) << "Feature " << i;
+    EXPECT_DOUBLE_EQ(expected.feature_map(i).value(),
+                     sanitized_request.feature_map(i).value())
+        << "Feature " << i;
+  }
+  EXPECT_EQ(expected.model_version(), sanitized_request.model_version());
+  ASSERT_EQ(expected.non_model_feature_map_size(),
+            sanitized_request.non_model_feature_map_size());
+  for (int i = 0; i < expected.non_model_feature_map_size(); ++i) {
+    EXPECT_EQ(expected.non_model_feature_map(i).name(),
+              sanitized_request.non_model_feature_map(i).name())
+        << "Non-model feature " << i;
+    EXPECT_DOUBLE_EQ(expected.non_model_feature_map(i).value(),
+                     sanitized_request.non_model_feature_map(i).value())
+        << "Non-model feature " << i;
+  }
+
+  // Also check the serialized forms in case there's a field that we forget
+  // to add above.
+  EXPECT_EQ(expected.SerializeAsString(),
+            sanitized_request.SerializeAsString());
+}
 }  // namespace safe_browsing