Add the client for accessing Subresource Filter only list.

chrome/browser/subresource_filter/subresource_filter_browsertest.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <map>
 #include <memory>
 #include <string>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/command_line.h"
 #include "base/files/file_path.h"
 #include "base/location.h"
 #include "base/macros.h"
 #include "base/path_service.h"
 #include "base/strings/string_piece.h"
+#include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
 #include "base/test/histogram_tester.h"
 #include "chrome/browser/browser_process.h"
 #include "chrome/browser/metrics/subprocess_metrics_provider.h"
 #include "chrome/browser/page_load_metrics/observers/subresource_filter_metrics_observer.h"
 #include "chrome/browser/safe_browsing/test_safe_browsing_service.h"
+#include "chrome/browser/safe_browsing/v4_test_utils.h"
 #include "chrome/browser/subresource_filter/test_ruleset_publisher.h"
 #include "chrome/browser/ui/browser.h"
 #include "chrome/browser/ui/browser_commands.h"
 #include "chrome/browser/ui/tabs/tab_strip_model.h"
 #include "chrome/common/chrome_paths.h"
 #include "chrome/common/url_constants.h"
 #include "chrome/test/base/in_process_browser_test.h"
 #include "chrome/test/base/ui_test_utils.h"
 #include "components/safe_browsing_db/test_database_manager.h"
 #include "components/safe_browsing_db/util.h"
+#include "components/safe_browsing_db/v4_database.h"
 #include "components/security_interstitials/content/unsafe_resource.h"
 #include "components/subresource_filter/content/browser/content_subresource_filter_driver_factory.h"
 #include "components/subresource_filter/core/browser/subresource_filter_features.h"
 #include "components/subresource_filter/core/browser/subresource_filter_features_test_support.h"
 #include "components/subresource_filter/core/common/activation_level.h"
 #include "components/subresource_filter/core/common/scoped_timers.h"
 #include "components/subresource_filter/core/common/test_ruleset_utils.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/notification_service.h"
 #include "content/public/browser/notification_types.h"
 #include "content/public/browser/render_frame_host.h"
 #include "content/public/browser/web_contents.h"
 #include "content/public/common/content_switches.h"
 #include "content/public/test/browser_test_utils.h"
 #include "content/public/test/test_navigation_observer.h"
 #include "content/public/test/test_utils.h"
 #include "net/dns/mock_host_resolver.h"
 #include "net/test/embedded_test_server/embedded_test_server.h"
 #include "net/test/spawned_test_server/spawned_test_server.h"
 #include "net/test/test_data_directory.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace {
 
 // The path to a multi-frame document used for tests.
 static constexpr const char kTestFrameSetPath[] =
-    "subresource_filter/frame_set.html";
+    "/subresource_filter/frame_set.html";
 
 // Names of DocumentLoad histograms.
 constexpr const char kDocumentLoadActivationLevel[] =
     "SubresourceFilter.DocumentLoad.ActivationState";
 constexpr const char kSubresourceLoadsTotal[] =
     "SubresourceFilter.DocumentLoad.NumSubresourceLoads.Total";
 constexpr const char kSubresourceLoadsEvaluated[] =
     "SubresourceFilter.DocumentLoad.NumSubresourceLoads.Evaluated";
 constexpr const char kSubresourceLoadsMatchedRules[] =
     "SubresourceFilter.DocumentLoad.NumSubresourceLoads.MatchedRules";
@@ skipping 20 matching lines @@
     "SubresourceFilter.PageLoad.SubresourceEvaluation.TotalCPUDuration";
 constexpr const char kEvaluationTotalWallDurationForDocument[] =
     "SubresourceFilter.DocumentLoad.SubresourceEvaluation.TotalWallDuration";
 constexpr const char kEvaluationTotalCPUDurationForDocument[] =
     "SubresourceFilter.DocumentLoad.SubresourceEvaluation.TotalCPUDuration";
 constexpr const char kEvaluationWallDuration[] =
     "SubresourceFilter.SubresourceLoad.Evaluation.WallDuration";
 constexpr const char kEvaluationCPUDuration[] =
     "SubresourceFilter.SubresourceLoad.Evaluation.CPUDuration";
 
+#if defined(GOOGLE_CHROME_BUILD)
+// Names of navigation chain patterns histogram.
+const char kMatchesPatternHistogramName[] =
+    "SubresourceFilter.PageLoad.RedirectChainMatchPattern";
+const char kNavigationChainSize[] =
+    "SubresourceFilter.PageLoad.RedirectChainLength";
+const char kSubresourceFilterOnlySuffix[] = ".SubresourceFilterOnly";
+#endif
+
 // Other histograms.
 const char kSubresourceFilterPromptHistogram[] =
     "SubresourceFilter.Prompt.NumVisibility";
 
-// Database manager that allows any URL to be configured as blacklisted for
-// testing.
-class FakeSafeBrowsingDatabaseManager
-    : public safe_browsing::TestSafeBrowsingDatabaseManager {
- public:
-  FakeSafeBrowsingDatabaseManager() {}
-
-  void AddBlacklistedURL(const GURL& url,
-                         safe_browsing::SBThreatType threat_type) {
-    url_to_threat_type_[url] = threat_type;
-  }
-
- protected:
-  ~FakeSafeBrowsingDatabaseManager() override {}
-
-  bool CheckBrowseUrl(const GURL& url, Client* client) override {
-    if (!url_to_threat_type_.count(url))
-      return true;
-
-    content::BrowserThread::PostTask(
-        content::BrowserThread::IO, FROM_HERE,
-        base::Bind(&Client::OnCheckBrowseUrlResult, base::Unretained(client),
-                   url, url_to_threat_type_[url],
-                   safe_browsing::ThreatMetadata()));
-    return false;
-  }
-
-  bool CheckResourceUrl(const GURL& url, Client* client) override {
-    return true;
-  }
-
-  bool IsSupported() const override { return true; }
-  bool ChecksAreAlwaysAsync() const override { return false; }
-  bool CanCheckResourceType(
-      content::ResourceType /* resource_type */) const override {
-    return true;
-  }
-
-  safe_browsing::ThreatSource GetThreatSource() const override {
-    return safe_browsing::ThreatSource::LOCAL_PVER4;
-  }
-
-  bool CheckExtensionIDs(const std::set<std::string>& extension_ids,
-                         Client* client) override {
-    return true;
-  }
-
- private:
-  std::map<GURL, safe_browsing::SBThreatType> url_to_threat_type_;
-
-  DISALLOW_COPY_AND_ASSIGN(FakeSafeBrowsingDatabaseManager);
+// Human readable representation of expected redirect chain match patterns.
+// The explanations for the buckets given for the following redirect chain:
+// A->B->C->D, where A is initial URL and D is a final URL.
+enum RedirectChainMatchPattern {
+  EMPTY,             // No histograms were recorded.
+  F0M0L1,            // D is a Safe Browsing match.
+  F0M1L0,            // B or C, or both are Safe Browsing matches.
+  F0M1L1,            // B or C, or both and D are Safe Browsing matches.
+  F1M0L0,            // A is Safe Browsing match
+  F1M0L1,            // A and D are Safe Browsing matches.
+  F1M1L0,            // B and/or C and A are Safe Browsing matches.
+  F1M1L1,            // B and/or C and A and D are Safe Browsing matches.
+  NO_REDIRECTS_HIT,  // Redirect chain consists of single URL, aka no redirects
+                     // has happened, and this URL was a Safe Browsing hit.
+  NUM_HIT_PATTERNS,
 };
 
 // UI manager that never actually shows any interstitials, but emulates as if
 // the user chose to proceed through them.
 class FakeSafeBrowsingUIManager
     : public safe_browsing::TestSafeBrowsingUIManager {
  public:
   FakeSafeBrowsingUIManager() {}
 
  protected:
   ~FakeSafeBrowsingUIManager() override {}
 
   void DisplayBlockingPage(const UnsafeResource& resource) override {
     resource.callback_thread->PostTask(
         FROM_HERE, base::Bind(resource.callback, true /* proceed */));
   }
 
  private:
   DISALLOW_COPY_AND_ASSIGN(FakeSafeBrowsingUIManager);
 };
 
 GURL GetURLWithFragment(const GURL& url, base::StringPiece fragment) {
   GURL::Replacements replacements;
   replacements.SetRefStr(fragment);
   return url.ReplaceComponents(replacements);
 }
 
-GURL GetURLWithQuery(const GURL& url, base::StringPiece query) {
-  GURL::Replacements replacements;
-  replacements.SetQueryStr(query);
-  return url.ReplaceComponents(replacements);
-}
-
 }  // namespace
 
 namespace subresource_filter {
 
 using subresource_filter::testing::ScopedSubresourceFilterFeatureToggle;
 using subresource_filter::testing::TestRulesetPublisher;
 using subresource_filter::testing::TestRulesetCreator;
 using subresource_filter::testing::TestRulesetPair;
 using ActivationDecision =
     ContentSubresourceFilterDriverFactory::ActivationDecision;
@@ skipping 23 matching lines @@
   // called after ChromeBrowserMainParts::PreBrowserStart(), which instantiates
   // the RulesetService.
   //
   // On the other hand, setting up field trials in this method would be too
   // early, as it is called before BrowserMain, which expects no FieldTrialList
   // singleton to exist. There are no other hooks we could use either.
   //
   // As a workaround, enable the feature here, then enable the feature once
   // again + set up the field trials later.
   void SetUpCommandLine(base::CommandLine* command_line) override {
-    command_line->AppendSwitchASCII(switches::kEnableFeatures,
-                                    kSafeBrowsingSubresourceFilter.name);
+    command_line->AppendSwitchASCII(
+        switches::kEnableFeatures,
+        base::JoinString({std::string(kSafeBrowsingSubresourceFilter.name),

[engedy, 2017/03/21 13:44:16]

nit: I think you can drop std::string, should work with literals.

[melandory, 2017/03/24 15:49:27]

Done.

+                          "SafeBrowsingV4OnlyEnabled"},
+                         ","));
   }
 
-  void SetUpInProcessBrowserTestFixture() override {
-    fake_safe_browsing_database_ = new FakeSafeBrowsingDatabaseManager();
-    test_safe_browsing_factory_.SetTestDatabaseManager(
-        fake_safe_browsing_database_.get());
-    test_safe_browsing_factory_.SetTestUIManager(new FakeSafeBrowsingUIManager);
-    safe_browsing::SafeBrowsingService::RegisterFactory(
-        &test_safe_browsing_factory_);
+  void SetUp() override {
+    sb_factory_ =
+        base::MakeUnique<safe_browsing::TestSafeBrowsingServiceFactory>(
+            safe_browsing::V4FeatureList::V4UsageStatus::V4_ONLY);
+    sb_factory_->SetTestUIManager(new FakeSafeBrowsingUIManager());
+    safe_browsing::SafeBrowsingService::RegisterFactory(sb_factory_.get());
+
+    safe_browsing::V4Database::RegisterStoreFactoryForTest(
+        base::WrapUnique(new safe_browsing::TestV4StoreFactory()));
+
+    v4_db_factory_ = new safe_browsing::TestV4DatabaseFactory();
+    safe_browsing::V4Database::RegisterDatabaseFactoryForTest(
+        base::WrapUnique(v4_db_factory_));
+
+    v4_get_hash_factory_ =
+        new safe_browsing::TestV4GetHashProtocolManagerFactory();
+    safe_browsing::V4GetHashProtocolManager::RegisterFactory(
+        base::WrapUnique(v4_get_hash_factory_));
+    InProcessBrowserTest::SetUp();
+  }
+
+  void TearDown() override {
+    InProcessBrowserTest::TearDown();
+    // Unregister test factories after InProcessBrowserTest::TearDown
+    // (which destructs SafeBrowsingService).
+    safe_browsing::V4GetHashProtocolManager::RegisterFactory(nullptr);
+    safe_browsing::V4Database::RegisterDatabaseFactoryForTest(nullptr);
+    safe_browsing::V4Database::RegisterStoreFactoryForTest(nullptr);
+    safe_browsing::SafeBrowsingService::RegisterFactory(nullptr);
   }
 
   void SetUpOnMainThread() override {
     scoped_feature_toggle_.reset(new ScopedSubresourceFilterFeatureToggle(
         base::FeatureList::OVERRIDE_ENABLE_FEATURE, kActivationLevelEnabled,
         kActivationScopeActivationList, kActivationListPhishingInterstitial,
         measure_performance_ ? "1" : "0", "" /* suppress_notifications */,
         whitelist_site_on_reload_ ? "true" : "false"));
 
     base::FilePath test_data_dir;
     PathService::Get(chrome::DIR_TEST_DATA, &test_data_dir);
     embedded_test_server()->ServeFilesFromDirectory(test_data_dir);
     host_resolver()->AddSimulatedFailure("host-with-dns-lookup-failure");
     host_resolver()->AddRule("*", "127.0.0.1");
     content::SetupCrossSiteRedirector(embedded_test_server());
     ASSERT_TRUE(embedded_test_server()->Start());
   }
 
   GURL GetTestUrl(const std::string& relative_url) {
     return embedded_test_server()->base_url().Resolve(relative_url);
   }
 
+  void MarkUrlAsMatchingListWithId(
+      const GURL& bad_url,
+      const safe_browsing::ListIdentifier& list_id,
+      safe_browsing::ThreatPatternType threat_pattern_type) {
+    safe_browsing::FullHashInfo full_hash_info =
+        GetFullHashInfoWithMetadata(bad_url, list_id, threat_pattern_type);
+    v4_db_factory_->MarkPrefixAsBad(list_id, full_hash_info.full_hash);
+    v4_get_hash_factory_->AddToFullHashCache(full_hash_info);
+  }
+
   void ConfigureAsPhishingURL(const GURL& url) {
-    fake_safe_browsing_database_->AddBlacklistedURL(
-        url, safe_browsing::SB_THREAT_TYPE_URL_PHISHING);
+    MarkUrlAsMatchingListWithId(url, safe_browsing::GetUrlSocEngId(),
+                                safe_browsing::ThreatPatternType::NONE);
+  }
+
+  void ConfigureAsSubresourceFilterOnlyURL(const GURL& url) {
+    MarkUrlAsMatchingListWithId(url, safe_browsing::GetUrlSubresourceFilterId(),
+                                safe_browsing::ThreatPatternType::NONE);
   }
 
   content::WebContents* web_contents() {
     return browser()->tab_strip_model()->GetActiveWebContents();
   }
 
   content::RenderFrameHost* FindFrameByName(const std::string& name) {
     for (content::RenderFrameHost* frame : web_contents()->GetAllFrames()) {
       if (frame->GetFrameName() == name)
         return frame;
@@ skipping 55 matching lines @@
 
   void SetRulesetWithRules(const std::vector<proto::UrlRule>& rules) {
     TestRulesetPair test_ruleset_pair;
     ruleset_creator_.CreateRulesetWithRules(rules, &test_ruleset_pair);
     ASSERT_NO_FATAL_FAILURE(
         test_ruleset_publisher_.SetRuleset(test_ruleset_pair.unindexed));
   }
 
  private:
   TestRulesetCreator ruleset_creator_;
-  safe_browsing::TestSafeBrowsingServiceFactory test_safe_browsing_factory_;
-  scoped_refptr<FakeSafeBrowsingDatabaseManager> fake_safe_browsing_database_;
 
   std::unique_ptr<ScopedSubresourceFilterFeatureToggle> scoped_feature_toggle_;
   TestRulesetPublisher test_ruleset_publisher_;
   const bool measure_performance_;
   const bool whitelist_site_on_reload_;
 
+  std::unique_ptr<safe_browsing::TestSafeBrowsingServiceFactory> sb_factory_;
+  // Owned by the V4Database.
+  safe_browsing::TestV4DatabaseFactory* v4_db_factory_;
+  // Owned by the V4GetHashProtocolManager.
+  safe_browsing::TestV4GetHashProtocolManagerFactory* v4_get_hash_factory_;
+
   DISALLOW_COPY_AND_ASSIGN(SubresourceFilterBrowserTestImpl);
 };
 
 class SubresourceFilterBrowserTest : public SubresourceFilterBrowserTestImpl {
  public:
   SubresourceFilterBrowserTest()
       : SubresourceFilterBrowserTestImpl(false, false) {}
 };
 
 // TODO(pkalinnikov): It should be possible to have only one fixture, i.e.,
@@ skipping 111 matching lines @@
   const std::vector<const char*> kSubframeNames{"one", "two", "three"};
   const std::vector<bool> kExpectScriptInFrameToLoad{false, true, false};
   ASSERT_NO_FATAL_FAILURE(ExpectParsedScriptElementLoadedStatusInFrames(
       kSubframeNames, kExpectScriptInFrameToLoad));
 
   tester.ExpectBucketCount(kSubresourceFilterPromptHistogram, true, 1);
 }
 
 IN_PROC_BROWSER_TEST_F(SubresourceFilterBrowserTest,
                        HistoryNavigationActivation) {
-  GURL url_without_activation(GetTestUrl(kTestFrameSetPath));
-  GURL url_with_activation(
-      GetURLWithQuery(url_without_activation, "activation"));
+  GURL url_with_activation(GetTestUrl(kTestFrameSetPath));
+  GURL url_without_activation(
+      embedded_test_server()->GetURL("a.com", kTestFrameSetPath));
   ConfigureAsPhishingURL(url_with_activation);
   ASSERT_NO_FATAL_FAILURE(
       SetRulesetToDisallowURLsWithPathSuffix("included_script.js"));
 
   const std::vector<const char*> kSubframeNames{"one", "two", "three"};
   const std::vector<bool> kExpectScriptInFrameToLoadWithoutActivation{
       true, true, true};
   const std::vector<bool> kExpectScriptInFrameToLoadWithActivation{false, true,
                                                                    false};
 
@@ skipping 505 matching lines @@
       internal::kHistogramSubresourceFilterActivationDecision,
       static_cast<int>(ActivationDecision::URL_WHITELISTED), 1);
 
   tester.ExpectTotalCount(
       internal::kHistogramSubresourceFilterActivationDecisionReload, 1);
   tester.ExpectBucketCount(
       internal::kHistogramSubresourceFilterActivationDecisionReload,
       static_cast<int>(ActivationDecision::URL_WHITELISTED), 1);
 }
 
+#if defined(GOOGLE_CHROME_BUILD)
+// This test is only enabled when GOOGLE_CHROME_BUILD is true because the store
+// that this test uses is only populated on GOOGLE_CHROME_BUILD builds.
+IN_PROC_BROWSER_TEST_F(
+    SubresourceFilterBrowserTest,
+    ExpectRedirectPatternHistogramsAreRecordedForSubresourceFilterOnlyMatch) {
+  ASSERT_NO_FATAL_FAILURE(SetRulesetToDisallowURLsWithPathSuffix(
+      "suffix-that-does-not-match-anything"));
+
+  GURL url(GetTestUrl("subresource_filter/frame_with_included_script.html"));
+  ConfigureAsSubresourceFilterOnlyURL(url);
+
+  base::HistogramTester tester;
+  ui_test_utils::NavigateToURL(browser(), url);
+
+  EXPECT_THAT(tester.GetAllSamples(std::string(kMatchesPatternHistogramName) +
+                                   std::string(kSubresourceFilterOnlySuffix)),
+              ::testing::ElementsAre(base::Bucket(NO_REDIRECTS_HIT, 1)));
+  EXPECT_THAT(tester.GetAllSamples(std::string(kNavigationChainSize) +
+                                   std::string(kSubresourceFilterOnlySuffix)),
+              ::testing::ElementsAre(base::Bucket(1, 1)));
+}
+
+IN_PROC_BROWSER_TEST_F(
+    SubresourceFilterBrowserTest,
+    ExpectRedirectPatternHistogramsAreRecordedForSubresourceFilterOnlyRedirectMatch) {
+  ASSERT_NO_FATAL_FAILURE(
+      SetRulesetToDisallowURLsWithPathSuffix("included_script.js"));
+  const std::string initial_host("a.com");
+  const std::string redirected_host("b.com");
+
+  GURL redirect_url(embedded_test_server()->GetURL(
+      redirected_host, "/subresource_filter/frame_with_included_script.html"));
+  GURL url(embedded_test_server()->GetURL(
+      initial_host, "/server-redirect?" + redirect_url.spec()));
+
+  ConfigureAsSubresourceFilterOnlyURL(url.GetOrigin());
+  base::HistogramTester tester;
+  ui_test_utils::NavigateToURL(browser(), url);
+  EXPECT_THAT(tester.GetAllSamples(std::string(kMatchesPatternHistogramName) +

[engedy, 2017/03/21 13:44:16]

nit: Could you please add a comment to explain that this is not (yet) supported?

[melandory, 2017/03/24 15:49:27]

You mean that the real list is empty?

+                                   std::string(kSubresourceFilterOnlySuffix)),
+              ::testing::IsEmpty());
+
+  EXPECT_THAT(tester.GetAllSamples(std::string(kNavigationChainSize) +
+                                   std::string(kSubresourceFilterOnlySuffix)),
+              ::testing::IsEmpty());
+}
+#endif
+
 }  // namespace subresource_filter

[engedy, 2017/03/21 13:44:16]

We should have at least one test that verifies end-to-end behavior on
SUBRESOURCE_FILTER sites:
  a) when that list is enabled via variations parameters, verify that
subresources end up getting filtered,
  b) when the list is not enabled, verify that subresources are not getting
filtered.

Given that the throttle goes live as of CL, it would be good to do it in this
CL.

[melandory, 2017/03/24 15:49:27]

Done.