Move full hash caching logic to v4_get_hash_protocol_manager

components/safe_browsing_db/v4_get_hash_protocol_manager_unittest.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/safe_browsing_db/v4_get_hash_protocol_manager.h"
 
 #include <memory>
 #include <vector>
 
 #include "base/base64.h"
 #include "base/memory/ptr_util.h"
+#include "base/run_loop.h"
 #include "base/strings/stringprintf.h"
 #include "base/test/simple_test_clock.h"
 #include "base/time/time.h"
 #include "components/safe_browsing_db/safebrowsing.pb.h"
 #include "components/safe_browsing_db/testing_util.h"
 #include "components/safe_browsing_db/util.h"
 #include "net/base/escape.h"
 #include "net/base/load_flags.h"
 #include "net/base/net_errors.h"
 #include "net/url_request/test_url_fetcher_factory.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 using base::Time;
 using base::TimeDelta;
 
 namespace {
 
 const char kClient[] = "unittest";
 const char kAppVer[] = "1.0";
 const char kKeyParam[] = "test_key_param";
 
 }  // namespace
 
 namespace safe_browsing {
 
-class SafeBrowsingV4GetHashProtocolManagerTest : public testing::Test {
+class V4GetHashProtocolManagerTest : public testing::Test {
  protected:
   std::unique_ptr<V4GetHashProtocolManager> CreateProtocolManager() {
     V4ProtocolConfig config;
     config.client_name = kClient;
     config.version = kAppVer;
     config.key_param = kKeyParam;
     return std::unique_ptr<V4GetHashProtocolManager>(
         V4GetHashProtocolManager::Create(NULL, config));
   }
 
@@ skipping 34 matching lines @@
   EXPECT_EQ(expected_cache_expire, cache_expire);
   ASSERT_EQ(expected_full_hashes.size(), full_hashes.size());
 
   for (unsigned int i = 0; i < expected_full_hashes.size(); ++i) {
     const SBFullHashResult& expected = expected_full_hashes[i];
     const SBFullHashResult& actual = full_hashes[i];
     EXPECT_TRUE(SBFullHashEqual(expected.hash, actual.hash));
     EXPECT_EQ(expected.metadata, actual.metadata);
     EXPECT_EQ(expected.cache_expire_after, actual.cache_expire_after);
   }
+
 }
 
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest,
+TEST_F(V4GetHashProtocolManagerTest,
        TestGetHashErrorHandlingNetwork) {
   net::TestURLFetcherFactory factory;
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   std::vector<SBPrefix> prefixes;
   std::vector<SBFullHashResult> expected_full_hashes;
   base::Time expected_cache_expire;
 
   pm->GetFullHashesWithApis(
       prefixes, base::Bind(&ValidateGetV4HashResults, expected_full_hashes,
                            expected_cache_expire));
 
   net::TestURLFetcher* fetcher = factory.GetFetcherByID(0);
   DCHECK(fetcher);
   // Failed request status should result in error.
   fetcher->set_status(net::URLRequestStatus(net::URLRequestStatus::FAILED,
                                             net::ERR_CONNECTION_RESET));
   fetcher->set_response_code(200);
   fetcher->SetResponseString(GetStockV4HashResponse());
   fetcher->delegate()->OnURLFetchComplete(fetcher);
 
   // Should have recorded one error, but back off multiplier is unchanged.
   EXPECT_EQ(1ul, pm->gethash_error_count_);
   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
 }
 
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest,
+TEST_F(V4GetHashProtocolManagerTest,
        TestGetHashErrorHandlingResponseCode) {
   net::TestURLFetcherFactory factory;
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   std::vector<SBPrefix> prefixes;
   std::vector<SBFullHashResult> expected_full_hashes;
   base::Time expected_cache_expire;
 
   pm->GetFullHashesWithApis(
       prefixes, base::Bind(&ValidateGetV4HashResults, expected_full_hashes,
                            expected_cache_expire));
 
   net::TestURLFetcher* fetcher = factory.GetFetcherByID(0);
   DCHECK(fetcher);
   fetcher->set_status(net::URLRequestStatus());
   // Response code of anything other than 200 should result in error.
   fetcher->set_response_code(204);
   fetcher->SetResponseString(GetStockV4HashResponse());
   fetcher->delegate()->OnURLFetchComplete(fetcher);
 
   // Should have recorded one error, but back off multiplier is unchanged.
   EXPECT_EQ(1ul, pm->gethash_error_count_);
   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
 }
 
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest, TestGetHashErrorHandlingOK) {
+TEST_F(V4GetHashProtocolManagerTest, TestGetHashErrorHandlingOK) {
   net::TestURLFetcherFactory factory;
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   base::Time now = base::Time::UnixEpoch();
   SetTestClock(now, pm.get());
 
-  std::vector<SBPrefix> prefixes;
+  std::vector<SBPrefix> prefixes = {2877448190};
   std::vector<SBFullHashResult> expected_full_hashes;
   SBFullHashResult hash_result;
   hash_result.hash = SBFullHashForString("Everything's shiny, Cap'n.");
   hash_result.metadata.api_permissions.insert("NOTIFICATIONS");
   hash_result.cache_expire_after = now + base::TimeDelta::FromSeconds(300);
   expected_full_hashes.push_back(hash_result);
   base::Time expected_cache_expire = now + base::TimeDelta::FromSeconds(600);
 
   pm->GetFullHashesWithApis(
       prefixes, base::Bind(&ValidateGetV4HashResults, expected_full_hashes,
                            expected_cache_expire));
 
   net::TestURLFetcher* fetcher = factory.GetFetcherByID(0);
   DCHECK(fetcher);
   fetcher->set_status(net::URLRequestStatus());
   fetcher->set_response_code(200);
   fetcher->SetResponseString(GetStockV4HashResponse());
   fetcher->delegate()->OnURLFetchComplete(fetcher);
 
   // No error, back off multiplier is unchanged.
   EXPECT_EQ(0ul, pm->gethash_error_count_);
   EXPECT_EQ(1ul, pm->gethash_back_off_mult_);
+
+  // Verify the state of the cache.
+  const V4GetHashProtocolManager::PrefixToFullHashResultsMap& cache =
+      pm->v4_full_hash_cache()->at(SB_THREAT_TYPE_API_ABUSE);
+  // Check the cache.
+  EXPECT_EQ(1u, cache.size());
+  EXPECT_EQ(1u, cache.count(prefixes[0]));
+  const SBCachedFullHashResult& cached_result = cache.at(prefixes[0]);
+  EXPECT_EQ(1u, cached_result.full_hashes.size());
+  EXPECT_TRUE(SBFullHashEqual(SBFullHashForString("Everything's shiny, Cap'n."), cached_result.full_hashes[0].hash));
 }
 
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest, TestGetHashRequest) {
+TEST_F(V4GetHashProtocolManagerTest, TestGetHashRequest) {
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   FindFullHashesRequest req;
   ThreatInfo* info = req.mutable_threat_info();
   info->add_threat_types(API_ABUSE);
   info->add_platform_types(CHROME_PLATFORM);
   info->add_threat_entry_types(URL);
 
   SBPrefix one = 1u;
   SBPrefix two = 2u;
@@ skipping 14 matching lines @@
 
   std::vector<PlatformType> platform;
   platform.push_back(CHROME_PLATFORM);
   std::vector<SBPrefix> prefixes;
   prefixes.push_back(one);
   prefixes.push_back(two);
   prefixes.push_back(three);
   EXPECT_EQ(req_base64, pm->GetHashRequest(prefixes, platform, API_ABUSE));
 }
 
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest, TestParseHashResponse) {
+TEST_F(V4GetHashProtocolManagerTest, TestParseHashResponse) {
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   base::Time now = base::Time::UnixEpoch();
   SetTestClock(now, pm.get());
 
   FindFullHashesResponse res;
   res.mutable_negative_cache_duration()->set_seconds(600);
   res.mutable_minimum_wait_duration()->set_seconds(400);
   ThreatMatch* m = res.add_matches();
   m->set_threat_type(API_ABUSE);
@@ skipping 21 matching lines @@
                               full_hashes[0].hash));
   EXPECT_EQ(1ul, full_hashes[0].metadata.api_permissions.size());
   EXPECT_EQ(1ul,
             full_hashes[0].metadata.api_permissions.count("NOTIFICATIONS"));
   EXPECT_EQ(now +
       base::TimeDelta::FromSeconds(300), full_hashes[0].cache_expire_after);
   EXPECT_EQ(now + base::TimeDelta::FromSeconds(400), pm->next_gethash_time_);
 }
 
 // Adds an entry with an ignored ThreatEntryType.
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest,
+TEST_F(V4GetHashProtocolManagerTest,
        TestParseHashResponseWrongThreatEntryType) {
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   base::Time now = base::Time::UnixEpoch();
   SetTestClock(now, pm.get());
 
   FindFullHashesResponse res;
   res.mutable_negative_cache_duration()->set_seconds(600);
   res.add_matches()->set_threat_entry_type(EXECUTABLE);
 
   // Serialize.
   std::string res_data;
   res.SerializeToString(&res_data);
 
   std::vector<SBFullHashResult> full_hashes;
   base::Time cache_expire;
   EXPECT_FALSE(pm->ParseHashResponse(res_data, &full_hashes, &cache_expire));
 
   EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
   // There should be no hash results.
   EXPECT_EQ(0ul, full_hashes.size());
 }
 
 // Adds entries with a ThreatPatternType metadata.
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest,
+TEST_F(V4GetHashProtocolManagerTest,
        TestParseHashThreatPatternType) {
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   base::Time now = base::Time::UnixEpoch();
   SetTestClock(now, pm.get());
 
   // Test social engineering pattern type.
   FindFullHashesResponse se_res;
   se_res.mutable_negative_cache_duration()->set_seconds(600);
   ThreatMatch* se = se_res.add_matches();
@@ skipping 58 matching lines @@
 
   std::string invalid_data;
   invalid_res.SerializeToString(&invalid_data);
   full_hashes.clear();
   EXPECT_FALSE(
       pm->ParseHashResponse(invalid_data, &full_hashes, &cache_expire));
   EXPECT_EQ(0ul, full_hashes.size());
 }
 
 // Adds metadata with a key value that is not "permission".
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest,
+TEST_F(V4GetHashProtocolManagerTest,
        TestParseHashResponseNonPermissionMetadata) {
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   base::Time now = base::Time::UnixEpoch();
   SetTestClock(now, pm.get());
 
   FindFullHashesResponse res;
   res.mutable_negative_cache_duration()->set_seconds(600);
   ThreatMatch* m = res.add_matches();
   m->set_threat_type(API_ABUSE);
@@ skipping 11 matching lines @@
   res.SerializeToString(&res_data);
 
   std::vector<SBFullHashResult> full_hashes;
   base::Time cache_expire;
   EXPECT_FALSE(pm->ParseHashResponse(res_data, &full_hashes, &cache_expire));
 
   EXPECT_EQ(now + base::TimeDelta::FromSeconds(600), cache_expire);
   EXPECT_EQ(0ul, full_hashes.size());
 }
 
-TEST_F(SafeBrowsingV4GetHashProtocolManagerTest,
+TEST_F(V4GetHashProtocolManagerTest,
        TestParseHashResponseInconsistentThreatTypes) {
   std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
 
   FindFullHashesResponse res;
   res.mutable_negative_cache_duration()->set_seconds(600);
   ThreatMatch* m1 = res.add_matches();
   m1->set_threat_type(API_ABUSE);
   m1->set_platform_type(CHROME_PLATFORM);
   m1->set_threat_entry_type(URL);
   m1->mutable_threat()->set_hash(
       SBFullHashToString(SBFullHashForString("Everything's shiny, Cap'n.")));
   m1->mutable_threat_entry_metadata()->add_entries();
   ThreatMatch* m2 = res.add_matches();
   m2->set_threat_type(MALWARE_THREAT);
   m2->set_threat_entry_type(URL);
   m2->mutable_threat()->set_hash(
       SBFullHashToString(SBFullHashForString("Not to fret.")));
 
   // Serialize.
   std::string res_data;
   res.SerializeToString(&res_data);
 
   std::vector<SBFullHashResult> full_hashes;
   base::Time cache_expire;
   EXPECT_FALSE(pm->ParseHashResponse(res_data, &full_hashes, &cache_expire));
 }
 
+// Checks that results are looked up correctly in the cache.
+TEST_F(V4GetHashProtocolManagerTest, GetCachedResults) {
+  base::Time now = base::Time::UnixEpoch();
+  std::vector<SBFullHash> full_hashes;
+  SBFullHash full_hash = SBFullHashForString("example.com/");
+  full_hashes.push_back(full_hash);
+  std::vector<SBFullHashResult> cached_results;
+  std::vector<SBPrefix> prefixes;
+  std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
+  pm->GetFullHashCachedResults(SB_THREAT_TYPE_API_ABUSE,
+      full_hashes, now, &prefixes, &cached_results);
+
+  // The cache is empty.
+  EXPECT_TRUE(cached_results.empty());
+  EXPECT_EQ(1ul, prefixes.size());
+  EXPECT_EQ(full_hash.prefix, prefixes[0]);
+
+  // Prefix has a cache entry but full hash is not there.
+  SBCachedFullHashResult& entry = pm->
+      v4_full_hash_cache()->at(SB_THREAT_TYPE_API_ABUSE)[full_hash.prefix] =
+      SBCachedFullHashResult(now + base::TimeDelta::FromMinutes(5));
+  pm->GetFullHashCachedResults(SB_THREAT_TYPE_API_ABUSE,
+      full_hashes, now, &prefixes, &cached_results);
+
+  EXPECT_TRUE(prefixes.empty());
+  EXPECT_TRUE(cached_results.empty());
+
+  // Expired negative cache entry.
+  entry.expire_after = now - base::TimeDelta::FromMinutes(5);
+  pm->GetFullHashCachedResults(SB_THREAT_TYPE_API_ABUSE,
+      full_hashes, now, &prefixes, &cached_results);
+
+  EXPECT_TRUE(cached_results.empty());
+  EXPECT_EQ(1ul, prefixes.size());
+  EXPECT_EQ(full_hash.prefix, prefixes[0]);
+
+  // Now put the full hash in the cache.
+  SBFullHashResult full_hash_result;
+  full_hash_result.hash = full_hash;
+  full_hash_result.cache_expire_after = now + base::TimeDelta::FromMinutes(3);
+  entry.full_hashes.push_back(full_hash_result);
+  pm->GetFullHashCachedResults(SB_THREAT_TYPE_API_ABUSE,
+      full_hashes, now, &prefixes, &cached_results);
+
+  EXPECT_TRUE(prefixes.empty());
+  EXPECT_EQ(1ul, cached_results.size());
+  EXPECT_TRUE(SBFullHashEqual(full_hash, cached_results[0].hash));
+
+  // Expired full hash in cache.
+  entry.full_hashes.clear();
+  full_hash_result.cache_expire_after = now - base::TimeDelta::FromMinutes(3);
+  entry.full_hashes.push_back(full_hash_result);
+  pm->GetFullHashCachedResults(SB_THREAT_TYPE_API_ABUSE,
+      full_hashes, now, &prefixes, &cached_results);
+
+  EXPECT_TRUE(cached_results.empty());
+  EXPECT_EQ(1ul, prefixes.size());
+  EXPECT_EQ(full_hash.prefix, prefixes[0]);
+}
+/*
+// Checks that the cached results and request results are merged.
+TEST_F(V4GetHashProtocolManagerTest, CachedResultsMerged) {
+  //TestClient client;
+  const GURL url("https://www.example.com/more");
+  std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
+  // Set now to max time so the cache expire times are in the future.
+  SBFullHash full_hash = SBFullHashForString("example.com/");
+  SBFullHashResult full_hash_result;
+  full_hash_result.hash = full_hash;
+  full_hash_result.metadata.api_permissions.insert("GEOLOCATION");
+  full_hash_result.cache_expire_after = base::Time::Max();
+  //pm->AddGetFullHashResponse(full_hash_result);
+  //pm->SetNegativeCacheDurationMins(base::Time::Max(), 0);
+
+  EXPECT_TRUE(pm->v4_full_hash_cache()->empty());
+  EXPECT_FALSE(pm->CheckApiBlacklistUrl(url, &client));
+  base::RunLoop().RunUntilIdle();
+
+  EXPECT_TRUE(client.callback_invoked());
+  const std::set<std::string>& permissions = client.GetBlockedPermissions();
+  EXPECT_EQ(1ul, permissions.size());
+  EXPECT_EQ(1ul, permissions.count("GEOLOCATION"));
+
+  // The results should be cached, so remove them from the protocol manager
+  // response.
+  //TestClient client2;
+  //pm->ClearFullHashResponse();
+  //pm->SetNegativeCacheDurationMins(base::Time(), 0);
+  EXPECT_FALSE(pm->CheckApiBlacklistUrl(url, &client2));
+  base::RunLoop().RunUntilIdle();
+
+  EXPECT_TRUE(client2.callback_invoked());
+  const std::set<std::string>& permissions2 =
+      client2.GetBlockedPermissions();
+  EXPECT_EQ(1ul, permissions2.size());
+  EXPECT_EQ(1ul, permissions2.count("GEOLOCATION"));
+
+  // Add a different result to the protocol manager response and ensure it is
+  // merged with the cached result in the metadata.
+  //TestClient client3;
+  const GURL url2("https://m.example.com/more");
+  full_hash_result.hash = SBFullHashForString("m.example.com/");
+  full_hash_result.metadata.api_permissions.insert("NOTIFICATIONS");
+  //pm->AddGetFullHashResponse(full_hash_result);
+  //pm->SetNegativeCacheDurationMins(base::Time::Max(), 0);
+  EXPECT_FALSE(pm->CheckApiBlacklistUrl(url2, &client3));
+  base::RunLoop().RunUntilIdle();
+
+  EXPECT_TRUE(client3.callback_invoked());
+  const std::set<std::string>& permissions3 =
+      client3.GetBlockedPermissions();
+  EXPECT_EQ(2ul, permissions3.size());
+  EXPECT_EQ(1ul, permissions3.count("GEOLOCATION"));
+  EXPECT_EQ(1ul, permissions3.count("NOTIFICATIONS"));
+}
+
+TEST_F(V4GetHashProtocolManagerTest, CachedResultsAreEvicted) {
+  base::Time epoch = base::Time::UnixEpoch();
+  SBFullHashResult full_hash_result;
+  full_hash_result.hash = SBFullHashForString("example.com/");
+  full_hash_result.cache_expire_after = epoch;
+
+  std::unique_ptr<V4GetHashProtocolManager> pm(CreateProtocolManager());
+  V4GetHashProtocolManager::PrefixToFullHashResultsMap& cache =
+      pm->v4_full_hash_cache()->at(SB_THREAT_TYPE_API_ABUSE);
+
+  // Fill the cache with some expired entries.
+  // Both negative cache and full hash expired.
+  cache[full_hash_result.hash.prefix] = SBCachedFullHashResult(epoch);
+  cache[full_hash_result.hash.prefix].full_hashes.push_back(full_hash_result);
+
+  TestClient client;
+  const GURL url("https://www.example.com/more");
+
+  EXPECT_EQ(1ul, cache.size());
+  EXPECT_FALSE(pm->CheckApiBlacklistUrl(url, &client));
+  base::RunLoop().RunUntilIdle();
+
+  // Cache should be empty.
+  EXPECT_TRUE(client.callback_invoked());
+  EXPECT_TRUE(cache.empty());
+
+  // Negative cache still valid and full hash expired.
+  cache[full_hash_result.hash.prefix] =
+      SBCachedFullHashResult(base::Time::Max());
+  cache[full_hash_result.hash.prefix].full_hashes.push_back(full_hash_result);
+
+  EXPECT_EQ(1ul, cache.size());
+  EXPECT_FALSE(pm->CheckApiBlacklistUrl(url, &client));
+  base::RunLoop().RunUntilIdle();
+
+  // Cache entry should still be there.
+  EXPECT_EQ(1ul, cache.size());
+  auto entry = cache.find(full_hash_result.hash.prefix);
+  EXPECT_NE(cache.end(), entry);
+  EXPECT_EQ(base::Time::Max(), entry->second.expire_after);
+  EXPECT_EQ(1ul, entry->second.full_hashes.size());
+  EXPECT_TRUE(SBFullHashEqual(full_hash_result.hash,
+                              entry->second.full_hashes[0].hash));
+  EXPECT_EQ(full_hash_result.cache_expire_after,
+            entry->second.full_hashes[0].cache_expire_after);
+
+  // Negative cache still valid and full hash still valid.
+  cache[full_hash_result.hash.prefix].full_hashes[0].
+      cache_expire_after = base::Time::Max();
+
+  EXPECT_EQ(1ul, cache.size());
+  EXPECT_FALSE(pm->CheckApiBlacklistUrl(url, &client));
+  base::RunLoop().RunUntilIdle();
+
+  // Cache entry should still be there.
+  EXPECT_EQ(1ul, cache.size());
+  entry = cache.find(full_hash_result.hash.prefix);
+  EXPECT_NE(cache.end(), entry);
+  EXPECT_EQ(base::Time::Max(), entry->second.expire_after);
+  EXPECT_EQ(1ul, entry->second.full_hashes.size());
+  EXPECT_TRUE(SBFullHashEqual(full_hash_result.hash,
+                              entry->second.full_hashes[0].hash));
+  EXPECT_EQ(base::Time::Max(),
+            entry->second.full_hashes[0].cache_expire_after);
+
+  // Negative cache expired and full hash still valid.
+  cache[full_hash_result.hash.prefix].expire_after = epoch;
+
+  EXPECT_EQ(1ul, cache.size());
+  EXPECT_FALSE(pm->CheckApiBlacklistUrl(url, &client));
+  base::RunLoop().RunUntilIdle();
+
+  // Cache entry should still be there.
+  EXPECT_EQ(1ul, cache.size());
+  entry = cache.find(full_hash_result.hash.prefix);
+  EXPECT_NE(cache.end(), entry);
+  EXPECT_EQ(epoch, entry->second.expire_after);
+  EXPECT_EQ(1ul, entry->second.full_hashes.size());
+  EXPECT_TRUE(SBFullHashEqual(full_hash_result.hash,
+                              entry->second.full_hashes[0].hash));
+  EXPECT_EQ(base::Time::Max(),
+            entry->second.full_hashes[0].cache_expire_after);
+}
+*/
 }  // namespace safe_browsing