Anti-DDoS enhancements: Log to net log, UMA stats, improved policy.

net/url_request/url_request_throttler_unittest.cc

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/memory/scoped_ptr.h"
+#include "base/metrics/histogram.h"
 #include "base/pickle.h"
 #include "base/stringprintf.h"
 #include "base/string_number_conversions.h"
 #include "base/time.h"
 #include "net/base/test_completion_callback.h"
 #include "net/url_request/url_request_context.h"
 #include "net/url_request/url_request_throttler_header_interface.h"
 #include "net/url_request/url_request_throttler_manager.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 using base::TimeDelta;
 using base::TimeTicks;
 
 namespace net {
 
 namespace {
+
+using base::Histogram;
+using base::StatisticsRecorder;
+
 class MockURLRequestThrottlerManager;
 
 class MockBackoffEntry : public BackoffEntry {
  public:
   explicit MockBackoffEntry(const BackoffEntry::Policy* const policy)
       : BackoffEntry(policy), fake_now_(TimeTicks()) {
   }
 
   virtual ~MockBackoffEntry() {}
 
-  TimeTicks GetTimeNow() const {
+  virtual TimeTicks ImplGetTimeNow() const OVERRIDE {
     return fake_now_;
   }
 
   void SetFakeNow(const TimeTicks& now) {
     fake_now_ = now;
   }
 
  private:
   TimeTicks fake_now_;
 };
 
 class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
  public :
   explicit MockURLRequestThrottlerEntry(
       net::URLRequestThrottlerManager* manager)
-      : net::URLRequestThrottlerEntry(manager),
+      : net::URLRequestThrottlerEntry(manager, ""),
         mock_backoff_entry_(&backoff_policy_) {
     InitPolicy();
   }
   MockURLRequestThrottlerEntry(
       net::URLRequestThrottlerManager* manager,
       const TimeTicks& exponential_backoff_release_time,
       const TimeTicks& sliding_window_release_time,
       const TimeTicks& fake_now)
-      : net::URLRequestThrottlerEntry(manager),
+      : net::URLRequestThrottlerEntry(manager, ""),
         fake_time_now_(fake_now),
         mock_backoff_entry_(&backoff_policy_) {
     InitPolicy();
 
     mock_backoff_entry_.SetFakeNow(fake_now);
     set_exponential_backoff_release_time(exponential_backoff_release_time);
     set_sliding_window_release_time(sliding_window_release_time);
   }
   virtual ~MockURLRequestThrottlerEntry() {}
 
@@ skipping 11 matching lines @@
   }
 
   BackoffEntry* GetBackoffEntry() {
     return &mock_backoff_entry_;
   }
 
   void ResetToBlank(const TimeTicks& time_now) {
     fake_time_now_ = time_now;
     mock_backoff_entry_.SetFakeNow(time_now);
 
-    GetBackoffEntry()->InformOfRequest(true);  // Sets failure count to 0.
+    GetBackoffEntry()->Reset();
     GetBackoffEntry()->SetCustomReleaseTime(time_now);
     set_sliding_window_release_time(time_now);
   }
 
   // Overridden for tests.
-  virtual TimeTicks GetTimeNow() const { return fake_time_now_; }
+  virtual TimeTicks ImplGetTimeNow() const OVERRIDE { return fake_time_now_; }
 
   void set_exponential_backoff_release_time(
       const base::TimeTicks& release_time) {
     GetBackoffEntry()->SetCustomReleaseTime(release_time);
   }
 
   base::TimeTicks sliding_window_release_time() const {
     return URLRequestThrottlerEntry::sliding_window_release_time();
   }
 
@@ skipping 106 matching lines @@
   GURL url;
   std::string result;
   int line;
 };
 
 }  // namespace
 
 class URLRequestThrottlerEntryTest : public testing::Test {
  protected:
   virtual void SetUp();
+
+  // After calling this function, histogram snapshots in |samples_| contain
+  // only the delta caused by the test case currently running.
+  void CalculateHistogramDeltas();
+
   TimeTicks now_;
   MockURLRequestThrottlerManager manager_;  // Dummy object, not used.
   scoped_refptr<MockURLRequestThrottlerEntry> entry_;
+
+  std::map<std::string, Histogram::SampleSet> original_samples_;
+  std::map<std::string, Histogram::SampleSet> samples_;
+};
+
+// List of all histograms we care about in these unit tests.
+const char* kHistogramNames[] = {
+  "Throttling.CustomRetryAfterMs",
+  "Throttling.FailureCountAtSuccess",
+  "Throttling.HttpResponseCode",
+  "Throttling.PerceivedDowntime",
+  "Throttling.RequestThrottled",
+  "Throttling.SiteOptedOut",
 };
 
 void URLRequestThrottlerEntryTest::SetUp() {
   now_ = TimeTicks::Now();
   entry_ = new MockURLRequestThrottlerEntry(&manager_);
   entry_->ResetToBlank(now_);
+
+  for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
+    // Must retrieve original samples for each histogram for comparison
+    // as other tests may affect them.
+    const char* name = kHistogramNames[i];
+    Histogram::SampleSet& original = original_samples_[name];
+    Histogram* histogram;
+    if (StatisticsRecorder::FindHistogram(name, &histogram)) {
+      histogram->SnapshotSample(&original);
+    }
+  }
+}
+
+void URLRequestThrottlerEntryTest::CalculateHistogramDeltas() {
+  for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
+    const char* name = kHistogramNames[i];
+    Histogram::SampleSet& original = original_samples_[name];
+    Histogram::SampleSet& sample = samples_[name];
+
+    Histogram* histogram;
+    if (StatisticsRecorder::FindHistogram(name, &histogram)) {
+      ASSERT_EQ(Histogram::kUmaTargetedHistogramFlag, histogram->flags());
+
+      histogram->SnapshotSample(&sample);
+      // Ensure |original| size is same as |sample|, then subtract original
+      // values.
+      original.Resize(*histogram);
+      sample.Subtract(original);
+    }
+  }
+
+  // Ensure we don't accidentally use the originals in our tests.
+  original_samples_.clear();
 }
 
 std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
   return out << time.ToInternalValue();
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
   entry_->set_exponential_backoff_release_time(
       entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
   EXPECT_TRUE(entry_->IsDuringExponentialBackoff());
+
+  CalculateHistogramDeltas();
+  ASSERT_EQ(0, samples_["Throttling.RequestThrottled"].counts(0));
+  ASSERT_EQ(1, samples_["Throttling.RequestThrottled"].counts(1));
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
   entry_->set_exponential_backoff_release_time(entry_->fake_time_now_);
   EXPECT_FALSE(entry_->IsDuringExponentialBackoff());
   entry_->set_exponential_backoff_release_time(
       entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1));
   EXPECT_FALSE(entry_->IsDuringExponentialBackoff());
+
+  CalculateHistogramDeltas();
+  ASSERT_EQ(2, samples_["Throttling.RequestThrottled"].counts(0));
+  ASSERT_EQ(0, samples_["Throttling.RequestThrottled"].counts(1));
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateRetryAfter) {
   // If the response we received has a retry-after field,
   // the request should be delayed.
   MockURLRequestThrottlerHeaderAdapter header_w_delay_header("5.5", "", 200);
   entry_->UpdateWithResponse("", &header_w_delay_header);
   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
       << "When the server put a positive value in retry-after we should "
          "increase release_time";
 
   entry_->ResetToBlank(now_);
-  header_w_delay_header.fake_retry_value_ = "-5.5";
+  MockURLRequestThrottlerHeaderAdapter header_w_negative_header(
+      "-5.5", "", 200);
+  entry_->UpdateWithResponse("", &header_w_negative_header);
   EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
       << "When given a negative value, it should not change the release_time";
+
+  CalculateHistogramDeltas();
+  ASSERT_EQ(1, samples_["Throttling.CustomRetryAfterMs"].TotalCount());
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
-  MockURLRequestThrottlerHeaderAdapter failure_response(505);
+  MockURLRequestThrottlerHeaderAdapter failure_response(503);
   entry_->UpdateWithResponse("", &failure_response);
   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
       << "A failure should increase the release_time";
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
   MockURLRequestThrottlerHeaderAdapter success_response(200);
   entry_->UpdateWithResponse("", &success_response);
   EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
       << "A success should not add any delay";
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
-  MockURLRequestThrottlerHeaderAdapter failure_response(500);
+  MockURLRequestThrottlerHeaderAdapter failure_response(503);
   MockURLRequestThrottlerHeaderAdapter success_response(200);
   entry_->UpdateWithResponse("", &success_response);
   entry_->UpdateWithResponse("", &failure_response);
   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
       << "This scenario should add delay";
+  entry_->UpdateWithResponse("", &success_response);
+
+  CalculateHistogramDeltas();
+  ASSERT_EQ(1, samples_["Throttling.HttpResponseCode"].counts(503));
+  ASSERT_EQ(2, samples_["Throttling.HttpResponseCode"].counts(200));
+  ASSERT_EQ(1, samples_["Throttling.FailureCountAtSuccess"].counts(1));
+  ASSERT_EQ(1, samples_["Throttling.PerceivedDowntime"].TotalCount());
 }
 
 TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
   TimeDelta lifetime = TimeDelta::FromMilliseconds(
       MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
   const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
 
   TimeAndBool test_values[] = {
       TimeAndBool(now_, false, __LINE__),
       TimeAndBool(now_ - kFiveMs, false, __LINE__),
       TimeAndBool(now_ + kFiveMs, false, __LINE__),
       TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
       TimeAndBool(now_ - lifetime, true, __LINE__),
       TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
 
   for (unsigned int i = 0; i < arraysize(test_values); ++i) {
     entry_->set_exponential_backoff_release_time(test_values[i].time);
     EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
         "Test case #" << i << " line " << test_values[i].line << " failed";
   }
 }
 
 TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
   for (int i = 0; i < 30; ++i) {
-    MockURLRequestThrottlerHeaderAdapter response_adapter(505);
+    MockURLRequestThrottlerHeaderAdapter response_adapter(503);
     entry_->UpdateWithResponse("", &response_adapter);
   }
 
   TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
   EXPECT_EQ(delay.InMilliseconds(),
             MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
 }
 
 TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
-  MockURLRequestThrottlerHeaderAdapter response_adapter(505);
+  MockURLRequestThrottlerHeaderAdapter response_adapter(503);
   for (int i = 0; i < 5; ++i)
     entry_->UpdateWithResponse("", &response_adapter);
 
   TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
 
   // Inform the entry that a response body was malformed, which is supposed to
   // increase the back-off time.  Note that we also submit a successful
   // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
   // is what happens in practice (if a body is received, then a non-500
   // response must also have been received).
-  entry_->ReceivedContentWasMalformed();
+  entry_->ReceivedContentWasMalformed(200);
   MockURLRequestThrottlerHeaderAdapter success_adapter(200);
   entry_->UpdateWithResponse("", &success_adapter);
   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
 }
 
 TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
   int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
   int sliding_window =
       URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
 
@@ skipping 36 matching lines @@
                     __LINE__),
       GurlAndString(GURL("http://www.example.com/0/token/false"),
                     std::string("http://www.example.com/0/token/false"),
                     __LINE__),
       GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
                     std::string("http://www.example.com/index.php"),
                     __LINE__),
       GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
                     std::string("http://www.example.com/index.php"),
                     __LINE__),
+      GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
+                    std::string("http://www.example.com/index.php"),
+                    __LINE__),
       GurlAndString(GURL("http://www.example.com:1234/"),
                     std::string("http://www.example.com:1234/"),
                     __LINE__)};
 
   for (unsigned int i = 0; i < arraysize(test_values); ++i) {
     std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
     EXPECT_EQ(temp, test_values[i].result) <<
         "Test case #" << i << " line " << test_values[i].line << " failed";
   }
 }
@@ skipping 79 matching lines @@
   MockURLRequestThrottlerHeaderAdapter wrong_adapter("", "yesplease", 200);
   no_opt_out_entry->UpdateWithResponse("www.nike.com", &wrong_adapter);
   ExpectEntryAllowsAllOnErrorIfOptedOut(no_opt_out_entry, false);
 
   // A localhost entry should always be opted out.
   scoped_refptr<net::URLRequestThrottlerEntryInterface> localhost_entry =
       manager.RegisterRequestUrl(GURL("http://localhost/hello"));
   ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry, true);
 }
 
+TEST(URLRequestThrottlerManager, ClearOnNetworkChange) {
+  for (int i = 0; i < 3; ++i) {
+    MockURLRequestThrottlerManager manager;
+    scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_before =
+        manager.RegisterRequestUrl(GURL("http://www.example.com/"));
+    MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
+    for (int j = 0; j < 10; ++j) {
+      // Host doesn't really matter in this scenario so we skip it.
+      entry_before->UpdateWithResponse("", &failure_adapter);
+    }
+    EXPECT_TRUE(entry_before->IsDuringExponentialBackoff());
+
+    switch (i) {
+      case 0:
+        manager.OnIPAddressChanged();
+        break;
+      case 1:
+        manager.OnOnlineStateChanged(true);
+        break;
+      case 2:
+        manager.OnOnlineStateChanged(false);
+        break;
+      default:
+        FAIL();
+    }
+
+    scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_after =
+        manager.RegisterRequestUrl(GURL("http://www.example.com/"));
+    EXPECT_FALSE(entry_after->IsDuringExponentialBackoff());
+  }
+}
+
 }  // namespace net