Anti-DDoS enhancements: Log to net log, UMA stats, improved policy.

net/url_request/url_request_throttler_unittest.cc

Index: net/url_request/url_request_throttler_unittest.cc
diff --git a/net/url_request/url_request_throttler_unittest.cc b/net/url_request/url_request_throttler_unittest.cc
index f6227b32ebabfbfec6c9a481b9ce5251b17180a7..bd431abe817b345060b306fe14fd0c07c959bc84 100644
--- a/net/url_request/url_request_throttler_unittest.cc
+++ b/net/url_request/url_request_throttler_unittest.cc
@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 #include "base/memory/scoped_ptr.h"
+#include "base/metrics/histogram.h"
 #include "base/pickle.h"
 #include "base/stringprintf.h"
 #include "base/string_number_conversions.h"
@@ -19,6 +20,10 @@ using base::TimeTicks;
 namespace net {
 
 namespace {
+
+using base::Histogram;
+using base::StatisticsRecorder;
+
 class MockURLRequestThrottlerManager;
 
 class MockBackoffEntry : public BackoffEntry {
@@ -29,7 +34,7 @@ class MockBackoffEntry : public BackoffEntry {
 
   virtual ~MockBackoffEntry() {}
 
-  TimeTicks GetTimeNow() const {
+  TimeTicks ImplGetTimeNow() const OVERRIDE {

[yzshen1, 2011/05/25 03:49:06]

It might be good to add 'virtual'. 

[Jói, 2011/05/25 14:30:35]

Done.

     return fake_now_;
   }
 
@@ -45,7 +50,7 @@ class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
  public :
   explicit MockURLRequestThrottlerEntry(
       net::URLRequestThrottlerManager* manager)
-      : net::URLRequestThrottlerEntry(manager),
+      : net::URLRequestThrottlerEntry(manager, ""),
         mock_backoff_entry_(&backoff_policy_) {
     InitPolicy();
   }
@@ -54,7 +59,7 @@ class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
       const TimeTicks& exponential_backoff_release_time,
       const TimeTicks& sliding_window_release_time,
       const TimeTicks& fake_now)
-      : net::URLRequestThrottlerEntry(manager),
+      : net::URLRequestThrottlerEntry(manager, ""),
         fake_time_now_(fake_now),
         mock_backoff_entry_(&backoff_policy_) {
     InitPolicy();
@@ -86,13 +91,13 @@ class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
     fake_time_now_ = time_now;
     mock_backoff_entry_.SetFakeNow(time_now);
 
-    GetBackoffEntry()->InformOfRequest(true);  // Sets failure count to 0.
+    GetBackoffEntry()->Reset();
     GetBackoffEntry()->SetCustomReleaseTime(time_now);
     set_sliding_window_release_time(time_now);
   }
 
   // Overridden for tests.
-  virtual TimeTicks GetTimeNow() const { return fake_time_now_; }
+  virtual TimeTicks ImplGetTimeNow() const OVERRIDE { return fake_time_now_; }
 
   void set_exponential_backoff_release_time(
       const base::TimeTicks& release_time) {
@@ -219,15 +224,66 @@ struct GurlAndString {
 class URLRequestThrottlerEntryTest : public testing::Test {
  protected:
   virtual void SetUp();
+
+  // After calling this function, histogram snapshots in |samples_| contain
+  // only the delta caused by the test case currently running.
+  void CalculateHistogramDeltas();
+
   TimeTicks now_;
   MockURLRequestThrottlerManager manager_;  // Dummy object, not used.
   scoped_refptr<MockURLRequestThrottlerEntry> entry_;
+
+  std::map<std::string, Histogram::SampleSet> original_samples_;
+  std::map<std::string, Histogram::SampleSet> samples_;
+};
+
+// List of all histograms we care about in these unit tests.
+const char* kHistogramNames[] = {
+  "Throttling.CustomRetryAfterMs",
+  "Throttling.FailureCountAtSuccess",
+  "Throttling.HttpResponseCode",
+  "Throttling.PerceivedDowntime",
+  "Throttling.RequestThrottled",
+  "Throttling.SiteOptedOut",
 };
 
 void URLRequestThrottlerEntryTest::SetUp() {
   now_ = TimeTicks::Now();
   entry_ = new MockURLRequestThrottlerEntry(&manager_);
   entry_->ResetToBlank(now_);
+
+  for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
+    // Must retrieve original samples for each histogram for comparison
+    // as other tests may affect them.
+    const char* name = kHistogramNames[i];
+    Histogram::SampleSet& original = original_samples_[name];
+    Histogram* histogram;
+    if (StatisticsRecorder::FindHistogram(name, &histogram)) {
+      histogram->SnapshotSample(&original);
+    }
+  }
+}
+
+void URLRequestThrottlerEntryTest::CalculateHistogramDeltas() {
+  for (size_t i = 0; i < arraysize(kHistogramNames); ++i) {
+    const char* name = kHistogramNames[i];
+    Histogram::SampleSet& original = original_samples_[name];
+    Histogram::SampleSet& sample = samples_[name];
+
+    Histogram* histogram;
+    if (StatisticsRecorder::FindHistogram(name, &histogram)) {
+      ASSERT_EQ(Histogram::kUmaTargetedHistogramFlag, histogram->flags());
+
+      histogram->SnapshotSample(&sample);
+      // Ensure |original| size is same as |sample|, then subtract original
+      // values.
+      original.Resize(*histogram);
+      sample.Subtract(original);
+    }
+  }
+
+  // Ensure we don't accidentally use the originals in our tests.
+  original_samples_.clear();
 }
 
 std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
@@ -238,6 +294,10 @@ TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
   entry_->set_exponential_backoff_release_time(
       entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
   EXPECT_TRUE(entry_->IsDuringExponentialBackoff());
+
+  CalculateHistogramDeltas();
+  ASSERT_EQ(0, samples_["Throttling.RequestThrottled"].counts(0));
+  ASSERT_EQ(1, samples_["Throttling.RequestThrottled"].counts(1));
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
@@ -246,6 +306,10 @@ TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
   entry_->set_exponential_backoff_release_time(
       entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1));
   EXPECT_FALSE(entry_->IsDuringExponentialBackoff());
+
+  CalculateHistogramDeltas();
+  ASSERT_EQ(2, samples_["Throttling.RequestThrottled"].counts(0));
+  ASSERT_EQ(0, samples_["Throttling.RequestThrottled"].counts(1));
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateRetryAfter) {
@@ -258,13 +322,15 @@ TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateRetryAfter) {
          "increase release_time";
 
   entry_->ResetToBlank(now_);
-  header_w_delay_header.fake_retry_value_ = "-5.5";
   EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
       << "When given a negative value, it should not change the release_time";

[yzshen1, 2011/05/25 03:49:06]

Please also update the string.

[Jói, 2011/05/25 14:30:35]

This was incorrect, I've fixed it now (the call to UpdateWithResponse with a
negative time header was missing).

+
+  CalculateHistogramDeltas();
+  ASSERT_EQ(1, samples_["Throttling.CustomRetryAfterMs"].TotalCount());
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
-  MockURLRequestThrottlerHeaderAdapter failure_response(505);
+  MockURLRequestThrottlerHeaderAdapter failure_response(503);
   entry_->UpdateWithResponse("", &failure_response);
   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
       << "A failure should increase the release_time";
@@ -278,12 +344,19 @@ TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
 }
 
 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
-  MockURLRequestThrottlerHeaderAdapter failure_response(500);
+  MockURLRequestThrottlerHeaderAdapter failure_response(503);
   MockURLRequestThrottlerHeaderAdapter success_response(200);
   entry_->UpdateWithResponse("", &success_response);
   entry_->UpdateWithResponse("", &failure_response);
   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
       << "This scenario should add delay";
+  entry_->UpdateWithResponse("", &success_response);
+
+  CalculateHistogramDeltas();
+  ASSERT_EQ(1, samples_["Throttling.HttpResponseCode"].counts(503));
+  ASSERT_EQ(2, samples_["Throttling.HttpResponseCode"].counts(200));
+  ASSERT_EQ(1, samples_["Throttling.FailureCountAtSuccess"].counts(1));
+  ASSERT_EQ(1, samples_["Throttling.PerceivedDowntime"].TotalCount());
 }
 
 TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
@@ -308,7 +381,7 @@ TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
 
 TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
   for (int i = 0; i < 30; ++i) {
-    MockURLRequestThrottlerHeaderAdapter response_adapter(505);
+    MockURLRequestThrottlerHeaderAdapter response_adapter(503);
     entry_->UpdateWithResponse("", &response_adapter);
   }
 
@@ -318,7 +391,7 @@ TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
 }
 
 TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
-  MockURLRequestThrottlerHeaderAdapter response_adapter(505);
+  MockURLRequestThrottlerHeaderAdapter response_adapter(503);
   for (int i = 0; i < 5; ++i)
     entry_->UpdateWithResponse("", &response_adapter);
 
@@ -329,7 +402,7 @@ TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
   // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
   // is what happens in practice (if a body is received, then a non-500
   // response must also have been received).
-  entry_->ReceivedContentWasMalformed();
+  entry_->ReceivedContentWasMalformed(200);
   MockURLRequestThrottlerHeaderAdapter success_adapter(200);
   entry_->UpdateWithResponse("", &success_adapter);
   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
@@ -386,6 +459,9 @@ TEST(URLRequestThrottlerManager, IsUrlStandardised) {
       GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
                     std::string("http://www.example.com/index.php"),
                     __LINE__),
+      GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
+                    std::string("http://www.example.com/index.php"),
+                    __LINE__),
       GurlAndString(GURL("http://www.example.com:1234/"),
                     std::string("http://www.example.com:1234/"),
                     __LINE__)};
@@ -485,4 +561,36 @@ TEST(URLRequestThrottlerManager, OptOutHeader) {
   ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry, true);
 }
 
+TEST(URLRequestThrottlerManager, ClearOnNetworkChange) {
+  for (int i = 0; i < 3; ++i) {
+    MockURLRequestThrottlerManager manager;
+    scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_before =
+        manager.RegisterRequestUrl(GURL("http://www.example.com/"));
+    MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
+    for (int j = 0; j < 10; ++j) {
+      // Host doesn't really matter in this scenario so we skip it.
+      entry_before->UpdateWithResponse("", &failure_adapter);
+    }
+    EXPECT_TRUE(entry_before->IsDuringExponentialBackoff());
+
+    switch (i) {
+      case 0:
+        manager.OnIPAddressChanged();
+        break;
+      case 1:
+        manager.OnOnlineStateChanged(true);
+        break;
+      case 2:
+        manager.OnOnlineStateChanged(false);
+        break;
+      default:
+        FAIL();
+    }
+
+    scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_after =
+        manager.RegisterRequestUrl(GURL("http://www.example.com/"));
+    EXPECT_FALSE(entry_after->IsDuringExponentialBackoff());
+  }
+}
+
 }  // namespace net