Implement exponential back-off mechanism and enforce it at the URLRequestHttpJob level.

chrome/common/net/url_fetcher_unittest.cc

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/message_loop_proxy.h"
 #include "base/thread.h"
 #include "base/waitable_event.h"
 #include "build/build_config.h"
 #include "chrome/common/chrome_plugin_lib.h"
 #include "chrome/common/net/url_fetcher.h"
-#include "chrome/common/net/url_fetcher_protect.h"
 #include "chrome/common/net/url_request_context_getter.h"
 #include "net/http/http_response_headers.h"
+#include "net/test/test_server.h"
+#include "net/url_request/request_throttler_manager.h"
 #include "net/url_request/url_request_unittest.h"
-#include "net/test/test_server.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 #if defined(USE_NSS)
 #include "net/ocsp/nss_ocsp.h"
 #endif
 
 using base::Time;
 using base::TimeDelta;
 
 // TODO(eroman): Add a regression test for http://crbug.com/40505.
@@ skipping 279 matching lines @@
   EXPECT_EQ("private", header);
   URLFetcherTest::OnURLFetchComplete(source, url, status, response_code,
                                      cookies, data);
 }
 
 void URLFetcherProtectTest::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcher(url, URLFetcher::GET, this);
   fetcher_->set_request_context(new TestURLRequestContextGetter(
       io_message_loop_proxy()));
   start_time_ = Time::Now();
+  fetcher_->set_max_retries(11);

[willchan no longer on Chromium, 2010/11/24 00:23:07]

11 is a magic number.  Does this have any specific meaning?  Perhaps you can use
a named constant instead.

[yzshen, 2010/11/24 09:42:08]

It is only meaningful for this specific test. I had a hard time to come up with
a name better than kMaxRetriesForURLFetcherProtectTest. :(

I think we have a lot of magic number in this file (say, line 338). And I
haven't added this magic number, I just moved it here from the other part of
this file.

I personally think it is fine. Please let me know if you have better idea. :)

   fetcher_->Start();
 }
 
 void URLFetcherProtectTest::OnURLFetchComplete(const URLFetcher* source,
                                                const GURL& url,
                                                const URLRequestStatus& status,
                                                int response_code,
                                                const ResponseCookies& cookies,
                                                const std::string& data) {
   const TimeDelta one_second = TimeDelta::FromMilliseconds(1000);
   if (response_code >= 500) {
     // Now running ServerUnavailable test.
     // It takes more than 1 second to finish all 11 requests.
     EXPECT_TRUE(Time::Now() - start_time_ >= one_second);
     EXPECT_TRUE(status.is_success());
     EXPECT_FALSE(data.empty());
     delete fetcher_;
     io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask());
   } else {
     // Now running Overload test.
     static int count = 0;
     count++;
     if (count < 20) {
       fetcher_->Start();
     } else {
       // We have already sent 20 requests continuously. And we expect that
-      // it takes more than 1 second due to the overload pretection settings.
+      // it takes more than 1 second due to the overload protection settings.
       EXPECT_TRUE(Time::Now() - start_time_ >= one_second);
       URLFetcherTest::OnURLFetchComplete(source, url, status, response_code,
                                          cookies, data);
     }
   }
 }
 
 void URLFetcherProtectTestPassedThrough::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcher(url, URLFetcher::GET, this);
   fetcher_->set_request_context(new TestURLRequestContextGetter(
       io_message_loop_proxy()));
   fetcher_->set_automatically_retry_on_5xx(false);
   start_time_ = Time::Now();
+  fetcher_->set_max_retries(11);
   fetcher_->Start();
 }
 
 void URLFetcherProtectTestPassedThrough::OnURLFetchComplete(
     const URLFetcher* source,
     const GURL& url,
     const URLRequestStatus& status,
     int response_code,
     const ResponseCookies& cookies,
     const std::string& data) {
@@ skipping 47 matching lines @@
   // The rest is the same as URLFetcherTest::OnURLFetchComplete.
   delete fetcher_;
   io_message_loop_proxy()->PostTask(FROM_HERE, new MessageLoop::QuitTask());
 }
 
 void URLFetcherCancelTest::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcher(url, URLFetcher::GET, this);
   CancelTestURLRequestContextGetter* context_getter =
       new CancelTestURLRequestContextGetter(io_message_loop_proxy());
   fetcher_->set_request_context(context_getter);
+  fetcher_->set_max_retries(2);
   fetcher_->Start();
   // We need to wait for the creation of the URLRequestContext, since we
   // rely on it being destroyed as a signal to end the test.
   context_getter->WaitForContextCreation();
   CancelRequest();
 }
 
 void URLFetcherCancelTest::OnURLFetchComplete(const URLFetcher* source,
                                               const GURL& url,
                                               const URLRequestStatus& status,
@@ skipping 61 matching lines @@
 }
 
 TEST_F(URLFetcherProtectTest, Overload) {
   net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot));
   ASSERT_TRUE(test_server.Start());
 
   GURL url(test_server.GetURL("defaultresponse"));
 
   // Registers an entry for test url. It only allows 3 requests to be sent
   // in 200 milliseconds.
-  URLFetcherProtectManager* manager = URLFetcherProtectManager::GetInstance();
-  URLFetcherProtectEntry* entry =
-      new URLFetcherProtectEntry(200, 3, 11, 1, 2.0, 0, 256);
-  manager->Register(url.host(), entry);
+  scoped_refptr<RequestThrottlerEntry> entry(new RequestThrottlerEntry(
+      200, 3, 1, 0, 2.0, 0.0, 256));
+  RequestThrottlerManager::GetInstance()->OverrideEntryForTests(url, entry);
 
   CreateFetcher(url);
 
   MessageLoop::current()->Run();
+
+  RequestThrottlerManager::GetInstance()->EraseEntryForTests(url);
 }
 
 TEST_F(URLFetcherProtectTest, ServerUnavailable) {
   net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot));
   ASSERT_TRUE(test_server.Start());
 
   GURL url(test_server.GetURL("files/server-unavailable.html"));
 
   // Registers an entry for test url. The backoff time is calculated by:
   //     new_backoff = 2.0 * old_backoff + 0
   // and maximum backoff time is 256 milliseconds.
   // Maximum retries allowed is set to 11.
-  URLFetcherProtectManager* manager = URLFetcherProtectManager::GetInstance();
-  URLFetcherProtectEntry* entry =
-      new URLFetcherProtectEntry(200, 3, 11, 1, 2.0, 0, 256);
-  manager->Register(url.host(), entry);
+  scoped_refptr<RequestThrottlerEntry> entry(new RequestThrottlerEntry(
+      200, 3, 1, 0, 2.0, 0.0, 256));
+  RequestThrottlerManager::GetInstance()->OverrideEntryForTests(url, entry);
 
   CreateFetcher(url);
 
   MessageLoop::current()->Run();
+
+  RequestThrottlerManager::GetInstance()->EraseEntryForTests(url);
 }
 
 TEST_F(URLFetcherProtectTestPassedThrough, ServerUnavailablePropagateResponse) {
   net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot));
   ASSERT_TRUE(test_server.Start());
 
   GURL url(test_server.GetURL("files/server-unavailable.html"));
 
   // Registers an entry for test url. The backoff time is calculated by:
   //     new_backoff = 2.0 * old_backoff + 0
-  // and maximum backoff time is 256 milliseconds.
+  // and maximum backoff time is 150000 milliseconds.
   // Maximum retries allowed is set to 11.
-  URLFetcherProtectManager* manager = URLFetcherProtectManager::GetInstance();
+  scoped_refptr<RequestThrottlerEntry> entry(new RequestThrottlerEntry(
+      200, 3, 100, 0, 2.0, 0.0, 150000));
   // Total time if *not* for not doing automatic backoff would be 150s.
   // In reality it should be "as soon as server responds".
-  URLFetcherProtectEntry* entry =
-      new URLFetcherProtectEntry(200, 3, 11, 100, 2.0, 0, 150000);
-  manager->Register(url.host(), entry);
+  RequestThrottlerManager::GetInstance()->OverrideEntryForTests(url, entry);
 
   CreateFetcher(url);
 
   MessageLoop::current()->Run();
+
+  RequestThrottlerManager::GetInstance()->EraseEntryForTests(url);
 }
 
-
 TEST_F(URLFetcherBadHTTPSTest, BadHTTPSTest) {
   net::TestServer::HTTPSOptions https_options(
       net::TestServer::HTTPSOptions::CERT_EXPIRED);
   net::TestServer test_server(https_options, FilePath(kDocRoot));
   ASSERT_TRUE(test_server.Start());
 
   CreateFetcher(test_server.GetURL("defaultresponse"));
   MessageLoop::current()->Run();
 }
 
 TEST_F(URLFetcherCancelTest, ReleasesContext) {
   net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot));
   ASSERT_TRUE(test_server.Start());
 
   GURL url(test_server.GetURL("files/server-unavailable.html"));
 
   // Registers an entry for test url. The backoff time is calculated by:
   //     new_backoff = 2.0 * old_backoff + 0
   // The initial backoff is 2 seconds and maximum backoff is 4 seconds.
   // Maximum retries allowed is set to 2.
-  URLFetcherProtectManager* manager = URLFetcherProtectManager::GetInstance();
-  URLFetcherProtectEntry* entry =
-      new URLFetcherProtectEntry(200, 3, 2, 2000, 2.0, 0, 4000);
-  manager->Register(url.host(), entry);
+  scoped_refptr<RequestThrottlerEntry> entry(new RequestThrottlerEntry(
+      200, 3, 2000, 0, 2.0, 0.0, 4000));
+  RequestThrottlerManager::GetInstance()->OverrideEntryForTests(url, entry);
 
   // Create a separate thread that will create the URLFetcher.  The current
   // (main) thread will do the IO, and when the fetch is complete it will
   // terminate the main thread's message loop; then the other thread's
   // message loop will be shut down automatically as the thread goes out of
   // scope.
   base::Thread t("URLFetcher test thread");
   ASSERT_TRUE(t.Start());
   t.message_loop()->PostTask(FROM_HERE, new FetcherWrapperTask(this, url));
 
   MessageLoop::current()->Run();
+
+  RequestThrottlerManager::GetInstance()->EraseEntryForTests(url);
 }
 
 TEST_F(URLFetcherCancelTest, CancelWhileDelayedStartTaskPending) {
   net::TestServer test_server(net::TestServer::TYPE_HTTP, FilePath(kDocRoot));
   ASSERT_TRUE(test_server.Start());
 
   GURL url(test_server.GetURL("files/server-unavailable.html"));
 
   // Register an entry for test url.
-  //
-  // Ideally we would mock URLFetcherProtectEntry to return XXX seconds
-  // in response to entry->UpdateBackoff(SEND).
-  //
-  // Unfortunately this function is time sensitive, so we fudge some numbers
-  // to make it at least somewhat likely to have a non-zero deferred
-  // delay when running.
-  //
-  // Using a sliding window of 2 seconds, and max of 1 request, under a fast
+  // Using a sliding window of 4 seconds, and max of 1 request, under a fast
   // run we expect to have a 4 second delay when posting the Start task.
-  URLFetcherProtectManager* manager = URLFetcherProtectManager::GetInstance();
-  URLFetcherProtectEntry* entry =
-      new URLFetcherProtectEntry(2000, 1, 2, 2000, 2.0, 0, 4000);
-  EXPECT_EQ(0, entry->UpdateBackoff(URLFetcherProtectEntry::SEND));
-  entry->UpdateBackoff(URLFetcherProtectEntry::SEND);  // Returns about 2000.
-  manager->Register(url.host(), entry);
+  scoped_refptr<RequestThrottlerEntry> entry(new RequestThrottlerEntry(
+      4000, 1, 2000, 0, 2.0, 0.0, 4000));
+  RequestThrottlerManager::GetInstance()->OverrideEntryForTests(url, entry);
+  // Fake that a request has just started.
+  entry->ReserveSendingTimeForNextRequest(base::TimeTicks());
 
   // The next request we try to send will be delayed by ~4 seconds.
   // The slower the test runs, the less the delay will be (since it takes the
   // time difference from now).
 
   base::Thread t("URLFetcher test thread");
   ASSERT_TRUE(t.Start());
   t.message_loop()->PostTask(FROM_HERE, new FetcherWrapperTask(this, url));
 
   MessageLoop::current()->Run();
+
+  RequestThrottlerManager::GetInstance()->EraseEntryForTests(url);
 }
 
 }  // namespace.