Introduce ERR_NETWORK_CHANGED and allow URLFetcher to automatically retry on that error.

net/url_request/url_fetcher_impl_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/url_request/url_fetcher_impl.h"
 
 #include <string>
 
 #include "base/bind.h"
 #include "base/file_util.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/message_loop_proxy.h"
+#include "base/stringprintf.h"
 #include "base/synchronization/waitable_event.h"
 #include "base/threading/thread.h"
 #include "build/build_config.h"
 #include "crypto/nss_util.h"
+#include "net/base/mock_host_resolver.h"
+#include "net/base/network_change_notifier.h"
 #include "net/http/http_response_headers.h"
 #include "net/test/test_server.h"
 #include "net/url_request/url_fetcher_delegate.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "net/url_request/url_request_test_util.h"
 #include "net/url_request/url_request_throttler_manager.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 #if defined(USE_NSS) || defined(OS_IOS)
 #include "net/ocsp/nss_ocsp.h"
@@ skipping 38 matching lines @@
   virtual TestURLRequestContext* GetURLRequestContext() OVERRIDE {
     return context_;
   }
 
  protected:
   virtual ~ThrottlingTestURLRequestContextGetter() {}
 
   TestURLRequestContext* const context_;
 };
 
+class TestNetworkChangeNotifier : public NetworkChangeNotifier {
+ public:
+  TestNetworkChangeNotifier() : connection_type_(CONNECTION_UNKNOWN) {}
+
+  // Implementation of NetworkChangeNotifier:
+  virtual ConnectionType GetCurrentConnectionType() const OVERRIDE {
+    return connection_type_;
+  }
+
+  void SetCurrentConnectionType(ConnectionType type) {
+    connection_type_ = type;
+    NotifyObserversOfConnectionTypeChange();
+  }
+
+ private:
+  ConnectionType connection_type_;
+};
+
 }  // namespace
 
 class URLFetcherTest : public testing::Test,
                        public URLFetcherDelegate {
  public:
   URLFetcherTest()
       : fetcher_(NULL),
-        context_(new ThrottlingTestURLRequestContext()) {
+        context_(NULL) {
   }
 
   static int GetNumFetcherCores() {
     return URLFetcherImpl::GetNumFetcherCores();
   }
 
   // Creates a URLFetcher, using the program's main thread to do IO.
   virtual void CreateFetcher(const GURL& url);
 
   // URLFetcherDelegate:
@@ skipping 11 matching lines @@
 
   TestURLRequestContext* request_context() {
     return context_.get();
   }
 
  protected:
   // testing::Test:
   virtual void SetUp() OVERRIDE {
     testing::Test::SetUp();
 
+    context_.reset(new ThrottlingTestURLRequestContext());
     io_message_loop_proxy_ = base::MessageLoopProxy::current();
 
 #if defined(USE_NSS) || defined(OS_IOS)
     crypto::EnsureNSSInit();
     EnsureNSSHttpIOInit();
 #endif
   }
 
   virtual void TearDown() OVERRIDE {
 #if defined(USE_NSS) || defined(OS_IOS)
     ShutdownNSSHttpIO();
 #endif
   }
 
   // URLFetcher is designed to run on the main UI thread, but in our tests
   // we assume that the current thread is the IO thread where the URLFetcher
   // dispatches its requests to.  When we wish to simulate being used from
   // a UI thread, we dispatch a worker thread to do so.
   scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
 
   URLFetcherImpl* fetcher_;
-  const scoped_ptr<TestURLRequestContext> context_;
+  scoped_ptr<TestURLRequestContext> context_;
+};
+
+// A test fixture that uses a MockHostResolver, so that name resolutions can
+// be manipulated by the tests to keep connections in the resolving state.
+class URLFetcherMockDnsTest : public URLFetcherTest {
+ public:
+  // testing::Test:
+  virtual void SetUp() OVERRIDE;
+
+  // URLFetcherTest:
+  virtual void CreateFetcher(const GURL& url) OVERRIDE;
+
+  // URLFetcherDelegate:
+  virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
+
+ protected:
+  GURL test_url_;
+  scoped_ptr<TestServer> test_server_;
+  MockHostResolver resolver_;
+  scoped_ptr<URLFetcher> completed_fetcher_;
+  NetworkChangeNotifier::DisableForTest disable_default_notifier_;
+  TestNetworkChangeNotifier network_change_notifier_;
 };
 
 void URLFetcherTest::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcherImpl(url, URLFetcher::GET, this);
   fetcher_->SetRequestContext(new ThrottlingTestURLRequestContextGetter(
       io_message_loop_proxy(), request_context()));
   fetcher_->Start();
 }
 
 void URLFetcherTest::OnURLFetchComplete(const URLFetcher* source) {
@@ skipping 10 matching lines @@
 void URLFetcherTest::CleanupAfterFetchComplete() {
   delete fetcher_;  // Have to delete this here and not in the destructor,
                     // because the destructor won't necessarily run on the
                     // same thread that CreateFetcher() did.
 
   io_message_loop_proxy()->PostTask(FROM_HERE, MessageLoop::QuitClosure());
   // If the current message loop is not the IO loop, it will be shut down when
   // the main loop returns and this thread subsequently goes out of scope.
 }
 
+void URLFetcherMockDnsTest::SetUp() {
+  URLFetcherTest::SetUp();
+
+  resolver_.set_ondemand_mode(true);
+  resolver_.rules()->AddRule("example.com", "127.0.0.1");
+
+  context_.reset(new TestURLRequestContext(true));
+  context_->set_host_resolver(&resolver_);
+  context_->Init();
+
+  test_server_.reset(new TestServer(TestServer::TYPE_HTTP,
+                                    TestServer::kLocalhost,
+                                    FilePath(kDocRoot)));
+  ASSERT_TRUE(test_server_->Start());
+
+  // test_server_.GetURL() returns a URL with 127.0.0.1 (kLocalhost), that is
+  // immediately resolved by the MockHostResolver. Use a hostname instead to
+  // trigger an async resolve.
+  test_url_ = GURL(
+      base::StringPrintf("http://example.com:%d/defaultresponse",
+      test_server_->host_port_pair().port()));
+  ASSERT_TRUE(test_url_.is_valid());
+}
+
+void URLFetcherMockDnsTest::CreateFetcher(const GURL& url) {
+  fetcher_ = new URLFetcherImpl(url, URLFetcher::GET, this);
+  fetcher_->SetRequestContext(new ThrottlingTestURLRequestContextGetter(
+      io_message_loop_proxy(), request_context()));
+}
+
+void URLFetcherMockDnsTest::OnURLFetchComplete(const URLFetcher* source) {
+  io_message_loop_proxy()->PostTask(FROM_HERE, MessageLoop::QuitClosure());
+  ASSERT_EQ(fetcher_, source);
+  EXPECT_EQ(test_url_, source->GetOriginalURL());
+  completed_fetcher_.reset(fetcher_);
+}
+
 namespace {
 
 // Version of URLFetcherTest that does a POST instead
 class URLFetcherPostTest : public URLFetcherTest {
  public:
   // URLFetcherTest:
   virtual void CreateFetcher(const GURL& url) OVERRIDE;
 
   // URLFetcherDelegate:
   virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
@@ skipping 401 matching lines @@
   EXPECT_EQ(ERR_ABORTED, source->GetStatus().error());
   EXPECT_EQ(301, source->GetResponseCode());
   CleanupAfterFetchComplete();
 }
 
 void URLFetcherProtectTest::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcherImpl(url, URLFetcher::GET, this);
   fetcher_->SetRequestContext(new ThrottlingTestURLRequestContextGetter(
       io_message_loop_proxy(), request_context()));
   start_time_ = Time::Now();
-  fetcher_->SetMaxRetries(11);
+  fetcher_->SetMaxRetriesOn5xx(11);
   fetcher_->Start();
 }
 
 void URLFetcherProtectTest::OnURLFetchComplete(const URLFetcher* source) {
   const TimeDelta one_second = TimeDelta::FromMilliseconds(1000);
   if (source->GetResponseCode() >= 500) {
     // Now running ServerUnavailable test.
     // It takes more than 1 second to finish all 11 requests.
     EXPECT_TRUE(Time::Now() - start_time_ >= one_second);
     EXPECT_TRUE(source->GetStatus().is_success());
@@ skipping 18 matching lines @@
     }
   }
 }
 
 void URLFetcherProtectTestPassedThrough::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcherImpl(url, URLFetcher::GET, this);
   fetcher_->SetRequestContext(new ThrottlingTestURLRequestContextGetter(
       io_message_loop_proxy(), request_context()));
   fetcher_->SetAutomaticallyRetryOn5xx(false);
   start_time_ = Time::Now();
-  fetcher_->SetMaxRetries(11);
+  fetcher_->SetMaxRetriesOn5xx(11);
   fetcher_->Start();
 }
 
 void URLFetcherProtectTestPassedThrough::OnURLFetchComplete(
     const URLFetcher* source) {
   const TimeDelta one_minute = TimeDelta::FromMilliseconds(60000);
   if (source->GetResponseCode() >= 500) {
     // Now running ServerUnavailable test.
     // It should get here on the first attempt, so almost immediately and
     // *not* to attempt to execute all 11 requests (2.5 minutes).
@@ skipping 38 matching lines @@
   EXPECT_TRUE(data.empty());
   CleanupAfterFetchComplete();
 }
 
 void URLFetcherCancelTest::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcherImpl(url, URLFetcher::GET, this);
   CancelTestURLRequestContextGetter* context_getter =
       new CancelTestURLRequestContextGetter(io_message_loop_proxy(),
                                             url);
   fetcher_->SetRequestContext(context_getter);
-  fetcher_->SetMaxRetries(2);
+  fetcher_->SetMaxRetriesOn5xx(2);
   fetcher_->Start();
   // We need to wait for the creation of the URLRequestContext, since we
   // rely on it being destroyed as a signal to end the test.
   context_getter->WaitForContextCreation();
   CancelRequest();
 }
 
 void URLFetcherCancelTest::OnURLFetchComplete(
     const URLFetcher* source) {
   // We should have cancelled the request before completion.
@@ skipping 125 matching lines @@
   CreateFetcher(test_server.GetURL("defaultresponse"));
   io_message_loop_proxy()->PostTaskAndReply(
       FROM_HERE,
       base::Bind(&CancelAllOnIO),
       MessageLoop::QuitClosure());
   MessageLoop::current()->Run();
   EXPECT_EQ(0, GetNumFetcherCores());
   delete fetcher_;
 }
 
+TEST_F(URLFetcherMockDnsTest, DontRetryOnNetworkChangedByDefault) {
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // This posts a task to start the fetcher.
+  CreateFetcher(test_url_);
+  fetcher_->Start();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now running, but is pending the host resolve.
+  EXPECT_EQ(1, GetNumFetcherCores());
+  EXPECT_TRUE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+
+  // A network change notification aborts the connect job.
+  NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+  MessageLoop::current()->RunUntilIdle();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+
+  // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
+  EXPECT_EQ(ERR_NETWORK_CHANGED, completed_fetcher_->GetStatus().error());
+}
+
+TEST_F(URLFetcherMockDnsTest, RetryOnNetworkChangedAndFail) {
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // This posts a task to start the fetcher.
+  CreateFetcher(test_url_);
+  fetcher_->SetAutomaticallyRetryOnNetworkChanges(3);
+  fetcher_->Start();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now running, but is pending the host resolve.
+  EXPECT_EQ(1, GetNumFetcherCores());
+  EXPECT_TRUE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+
+  // Make it fail 3 times.
+  for (int i = 0; i < 3; ++i) {
+    // A network change notification aborts the connect job.
+    NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+    MessageLoop::current()->RunUntilIdle();
+
+    // But the fetcher retries automatically.
+    EXPECT_EQ(1, GetNumFetcherCores());
+    EXPECT_TRUE(resolver_.has_pending_requests());
+    ASSERT_FALSE(completed_fetcher_);
+  }
+
+  // A 4th failure doesn't trigger another retry, and propagates the error
+  // to the owner of the fetcher.
+  NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+  MessageLoop::current()->RunUntilIdle();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+
+  // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
+  EXPECT_EQ(ERR_NETWORK_CHANGED, completed_fetcher_->GetStatus().error());
+}
+
+TEST_F(URLFetcherMockDnsTest, RetryOnNetworkChangedAndSucceed) {
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // This posts a task to start the fetcher.
+  CreateFetcher(test_url_);
+  fetcher_->SetAutomaticallyRetryOnNetworkChanges(3);
+  fetcher_->Start();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now running, but is pending the host resolve.
+  EXPECT_EQ(1, GetNumFetcherCores());
+  EXPECT_TRUE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+
+  // Make it fail 3 times.
+  for (int i = 0; i < 3; ++i) {
+    // A network change notification aborts the connect job.
+    NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+    MessageLoop::current()->RunUntilIdle();
+
+    // But the fetcher retries automatically.
+    EXPECT_EQ(1, GetNumFetcherCores());
+    EXPECT_TRUE(resolver_.has_pending_requests());
+    ASSERT_FALSE(completed_fetcher_);
+  }
+
+  // Now let it succeed by resolving the pending request.
+  resolver_.ResolveAllPending();
+  MessageLoop::current()->Run();
+
+  // URLFetcherMockDnsTest::OnURLFetchComplete() will quit the loop.
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+
+  // This time the request succeeded.
+  EXPECT_EQ(OK, completed_fetcher_->GetStatus().error());
+  EXPECT_EQ(200, completed_fetcher_->GetResponseCode());
+}
+
+TEST_F(URLFetcherMockDnsTest, RetryAfterComingBackOnline) {
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // This posts a task to start the fetcher.
+  CreateFetcher(test_url_);
+  fetcher_->SetAutomaticallyRetryOnNetworkChanges(1);
+  fetcher_->Start();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now running, but is pending the host resolve.
+  EXPECT_EQ(1, GetNumFetcherCores());
+  EXPECT_TRUE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+
+  // Make it fail by changing the connection type to offline.
+  EXPECT_EQ(NetworkChangeNotifier::CONNECTION_UNKNOWN,
+            NetworkChangeNotifier::GetConnectionType());
+  EXPECT_FALSE(NetworkChangeNotifier::IsOffline());
+  network_change_notifier_.SetCurrentConnectionType(
+      NetworkChangeNotifier::CONNECTION_NONE);
+  // This makes the connect job fail:
+  NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+  resolver_.ResolveAllPending();
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now waiting for the connection to become online again.
+  EXPECT_EQ(NetworkChangeNotifier::CONNECTION_NONE,
+            NetworkChangeNotifier::GetConnectionType());
+  EXPECT_TRUE(NetworkChangeNotifier::IsOffline());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+  // The core is still alive, but it dropped its request.
+  EXPECT_EQ(0, GetNumFetcherCores());
+
+  // It should retry once the connection is back.
+  network_change_notifier_.SetCurrentConnectionType(
+      NetworkChangeNotifier::CONNECTION_WIFI);
+  MessageLoop::current()->RunUntilIdle();
+  EXPECT_EQ(1, GetNumFetcherCores());
+  ASSERT_FALSE(completed_fetcher_);
+  EXPECT_TRUE(resolver_.has_pending_requests());
+
+  // Resolve the pending request; the fetcher should complete now.
+  resolver_.ResolveAllPending();
+  MessageLoop::current()->Run();
+
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+  EXPECT_EQ(OK, completed_fetcher_->GetStatus().error());
+  EXPECT_EQ(200, completed_fetcher_->GetResponseCode());
+}
+
+TEST_F(URLFetcherMockDnsTest, StartOnlyWhenOnline) {
+  // Start offline.
+  network_change_notifier_.SetCurrentConnectionType(
+      NetworkChangeNotifier::CONNECTION_NONE);
+  EXPECT_TRUE(NetworkChangeNotifier::IsOffline());
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // Create a fetcher that retries on network changes. It will try to connect
+  // only once the network is back online.
+  CreateFetcher(test_url_);
+  fetcher_->SetAutomaticallyRetryOnNetworkChanges(1);
+  fetcher_->Start();
+  MessageLoop::current()->RunUntilIdle();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // It should retry once the connection is back.
+  network_change_notifier_.SetCurrentConnectionType(
+      NetworkChangeNotifier::CONNECTION_WIFI);
+  MessageLoop::current()->RunUntilIdle();
+  EXPECT_EQ(1, GetNumFetcherCores());
+  ASSERT_FALSE(completed_fetcher_);
+  EXPECT_TRUE(resolver_.has_pending_requests());
+
+  // Resolve the pending request; the fetcher should complete now.
+  resolver_.ResolveAllPending();
+  MessageLoop::current()->Run();
+
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+  EXPECT_EQ(OK, completed_fetcher_->GetStatus().error());
+  EXPECT_EQ(200, completed_fetcher_->GetResponseCode());
+}
+
 #if defined(OS_MACOSX)
 // SIGSEGV on Mac: http://crbug.com/60426
 TEST_F(URLFetcherPostTest, DISABLED_Basic) {
 #else
 TEST_F(URLFetcherPostTest, Basic) {
 #endif
   TestServer test_server(TestServer::TYPE_HTTP,
                          TestServer::kLocalhost,
                          FilePath(kDocRoot));
   ASSERT_TRUE(test_server.Start());
@@ skipping 430 matching lines @@
   MessageLoop::current()->Run();  // OnURLFetchComplete() will Quit().
 
   MessageLoop::current()->RunUntilIdle();
   ASSERT_FALSE(file_util::PathExists(file_path_))
       << file_path_.value() << " not removed.";
 }
 
 }  // namespace
 
 }  // namespace net