Introduce ERR_NETWORK_CHANGED and allow URLFetcher to automatically retry on that error.

net/url_request/url_fetcher_impl_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/url_request/url_fetcher_impl.h"
 
 #include <string>
 
 #include "base/bind.h"
 #include "base/file_util.h"
 #include "base/files/scoped_temp_dir.h"
 #include "base/message_loop_proxy.h"
+#include "base/stringprintf.h"
 #include "base/synchronization/waitable_event.h"
 #include "base/threading/thread.h"
 #include "build/build_config.h"
 #include "crypto/nss_util.h"
+#include "net/base/mock_host_resolver.h"
+#include "net/base/network_change_notifier.h"
 #include "net/http/http_response_headers.h"
 #include "net/test/test_server.h"
 #include "net/url_request/url_fetcher_delegate.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "net/url_request/url_request_test_util.h"
 #include "net/url_request/url_request_throttler_manager.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 #if defined(USE_NSS) || defined(OS_IOS)
 #include "net/ocsp/nss_ocsp.h"
@@ skipping 38 matching lines @@
   virtual TestURLRequestContext* GetURLRequestContext() OVERRIDE {
     return context_;
   }
 
  protected:
   virtual ~ThrottlingTestURLRequestContextGetter() {}
 
   TestURLRequestContext* const context_;
 };
 
+class TestNetworkChangeNotifier : public NetworkChangeNotifier {

[pauljensen, 2012/12/07 23:00:05]

Using GMOCK for this might be a few lines shorter, for an example see
chrome/browser/chromeos/drive/drive_sync_client_unittest.cc

[Joao da Silva, 2012/12/10 15:32:42]

These tests still need to invoke NotifyObserversOfConnectionTypeChange(), which
is protected in the superclass. So the mock would be very similar to this, to
make that method public.

+ public:
+  TestNetworkChangeNotifier() : connection_type_(CONNECTION_UNKNOWN) {}
+
+  // Implementation of NetworkChangeNotifier:
+  virtual ConnectionType GetCurrentConnectionType() const OVERRIDE {
+    return connection_type_;
+  }
+
+  void SetCurrentConnectionType(ConnectionType type) {
+    connection_type_ = type;
+    NotifyObserversOfConnectionTypeChange();
+  }
+
+ private:
+  ConnectionType connection_type_;
+};
+
 }  // namespace
 
 class URLFetcherTest : public testing::Test,
                        public URLFetcherDelegate {
  public:
   URLFetcherTest()
       : fetcher_(NULL),
         context_(new ThrottlingTestURLRequestContext()) {
   }
 
@@ skipping 14 matching lines @@
   void CleanupAfterFetchComplete();
 
   scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy() {
     return io_message_loop_proxy_;
   }
 
   TestURLRequestContext* request_context() {
     return context_.get();
   }
 
+  void UseTestNetworkChangeNotifier() {
+    disable_default_notifier_.reset(new NetworkChangeNotifier::DisableForTest);
+    network_change_notifier_.reset(new TestNetworkChangeNotifier);

[pauljensen, 2012/12/07 23:00:05]

Could |disable_default_notifier_| and |network_change_notifier_| just be simple
members rather than scoped_ptr's?  Then UseTestNetworkChangeNotifier() could be
eliminated.

[Joao da Silva, 2012/12/10 15:32:42]

Done.

[Joao da Silva, 2012/12/10 15:32:42]

Done.

+  }
+
  protected:
   // testing::Test:
   virtual void SetUp() OVERRIDE {
     testing::Test::SetUp();
 
     io_message_loop_proxy_ = base::MessageLoopProxy::current();
 
 #if defined(USE_NSS) || defined(OS_IOS)
     crypto::EnsureNSSInit();
     EnsureNSSHttpIOInit();
 #endif
   }
 
   virtual void TearDown() OVERRIDE {
 #if defined(USE_NSS) || defined(OS_IOS)
     ShutdownNSSHttpIO();
 #endif
   }
 
   // URLFetcher is designed to run on the main UI thread, but in our tests
   // we assume that the current thread is the IO thread where the URLFetcher
   // dispatches its requests to.  When we wish to simulate being used from
   // a UI thread, we dispatch a worker thread to do so.
   scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
 
   URLFetcherImpl* fetcher_;
-  const scoped_ptr<TestURLRequestContext> context_;
+  scoped_ptr<TestURLRequestContext> context_;
+
+  scoped_ptr<NetworkChangeNotifier::DisableForTest> disable_default_notifier_;
+  scoped_ptr<TestNetworkChangeNotifier> network_change_notifier_;
 };
 
 void URLFetcherTest::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcherImpl(url, URLFetcher::GET, this);
   fetcher_->SetRequestContext(new ThrottlingTestURLRequestContextGetter(
       io_message_loop_proxy(), request_context()));
-  fetcher_->Start();
 }
 
 void URLFetcherTest::OnURLFetchComplete(const URLFetcher* source) {
   EXPECT_TRUE(source->GetStatus().is_success());
   EXPECT_EQ(200, source->GetResponseCode());  // HTTP OK
 
   std::string data;
   EXPECT_TRUE(source->GetResponseAsString(&data));
   EXPECT_FALSE(data.empty());
 
   CleanupAfterFetchComplete();
 }
 
 void URLFetcherTest::CleanupAfterFetchComplete() {
   delete fetcher_;  // Have to delete this here and not in the destructor,
                     // because the destructor won't necessarily run on the
                     // same thread that CreateFetcher() did.
 
   io_message_loop_proxy()->PostTask(FROM_HERE, MessageLoop::QuitClosure());
   // If the current message loop is not the IO loop, it will be shut down when
   // the main loop returns and this thread subsequently goes out of scope.
 }
 
 namespace {
 
+class URLFetcherMockDNSTest : public URLFetcherTest {
+ public:
+  // testing::Test:
+  virtual void SetUp() OVERRIDE;
+
+  // URLFetcherDelegate:
+  virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
+
+ protected:
+  GURL test_url_;
+  scoped_ptr<TestServer> test_server_;
+  MockHostResolver resolver_;
+  scoped_ptr<URLFetcher> completed_fetcher_;
+};
+
 // Version of URLFetcherTest that does a POST instead
 class URLFetcherPostTest : public URLFetcherTest {
  public:
   // URLFetcherTest:
   virtual void CreateFetcher(const GURL& url) OVERRIDE;
 
   // URLFetcherDelegate:
   virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
 };
 
@@ skipping 241 matching lines @@
   // Set by the test. Used in OnURLFetchComplete() to decide if
   // the URLFetcher should own the temp file, so that we can test
   // disowning prevents the file from being deleted.
   bool take_ownership_of_file_;
 
   // Expected file error code for the test.
   // PLATFORM_FILE_OK when expecting success.
   base::PlatformFileError expected_file_error_;
 };
 
+void URLFetcherMockDNSTest::SetUp() {
+  URLFetcherTest::SetUp();
+
+  context_.reset();
+  UseTestNetworkChangeNotifier();
+  ASSERT_TRUE(network_change_notifier_);
+
+  resolver_.set_synchronous_mode(false);
+  resolver_.set_resolve_automatically(false);
+  resolver_.rules()->AddRule("example.com", "127.0.0.1");
+
+  context_.reset(new TestURLRequestContext(true));
+  context_->set_host_resolver(&resolver_);
+  context_->Init();
+
+  test_server_.reset(new TestServer(TestServer::TYPE_HTTP,
+                                    TestServer::kLocalhost,
+                                    FilePath(kDocRoot)));
+  ASSERT_TRUE(test_server_->Start());
+
+  // test_server_.GetURL() returns a URL with 127.0.0.1 (kLocalhost), that is
+  // immediately resolved by the MockHostResolver. Use a hostname instead to
+  // trigger an async resolve.
+  test_url_ = GURL(
+      base::StringPrintf("http://example.com:%d/defaultresponse",
+      test_server_->host_port_pair().port()));
+  ASSERT_TRUE(test_url_.is_valid());
+}
+
+void URLFetcherMockDNSTest::OnURLFetchComplete(const URLFetcher* source) {
+  io_message_loop_proxy()->PostTask(FROM_HERE, MessageLoop::QuitClosure());
+  ASSERT_EQ(fetcher_, source);
+  EXPECT_EQ(test_url_, source->GetOriginalURL());
+  completed_fetcher_.reset(fetcher_);
+}
+
 void URLFetcherPostTest::CreateFetcher(const GURL& url) {
   fetcher_ = new URLFetcherImpl(url, URLFetcher::POST, this);
   fetcher_->SetRequestContext(new ThrottlingTestURLRequestContextGetter(
       io_message_loop_proxy(), request_context()));
   fetcher_->SetUploadData("application/x-www-form-urlencoded",
                           "bobsyeruncle");
   fetcher_->Start();
 }
 
 void URLFetcherPostTest::OnURLFetchComplete(const URLFetcher* source) {
@@ skipping 382 matching lines @@
   CreateFetcher(test_server.GetURL("defaultresponse"));
   io_message_loop_proxy()->PostTaskAndReply(
       FROM_HERE,
       base::Bind(&CancelAllOnIO),
       MessageLoop::QuitClosure());
   MessageLoop::current()->Run();
   EXPECT_EQ(0, GetNumFetcherCores());
   delete fetcher_;
 }
 
+TEST_F(URLFetcherMockDNSTest, DontRetryOnNetworkChangedByDefault) {
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // This posts a task to start the fetcher.
+  CreateFetcher(test_url_);
+  fetcher_->Start();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now running, but is pending the host resolve.
+  EXPECT_EQ(1, GetNumFetcherCores());
+  EXPECT_TRUE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+
+  // A network change notification aborts the connect job.
+  NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+  MessageLoop::current()->RunUntilIdle();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+
+  // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
+  EXPECT_EQ(ERR_NETWORK_CHANGED, completed_fetcher_->GetStatus().error());
+}
+
+TEST_F(URLFetcherMockDNSTest, RetryOnNetworkChangedAndFail) {
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // This posts a task to start the fetcher.
+  CreateFetcher(test_url_);
+  fetcher_->SetAutomaticallyRetryOnNetworkChanges(3);
+  fetcher_->Start();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now running, but is pending the host resolve.
+  EXPECT_EQ(1, GetNumFetcherCores());
+  EXPECT_TRUE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+
+  // Make it fail 3 times.
+  for (int i = 0; i < 3; ++i) {
+    // A network change notification aborts the connect job.
+    NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+    MessageLoop::current()->RunUntilIdle();
+
+    // But the fetcher retries automatically.
+    EXPECT_EQ(1, GetNumFetcherCores());
+    EXPECT_TRUE(resolver_.has_pending_requests());
+    ASSERT_FALSE(completed_fetcher_);
+  }
+
+  // A 4th failure doesn't trigger another retry, and propagates the error
+  // to the owner of the fetcher.
+  NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+  MessageLoop::current()->RunUntilIdle();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+
+  // And the owner of the fetcher gets the ERR_NETWORK_CHANGED error.
+  EXPECT_EQ(ERR_NETWORK_CHANGED, completed_fetcher_->GetStatus().error());
+}
+
+TEST_F(URLFetcherMockDNSTest, RetryOnNetworkChangedAndSucceed) {
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // This posts a task to start the fetcher.
+  CreateFetcher(test_url_);
+  fetcher_->SetAutomaticallyRetryOnNetworkChanges(3);
+  fetcher_->Start();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now running, but is pending the host resolve.
+  EXPECT_EQ(1, GetNumFetcherCores());
+  EXPECT_TRUE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+
+  // Make it fail 3 times.
+  for (int i = 0; i < 3; ++i) {
+    // A network change notification aborts the connect job.
+    NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+    MessageLoop::current()->RunUntilIdle();
+
+    // But the fetcher retries automatically.
+    EXPECT_EQ(1, GetNumFetcherCores());
+    EXPECT_TRUE(resolver_.has_pending_requests());
+    ASSERT_FALSE(completed_fetcher_);
+  }
+
+  // Now let it succeed by resolving the pending request.
+  resolver_.ResolveAllPending();
+  MessageLoop::current()->Run();
+
+  // URLFetcherMockDNSTest::OnURLFetchComplete() will quit the loop.
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+
+  // This time the request succeeded.
+  EXPECT_EQ(OK, completed_fetcher_->GetStatus().error());
+  EXPECT_EQ(200, completed_fetcher_->GetResponseCode());
+}
+
+TEST_F(URLFetcherMockDNSTest, RetryAfterComingBackOnline) {
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+
+  // This posts a task to start the fetcher.
+  CreateFetcher(test_url_);
+  fetcher_->SetAutomaticallyRetryOnNetworkChanges(1);
+  fetcher_->Start();
+  EXPECT_EQ(0, GetNumFetcherCores());
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now running, but is pending the host resolve.
+  EXPECT_EQ(1, GetNumFetcherCores());
+  EXPECT_TRUE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+
+  // Make it fail by changing the connection type to offline.
+  EXPECT_EQ(NetworkChangeNotifier::CONNECTION_UNKNOWN,
+            NetworkChangeNotifier::GetConnectionType());
+  EXPECT_FALSE(NetworkChangeNotifier::IsOffline());
+  network_change_notifier_->SetCurrentConnectionType(
+      NetworkChangeNotifier::CONNECTION_NONE);
+  // This makes the connect job fail:
+  NetworkChangeNotifier::NotifyObserversOfIPAddressChangeForTests();
+  resolver_.ResolveAllPending();
+  MessageLoop::current()->RunUntilIdle();
+
+  // The fetcher is now waiting for the connection to become online again.
+  EXPECT_EQ(NetworkChangeNotifier::CONNECTION_NONE,
+            NetworkChangeNotifier::GetConnectionType());
+  EXPECT_TRUE(NetworkChangeNotifier::IsOffline());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_FALSE(completed_fetcher_);
+  // The core is still alive, but it dropped its request.
+  EXPECT_EQ(0, GetNumFetcherCores());
+
+  // It should retry once the connection is back.
+  network_change_notifier_->SetCurrentConnectionType(
+      NetworkChangeNotifier::CONNECTION_WIFI);
+  MessageLoop::current()->RunUntilIdle();
+  EXPECT_EQ(1, GetNumFetcherCores());
+  ASSERT_FALSE(completed_fetcher_);
+  EXPECT_TRUE(resolver_.has_pending_requests());
+
+  // Resolve the pending request; the fetcher should complete now.
+  resolver_.ResolveAllPending();
+  MessageLoop::current()->Run();
+
+  EXPECT_EQ(0, GetNumFetcherCores());
+  EXPECT_FALSE(resolver_.has_pending_requests());
+  ASSERT_TRUE(completed_fetcher_);
+  EXPECT_EQ(OK, completed_fetcher_->GetStatus().error());
+  EXPECT_EQ(200, completed_fetcher_->GetResponseCode());
+}
+
 #if defined(OS_MACOSX)
 // SIGSEGV on Mac: http://crbug.com/60426
 TEST_F(URLFetcherPostTest, DISABLED_Basic) {
 #else
 TEST_F(URLFetcherPostTest, Basic) {
 #endif
   TestServer test_server(TestServer::TYPE_HTTP,
                          TestServer::kLocalhost,
                          FilePath(kDocRoot));
   ASSERT_TRUE(test_server.Start());
@@ skipping 430 matching lines @@
   MessageLoop::current()->Run();  // OnURLFetchComplete() will Quit().
 
   MessageLoop::current()->RunUntilIdle();
   ASSERT_FALSE(file_util::PathExists(file_path_))
       << file_path_.value() << " not removed.";
 }
 
 }  // namespace
 
 }  // namespace net