Close idle connections / SPDY sessions when needed

net/socket/client_socket_pool_base_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/socket/client_socket_pool_base.h"
 
+#include <vector>
+
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/compiler_specific.h"
 #include "base/memory/ref_counted.h"
 #include "base/memory/scoped_vector.h"
 #include "base/memory/weak_ptr.h"
 #include "base/message_loop.h"
 #include "base/stringprintf.h"
 #include "base/string_number_conversions.h"
 #include "base/threading/platform_thread.h"
 #include "base/values.h"
 #include "net/base/net_errors.h"
 #include "net/base/net_log.h"
 #include "net/base/net_log_unittest.h"
 #include "net/base/request_priority.h"
 #include "net/base/test_completion_callback.h"
 #include "net/http/http_response_headers.h"
 #include "net/socket/client_socket_factory.h"
 #include "net/socket/client_socket_handle.h"
 #include "net/socket/client_socket_pool_histograms.h"
 #include "net/socket/socket_test_util.h"
 #include "net/socket/ssl_host_info.h"
 #include "net/socket/stream_socket.h"
+#include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
+using ::testing::Invoke;
+using ::testing::Return;
+
 namespace net {
 
 namespace {
 
 const int kDefaultMaxSockets = 4;
 const int kDefaultMaxSocketsPerGroup = 2;
 const net::RequestPriority kDefaultPriority = MEDIUM;
 
 class TestSocketParams : public base::RefCounted<TestSocketParams> {
  public:
-  bool ignore_limits() { return false; }
+  TestSocketParams() : ignore_limits_(false) {}
+
+  void set_ignore_limits(bool ignore_limits) {
+    ignore_limits_ = ignore_limits;
+  }
+  bool ignore_limits() { return ignore_limits_; }
+
  private:
   friend class base::RefCounted<TestSocketParams>;
   ~TestSocketParams() {}
+
+  bool ignore_limits_;
 };
 typedef ClientSocketPoolBase<TestSocketParams> TestClientSocketPoolBase;
 
 class MockClientSocket : public StreamSocket {
  public:
   MockClientSocket() : connected_(false), was_used_to_convey_data_(false),
                        num_bytes_read_(0) {}
 
   // Socket implementation.
   virtual int Read(
@@ skipping 291 matching lines @@
   bool store_additional_error_state_;
 
   DISALLOW_COPY_AND_ASSIGN(TestConnectJob);
 };
 
 class TestConnectJobFactory
     : public TestClientSocketPoolBase::ConnectJobFactory {
  public:
   explicit TestConnectJobFactory(MockClientSocketFactory* client_socket_factory)
       : job_type_(TestConnectJob::kMockJob),
+        job_types_(NULL),
         client_socket_factory_(client_socket_factory) {}
 
   virtual ~TestConnectJobFactory() {}
 
   void set_job_type(TestConnectJob::JobType job_type) { job_type_ = job_type; }
 
+  void set_job_types(std::vector<TestConnectJob::JobType>* job_types) {
+    job_types_ = job_types;
+    CHECK(!job_types_->empty());
+  }
+
   void set_timeout_duration(base::TimeDelta timeout_duration) {
     timeout_duration_ = timeout_duration;
   }
 
   // ConnectJobFactory implementation.
 
   virtual ConnectJob* NewConnectJob(
       const std::string& group_name,
       const TestClientSocketPoolBase::Request& request,
       ConnectJob::Delegate* delegate) const {
-    return new TestConnectJob(job_type_,
+    TestConnectJob::JobType job_type = job_type_;

[mmenke, 2012/03/19 18:43:35]

nit:  Maybe an EXPECT_TRUE(!job_types_ || !job_types_->empty());?

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.

+    if (job_types_ && !job_types_->empty()) {
+      job_type = job_types_->back();
+      job_types_->pop_back();

[mmenke, 2012/03/19 18:43:35]

Think going front to back is much clearer.  Wasn't able to figure out how this
test worked until I realized the second failing job you push onto the vector is
for the third socket request, not the second.

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.

+    }
+    return new TestConnectJob(job_type,
                               group_name,
                               request,
                               timeout_duration_,
                               delegate,
                               client_socket_factory_,
                               NULL);
   }
 
   virtual base::TimeDelta ConnectionTimeout() const {
     return timeout_duration_;
   }
 
  private:
   TestConnectJob::JobType job_type_;
+  std::vector<TestConnectJob::JobType>* job_types_;
   base::TimeDelta timeout_duration_;
   MockClientSocketFactory* const client_socket_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(TestConnectJobFactory);
 };
 
 class TestClientSocketPool : public ClientSocketPool {
  public:
   TestClientSocketPool(
       int max_sockets,
@@ skipping 41 matching lines @@
       const std::string& group_name,
       StreamSocket* socket,
       int id) OVERRIDE {
     base_.ReleaseSocket(group_name, socket, id);
   }
 
   virtual void Flush() OVERRIDE {
     base_.Flush();
   }
 
+  virtual bool IsStalled() const OVERRIDE {
+    return base_.IsStalled();
+  }
+
   virtual void CloseIdleSockets() OVERRIDE {
     base_.CloseIdleSockets();
   }
 
   virtual int IdleSocketCount() const OVERRIDE {
     return base_.idle_socket_count();
   }
 
   virtual int IdleSocketCountInGroup(
       const std::string& group_name) const OVERRIDE {
     return base_.IdleSocketCountInGroup(group_name);
   }
 
   virtual LoadState GetLoadState(
       const std::string& group_name,
       const ClientSocketHandle* handle) const OVERRIDE {
     return base_.GetLoadState(group_name, handle);
   }
 
+  virtual void AddLayeredPool(LayeredPool* pool) OVERRIDE {
+    base_.AddLayeredPool(pool);
+  }
+
+  virtual void RemoveLayeredPool(LayeredPool* pool) OVERRIDE {
+    base_.RemoveLayeredPool(pool);
+  }
+
   virtual DictionaryValue* GetInfoAsValue(
       const std::string& name,
       const std::string& type,
       bool include_nested_pools) const OVERRIDE {
     return base_.GetInfoAsValue(name, type);
   }
 
   virtual base::TimeDelta ConnectionTimeout() const OVERRIDE {
     return base_.ConnectionTimeout();
   }
@@ skipping 13 matching lines @@
   }
 
   bool HasGroup(const std::string& group_name) const {
     return base_.HasGroup(group_name);
   }
 
   void CleanupTimedOutIdleSockets() { base_.CleanupIdleSockets(false); }
 
   void EnableConnectBackupJobs() { base_.EnableConnectBackupJobs(); }
 
+  bool CloseOneIdleConnectionInLayeredPool() {
+    return base_.CloseOneIdleConnectionInLayeredPool();
+  }
+
  private:
   TestClientSocketPoolBase base_;
 
   DISALLOW_COPY_AND_ASSIGN(TestClientSocketPool);
 };
 
 }  // namespace
 
 REGISTER_SOCKET_PARAMS_FOR_POOL(TestClientSocketPool, TestSocketParams);
 
@@ skipping 645 matching lines @@
   EXPECT_EQ(1, pool_->IdleSocketCount());
 }
 
 TEST_F(ClientSocketPoolBaseTest, WaitForStalledSocketAtSocketLimit) {
   CreatePool(kDefaultMaxSockets, kDefaultMaxSocketsPerGroup);
   connect_job_factory_->set_job_type(TestConnectJob::kMockJob);
 
   ClientSocketHandle stalled_handle;
   TestCompletionCallback callback;
   {
+    EXPECT_FALSE(pool_->IsStalled());
     ClientSocketHandle handles[kDefaultMaxSockets];
     for (int i = 0; i < kDefaultMaxSockets; ++i) {
       TestCompletionCallback callback;
       EXPECT_EQ(OK, handles[i].Init(base::StringPrintf(
           "Take 2: %d", i),
           params_,
           kDefaultPriority,
           callback.callback(),
           pool_.get(),
           BoundNetLog()));
     }
 
     EXPECT_EQ(kDefaultMaxSockets, client_socket_factory_.allocation_count());
     EXPECT_EQ(0, pool_->IdleSocketCount());
+    EXPECT_FALSE(pool_->IsStalled());
 
     // Now we will hit the socket limit.
     EXPECT_EQ(ERR_IO_PENDING, stalled_handle.Init("foo",
                                                   params_,
                                                   kDefaultPriority,
                                                   callback.callback(),
                                                   pool_.get(),
                                                   BoundNetLog()));
+    EXPECT_TRUE(pool_->IsStalled());
 
     // Dropping out of scope will close all handles and return them to idle.
   }
 
   // But if we wait for it, the released idle sockets will be closed in
   // preference of the waiting request.
   EXPECT_EQ(OK, callback.WaitForResult());
 
   EXPECT_EQ(kDefaultMaxSockets + 1, client_socket_factory_.allocation_count());
   EXPECT_EQ(3, pool_->IdleSocketCount());
@@ skipping 799 matching lines @@
   // job.  Release the socket.  Run the loop again to make sure the second
   // socket is sitting idle and the first one is released (since ReleaseSocket()
   // just posts a DoReleaseSocket() task).
 
   handle.Reset();
   EXPECT_EQ(OK, callback2.WaitForResult());
   // Use the socket.
   EXPECT_EQ(1, handle2.socket()->Write(NULL, 1, CompletionCallback()));
   handle2.Reset();
 
-  // The idle socket timeout value was set to 10 milliseconds. Wait 20
+  // The idle socket timeout value was set to 10 milliseconds. Wait 100
   // milliseconds so the sockets timeout.
-  base::PlatformThread::Sleep(base::TimeDelta::FromMilliseconds(20));
+  base::PlatformThread::Sleep(base::TimeDelta::FromMilliseconds(100));
   MessageLoop::current()->RunAllPending();
 
   ASSERT_EQ(2, pool_->IdleSocketCount());
 
   // Request a new socket. This should cleanup the unused and timed out ones.
   // A new socket will be created rather than reusing the idle one.
   CapturingBoundNetLog log(CapturingNetLog::kUnbounded);
   TestCompletionCallback callback3;
   rv = handle.Init("a",
                    params_,
@@ skipping 1011 matching lines @@
   CreatePool(kDefaultMaxSockets, kDefaultMaxSockets);
   connect_job_factory_->set_job_type(TestConnectJob::kMockPendingJob);
 
   ASSERT_FALSE(pool_->HasGroup("a"));
 
   pool_->RequestSockets("a", &params_, kDefaultMaxSockets - 1,
                         BoundNetLog());
 
   ASSERT_TRUE(pool_->HasGroup("a"));
   EXPECT_EQ(kDefaultMaxSockets - 1, pool_->NumConnectJobsInGroup("a"));
+  EXPECT_FALSE(pool_->IsStalled());
 
   ASSERT_FALSE(pool_->HasGroup("b"));
 
   pool_->RequestSockets("b", &params_, kDefaultMaxSockets,
                         BoundNetLog());
 
   ASSERT_TRUE(pool_->HasGroup("b"));
   EXPECT_EQ(1, pool_->NumConnectJobsInGroup("b"));
+  EXPECT_FALSE(pool_->IsStalled());
 }
 
 TEST_F(ClientSocketPoolBaseTest, RequestSocketsCountIdleSockets) {
   CreatePool(4, 4);
   connect_job_factory_->set_job_type(TestConnectJob::kMockPendingJob);
 
   ClientSocketHandle handle1;
   TestCompletionCallback callback1;
   EXPECT_EQ(ERR_IO_PENDING, handle1.Init("a",
                                          params_,
@@ skipping 298 matching lines @@
   EXPECT_EQ(0, pool_->NumActiveSocketsInGroup("a"));
   ASSERT_EQ(OK, callback.WaitForResult());
 
   // The hung connect job should still be there, but everything else should be
   // complete.
   EXPECT_EQ(1, pool_->NumConnectJobsInGroup("a"));
   EXPECT_EQ(0, pool_->IdleSocketCountInGroup("a"));
   EXPECT_EQ(1, pool_->NumActiveSocketsInGroup("a"));
 }
 
+class MockLayeredPool : public LayeredPool {
+ public:
+  MockLayeredPool(TestClientSocketPool* pool,
+                  const std::string& group_name)
+      : pool_(pool),
+        params_(new TestSocketParams),
+        group_name_(group_name),
+        can_release_connection_(true) {
+    pool_->AddLayeredPool(this);
+  }
+
+  ~MockLayeredPool() {
+    pool_->RemoveLayeredPool(this);
+  }
+
+  int RequestSocket(TestClientSocketPool* pool) {
+    return handle_.Init(group_name_, params_, kDefaultPriority,
+                        callback_.callback(), pool, BoundNetLog());
+  }
+
+  int RequestSocketWithoutLimits(TestClientSocketPool* pool) {
+    params_->set_ignore_limits(true);
+    return handle_.Init(group_name_, params_, kDefaultPriority,
+                        callback_.callback(), pool, BoundNetLog());
+  }
+
+  bool ReleaseOneConnection() {
+    if (!handle_.is_initialized() || !can_release_connection_) {
+      return false;
+    }
+    handle_.socket()->Disconnect();
+    handle_.Reset();
+    return true;
+  }
+
+  void set_can_release_connection(bool can_release_connection) {
+    can_release_connection_ = can_release_connection;
+  }
+
+  MOCK_METHOD0(CloseOneIdleConnection, bool());
+
+ private:
+  TestClientSocketPool* const pool_;
+  scoped_refptr<TestSocketParams> params_;
+  ClientSocketHandle handle_;
+  TestCompletionCallback callback_;
+  const std::string group_name_;
+  bool can_release_connection_;
+};
+
+TEST_F(ClientSocketPoolBaseTest, FailToCloseIdleSocketsNotHeldByLayeredPool) {
+  CreatePool(kDefaultMaxSockets, kDefaultMaxSocketsPerGroup);
+  connect_job_factory_->set_job_type(TestConnectJob::kMockJob);
+
+  MockLayeredPool mock_layered_pool(pool_.get(), "foo");
+  EXPECT_CALL(mock_layered_pool, CloseOneIdleConnection())
+      .WillOnce(Return(false));

[mmenke, 2012/03/19 18:43:35]

nit:  To make this test more closely match the next one, suggest moving the
EXPECT_CALL down one line.

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.

+  EXPECT_EQ(OK, mock_layered_pool.RequestSocket(pool_.get()));
+  EXPECT_FALSE(pool_->CloseOneIdleConnectionInLayeredPool());
+}
+
+TEST_F(ClientSocketPoolBaseTest, ForciblyCloseIdleSocketsHeldByLayeredPool) {
+  CreatePool(kDefaultMaxSockets, kDefaultMaxSocketsPerGroup);
+  connect_job_factory_->set_job_type(TestConnectJob::kMockJob);
+
+  MockLayeredPool mock_layered_pool(pool_.get(), "foo");
+  EXPECT_EQ(OK, mock_layered_pool.RequestSocket(pool_.get()));
+  EXPECT_CALL(mock_layered_pool, CloseOneIdleConnection())
+      .WillOnce(Invoke(&mock_layered_pool,
+                       &MockLayeredPool::ReleaseOneConnection));
+  EXPECT_TRUE(pool_->CloseOneIdleConnectionInLayeredPool());
+}
+
+// This test exercises the codepath which caused http://crbug.com/109876
+TEST_F(ClientSocketPoolBaseTest,
+       CloseIdleSocketsHeldByLayeredPoolInSameGroupWhenNeeded) {
+  CreatePool(2, 2);
+  std::vector<TestConnectJob::JobType> job_types;
+  job_types.push_back(TestConnectJob::kMockJob);
+  job_types.push_back(TestConnectJob::kMockFailingJob);
+  job_types.push_back(TestConnectJob::kMockJob);
+  job_types.push_back(TestConnectJob::kMockJob);
+  connect_job_factory_->set_job_types(&job_types);
+
+  ClientSocketHandle handle1;
+  TestCompletionCallback callback1;
+  EXPECT_EQ(OK, handle1.Init("group1",
+                            params_,
+                            kDefaultPriority,
+                            callback1.callback(),
+                            pool_.get(),
+                            BoundNetLog()));

[mmenke, 2012/03/19 18:43:35]

nit:  Fix indent.

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.

+
+  MockLayeredPool mock_layered_pool(pool_.get(), "group2");
+  EXPECT_EQ(OK, mock_layered_pool.RequestSocket(pool_.get()));
+  EXPECT_CALL(mock_layered_pool, CloseOneIdleConnection())
+      .WillRepeatedly(Invoke(&mock_layered_pool,
+                       &MockLayeredPool::ReleaseOneConnection));

[mmenke, 2012/03/19 18:43:35]

nit:  -1 indent.

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.

+  mock_layered_pool.set_can_release_connection(false);
+
+  ClientSocketHandle handle2;

[mmenke, 2012/03/19 18:43:35]

Suggest you name these handle3 and handle4, so they match the |job_types|
vector.

[mmenke, 2012/03/19 18:43:35]

Suggest you add a comment here saying that this connection attempt will fail
when the next request causes the MockLayeredPool to delete the socket it's
holding.  And that this request is needed to trigger the destruction of the
"group2" Group.

Does seem a little weird that we're not deleting empty groups instantly.

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.  Good idea.

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.

+  TestCompletionCallback callback2;
+  EXPECT_EQ(ERR_IO_PENDING, handle2.Init("group2",
+                                         params_,
+                                         HIGHEST,

[mmenke, 2012/03/19 18:43:35]

I don't think this or the next request has to be at HIGHEST priority to trigger
the bug.

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.  Yes, you're right.  This was a leftover from an earlier attempt to
trigger the bug.

+                                         callback2.callback(),
+                                         pool_.get(),
+                                         BoundNetLog()));
+
+  mock_layered_pool.set_can_release_connection(true);
+  ClientSocketHandle handle3;
+  TestCompletionCallback callback3;
+  EXPECT_EQ(OK, handle3.Init("group2",
+                                         params_,
+                                         HIGHEST,
+                                         callback3.callback(),
+                                         pool_.get(),
+                                         BoundNetLog()));

[mmenke, 2012/03/19 18:43:35]

nit:  Fix indent.

[Ryan Hamilton, 2012/03/19 19:04:15]

Done.

+}
+
+TEST_F(ClientSocketPoolBaseTest, CloseIdleSocketsHeldByLayeredPoolWhenNeeded) {
+  CreatePool(1, 1);
+  connect_job_factory_->set_job_type(TestConnectJob::kMockJob);
+
+  MockLayeredPool mock_layered_pool(pool_.get(), "foo");
+  EXPECT_EQ(OK, mock_layered_pool.RequestSocket(pool_.get()));
+  EXPECT_CALL(mock_layered_pool, CloseOneIdleConnection())
+      .WillOnce(Invoke(&mock_layered_pool,
+                       &MockLayeredPool::ReleaseOneConnection));
+  ClientSocketHandle handle;
+  TestCompletionCallback callback;
+  EXPECT_EQ(OK, handle.Init("a",
+                            params_,
+                            kDefaultPriority,
+                            callback.callback(),
+                            pool_.get(),
+                            BoundNetLog()));
+}
+
+TEST_F(ClientSocketPoolBaseTest,
+       CloseMultipleIdleSocketsHeldByLayeredPoolWhenNeeded) {
+  CreatePool(1, 1);
+  connect_job_factory_->set_job_type(TestConnectJob::kMockJob);
+
+  MockLayeredPool mock_layered_pool1(pool_.get(), "foo");
+  EXPECT_EQ(OK, mock_layered_pool1.RequestSocket(pool_.get()));
+  EXPECT_CALL(mock_layered_pool1, CloseOneIdleConnection())
+      .WillRepeatedly(Invoke(&mock_layered_pool1,
+                             &MockLayeredPool::ReleaseOneConnection));
+  MockLayeredPool mock_layered_pool2(pool_.get(), "bar");
+  EXPECT_EQ(OK, mock_layered_pool2.RequestSocketWithoutLimits(pool_.get()));
+  EXPECT_CALL(mock_layered_pool2, CloseOneIdleConnection())
+      .WillRepeatedly(Invoke(&mock_layered_pool2,
+                             &MockLayeredPool::ReleaseOneConnection));
+  ClientSocketHandle handle;
+  TestCompletionCallback callback;
+  EXPECT_EQ(OK, handle.Init("a",
+                            params_,
+                            kDefaultPriority,
+                            callback.callback(),
+                            pool_.get(),
+                            BoundNetLog()));
+}
+
 }  // namespace
 
 }  // namespace net