Add the capability to run multiple proxy PAC scripts in parallel....

net/proxy/multi_threaded_proxy_resolver_unittest.cc

 // Copyright (c) 2009 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include "net/proxy/multi_threaded_proxy_resolver.h"
+
+#include "base/stl_util-inl.h"
 #include "base/string_util.h"
 #include "base/waitable_event.h"
 #include "googleurl/src/gurl.h"
 #include "net/base/net_log.h"
 #include "net/base/net_log_unittest.h"
 #include "net/base/net_errors.h"
 #include "net/base/test_completion_callback.h"
 #include "net/proxy/proxy_info.h"
-#include "net/proxy/single_threaded_proxy_resolver.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace net {
+
 namespace {
 
 // A synchronous mock ProxyResolver implementation, which can be used in
-// conjunction with SingleThreadedProxyResolver.
+// conjunction with MultiThreadedProxyResolver.
 //       - returns a single-item proxy list with the query's host.
 class MockProxyResolver : public ProxyResolver {
  public:
   MockProxyResolver()
       : ProxyResolver(true /*expects_pac_bytes*/),
         wrong_loop_(MessageLoop::current()),
         request_count_(0),
         purge_count_(0),
         resolve_latency_ms_(0) {}
 
@@ skipping 31 matching lines @@
     last_pac_script_ = text;
     return OK;
   }
 
   virtual void PurgeMemory() {
     CheckIsOnWorkerThread();
     ++purge_count_;
   }
 
   int purge_count() const { return purge_count_; }
+  int request_count() const { return request_count_; }
 
   const string16& last_pac_script() const { return last_pac_script_; }
 
   void SetResolveLatency(int latency_ms) {
     resolve_latency_ms_ = latency_ms;
   }
 
  private:
   void CheckIsOnWorkerThread() {
     // We should be running on the worker thread -- while we don't know the
-    // message loop of SingleThreadedProxyResolver's worker thread, we do
+    // message loop of MultiThreadedProxyResolver's worker thread, we do
     // know that it is going to be distinct from the loop running the
     // test, so at least make sure it isn't the main loop.
     EXPECT_NE(MessageLoop::current(), wrong_loop_);
   }
 
   MessageLoop* wrong_loop_;
   int request_count_;
   int purge_count_;
   string16 last_pac_script_;
   int resolve_latency_ms_;
@@ skipping 41 matching lines @@
     return MockProxyResolver::GetProxyForURL(
         query_url, results, callback, request, net_log);
   }
 
  private:
   bool should_block_;
   base::WaitableEvent unblocked_;
   base::WaitableEvent blocked_;
 };
 
-TEST(SingleThreadedProxyResolverTest, Basic) {
-  MockProxyResolver* mock = new MockProxyResolver;
-  SingleThreadedProxyResolver resolver(mock);
+// ForwardingProxyResolver forwards all requests to |impl|.
+class ForwardingProxyResolver : public ProxyResolver {
+ public:
+  explicit ForwardingProxyResolver(ProxyResolver* impl)
+      : ProxyResolver(impl->expects_pac_bytes()),
+        impl_(impl) {}
+
+  virtual int GetProxyForURL(const GURL& query_url,
+                             ProxyInfo* results,
+                             CompletionCallback* callback,
+                             RequestHandle* request,
+                             const BoundNetLog& net_log) {
+    return impl_->GetProxyForURL(
+        query_url, results, callback, request, net_log);
+  }
+
+  virtual void CancelRequest(RequestHandle request) {
+    impl_->CancelRequest(request);
+  }
+
+  virtual int SetPacScript(const GURL& pac_url,
+                           const string16& script,
+                           CompletionCallback* callback) {
+    if (impl_->expects_pac_bytes())
+      return impl_->SetPacScriptByData(script, callback);
+    else
+      return impl_->SetPacScriptByUrl(pac_url, callback);
+  }
+
+  virtual void PurgeMemory() {
+    impl_->PurgeMemory();
+  }
+
+ private:
+  ProxyResolver* impl_;
+};
+
+// This factory returns ProxyResolvers that forward all requests to
+// |resolver|.
+class ForwardingProxyResolverFactory : public ProxyResolverFactory {
+ public:
+  explicit ForwardingProxyResolverFactory(ProxyResolver* resolver)
+      : ProxyResolverFactory(resolver->expects_pac_bytes()),
+        resolver_(resolver) {}
+
+  virtual ProxyResolver* CreateProxyResolver() {
+    return new ForwardingProxyResolver(resolver_);
+  }
+
+ private:
+  ProxyResolver* resolver_;
+};
+
+// This factory returns new instances of BlockableProxyResolver.
+class BlockableProxyResolverFactory : public ProxyResolverFactory {
+ public:
+  BlockableProxyResolverFactory() : ProxyResolverFactory(true) {}
+
+  ~BlockableProxyResolverFactory() {
+    STLDeleteElements(&resolvers_);
+  }
+
+  virtual ProxyResolver* CreateProxyResolver() {
+    BlockableProxyResolver* resolver = new BlockableProxyResolver;
+    resolvers_.push_back(resolver);
+    return new ForwardingProxyResolver(resolver);
+  }
+
+  std::vector<BlockableProxyResolver*> resolvers() {
+    return resolvers_;
+  }
+
+ private:
+  std::vector<BlockableProxyResolver*> resolvers_;
+};
+
+TEST(MultiThreadedProxyResolverTest, SingleThread_Basic) {
+  const size_t kNumThreads = 1u;
+  scoped_ptr<MockProxyResolver> mock(new MockProxyResolver);
+  MultiThreadedProxyResolver resolver(
+      new ForwardingProxyResolverFactory(mock.get()), kNumThreads);
 
   int rv;
 
   EXPECT_TRUE(resolver.expects_pac_bytes());
 
   // Call SetPacScriptByData() -- verify that it reaches the synchronous
   // resolver.
   TestCompletionCallback set_script_callback;
   rv = resolver.SetPacScriptByData(ASCIIToUTF16("pac script bytes"),
                                    &set_script_callback);
   EXPECT_EQ(ERR_IO_PENDING, rv);
   EXPECT_EQ(OK, set_script_callback.WaitForResult());
   EXPECT_EQ(ASCIIToUTF16("pac script bytes"), mock->last_pac_script());
 
   // Start request 0.
   TestCompletionCallback callback0;
   CapturingBoundNetLog log0(CapturingNetLog::kUnbounded);
   ProxyInfo results0;
   rv = resolver.GetProxyForURL(
       GURL("http://request0"), &results0, &callback0, NULL, log0.bound());
   EXPECT_EQ(ERR_IO_PENDING, rv);
 
   // Wait for request 0 to finish.
   rv = callback0.WaitForResult();
   EXPECT_EQ(0, rv);
   EXPECT_EQ("PROXY request0:80", results0.ToPacString());
 
   // The mock proxy resolver should have written 1 log entry. And
   // on completion, this should have been copied into |log0|.
-  EXPECT_EQ(1u, log0.entries().size());
+  // We also have 1 log entry that was emitted by the
+  // MultiThreadedProxyResolver.
+  ASSERT_EQ(2u, log0.entries().size());
+  EXPECT_EQ(NetLog::TYPE_SUBMITTED_TO_RESOLVER_THREAD,
+            log0.entries()[0].type);
 
   // Start 3 more requests (request1 to request3).
 
   TestCompletionCallback callback1;
   ProxyInfo results1;
   rv = resolver.GetProxyForURL(
       GURL("http://request1"), &results1, &callback1, NULL, BoundNetLog());
   EXPECT_EQ(ERR_IO_PENDING, rv);
 
   TestCompletionCallback callback2;
@@ skipping 31 matching lines @@
   // we queue up a dummy request after the PurgeMemory() call and wait until it
   // finishes to ensure PurgeMemory() has had a chance to run.
   TestCompletionCallback dummy_callback;
   rv = resolver.SetPacScriptByData(ASCIIToUTF16("dummy"), &dummy_callback);
   EXPECT_EQ(OK, dummy_callback.WaitForResult());
   EXPECT_EQ(1, mock->purge_count());
 }
 
 // Tests that the NetLog is updated to include the time the request was waiting
 // to be scheduled to a thread.
-TEST(SingleThreadedProxyResolverTest, UpdatesNetLogWithThreadWait) {
-  BlockableProxyResolver* mock = new BlockableProxyResolver;
-  SingleThreadedProxyResolver resolver(mock);
+TEST(MultiThreadedProxyResolverTest,
+     SingleThread_UpdatesNetLogWithThreadWait) {
+  const size_t kNumThreads = 1u;
+  scoped_ptr<BlockableProxyResolver> mock(new BlockableProxyResolver);
+  MultiThreadedProxyResolver resolver(
+      new ForwardingProxyResolverFactory(mock.get()), kNumThreads);
 
   int rv;
 
+  // Initialize the resolver.
+  TestCompletionCallback init_callback;
+  rv = resolver.SetPacScriptByData(ASCIIToUTF16("foo"), &init_callback);
+  EXPECT_EQ(OK, init_callback.WaitForResult());
+
   // Block the proxy resolver, so no request can complete.
   mock->Block();
 
   // Start request 0.
   ProxyResolver::RequestHandle request0;
   TestCompletionCallback callback0;
   ProxyInfo results0;
   CapturingBoundNetLog log0(CapturingNetLog::kUnbounded);
   rv = resolver.GetProxyForURL(
       GURL("http://request0"), &results0, &callback0, &request0, log0.bound());
@@ skipping 14 matching lines @@
   CapturingBoundNetLog log2(CapturingNetLog::kUnbounded);
   rv = resolver.GetProxyForURL(
       GURL("http://request2"), &results2, &callback2, &request2, log2.bound());
   EXPECT_EQ(ERR_IO_PENDING, rv);
 
   // Unblock the worker thread so the requests can continue running.
   mock->WaitUntilBlocked();
   mock->Unblock();
 
   // Check that request 0 completed as expected.
-  // The NetLog only has 1 entry (that came from the mock proxy resolver.)
+  // The NetLog has 1 entry that came from the MultiThreadedProxyResolver, and
+  // 1 entry from the mock proxy resolver.
   EXPECT_EQ(0, callback0.WaitForResult());
   EXPECT_EQ("PROXY request0:80", results0.ToPacString());
-  ASSERT_EQ(1u, log0.entries().size());
+  ASSERT_EQ(2u, log0.entries().size());
+  EXPECT_EQ(NetLog::TYPE_SUBMITTED_TO_RESOLVER_THREAD,
+            log0.entries()[0].type);
 
   // Check that request 1 completed as expected.
   EXPECT_EQ(1, callback1.WaitForResult());
   EXPECT_EQ("PROXY request1:80", results1.ToPacString());
-  ASSERT_EQ(3u, log1.entries().size());
+  ASSERT_EQ(4u, log1.entries().size());
   EXPECT_TRUE(LogContainsBeginEvent(
       log1.entries(), 0,
-      NetLog::TYPE_WAITING_FOR_SINGLE_PROXY_RESOLVER_THREAD));
+      NetLog::TYPE_WAITING_FOR_PROXY_RESOLVER_THREAD));
   EXPECT_TRUE(LogContainsEndEvent(
       log1.entries(), 1,
-      NetLog::TYPE_WAITING_FOR_SINGLE_PROXY_RESOLVER_THREAD));
+      NetLog::TYPE_WAITING_FOR_PROXY_RESOLVER_THREAD));
 
   // Check that request 2 completed as expected.
   EXPECT_EQ(2, callback2.WaitForResult());
   EXPECT_EQ("PROXY request2:80", results2.ToPacString());
-  ASSERT_EQ(3u, log2.entries().size());
+  ASSERT_EQ(4u, log2.entries().size());
   EXPECT_TRUE(LogContainsBeginEvent(
       log2.entries(), 0,
-      NetLog::TYPE_WAITING_FOR_SINGLE_PROXY_RESOLVER_THREAD));
+      NetLog::TYPE_WAITING_FOR_PROXY_RESOLVER_THREAD));
   EXPECT_TRUE(LogContainsEndEvent(
       log2.entries(), 1,
-      NetLog::TYPE_WAITING_FOR_SINGLE_PROXY_RESOLVER_THREAD));
+      NetLog::TYPE_WAITING_FOR_PROXY_RESOLVER_THREAD));
 }
 
 // Cancel a request which is in progress, and then cancel a request which
 // is pending.
-TEST(SingleThreadedProxyResolverTest, CancelRequest) {
-  BlockableProxyResolver* mock = new BlockableProxyResolver;
-  SingleThreadedProxyResolver resolver(mock);
+TEST(MultiThreadedProxyResolverTest, SingleThread_CancelRequest) {
+  const size_t kNumThreads = 1u;
+  scoped_ptr<BlockableProxyResolver> mock(new BlockableProxyResolver);
+  MultiThreadedProxyResolver resolver(
+      new ForwardingProxyResolverFactory(mock.get()),
+                                      kNumThreads);
 
   int rv;
 
+  // Initialize the resolver.
+  TestCompletionCallback init_callback;
+  rv = resolver.SetPacScriptByData(ASCIIToUTF16("foo"), &init_callback);
+  EXPECT_EQ(OK, init_callback.WaitForResult());
+
   // Block the proxy resolver, so no request can complete.
   mock->Block();
 
   // Start request 0.
   ProxyResolver::RequestHandle request0;
   TestCompletionCallback callback0;
   ProxyInfo results0;
   rv = resolver.GetProxyForURL(
       GURL("http://request0"), &results0, &callback0, &request0, BoundNetLog());
   EXPECT_EQ(ERR_IO_PENDING, rv);
@@ skipping 40 matching lines @@
   // the request count is 2 and not 3 here.
   EXPECT_EQ(2, rv);
   EXPECT_EQ("PROXY request3:80", results3.ToPacString());
 
   // Requests 0 and 2 which were cancelled, hence their completion callbacks
   // were never summoned.
   EXPECT_FALSE(callback0.have_result());
   EXPECT_FALSE(callback2.have_result());
 }
 
-// Test that deleting SingleThreadedProxyResolver while requests are
+// Test that deleting MultiThreadedProxyResolver while requests are
 // outstanding cancels them (and doesn't leak anything).
-TEST(SingleThreadedProxyResolverTest, CancelRequestByDeleting) {
-  BlockableProxyResolver* mock = new BlockableProxyResolver;
-  scoped_ptr<SingleThreadedProxyResolver> resolver(
-      new SingleThreadedProxyResolver(mock));
+TEST(MultiThreadedProxyResolverTest, SingleThread_CancelRequestByDeleting) {
+  const size_t kNumThreads = 1u;
+  scoped_ptr<BlockableProxyResolver> mock(new BlockableProxyResolver);
+  scoped_ptr<MultiThreadedProxyResolver> resolver(
+      new MultiThreadedProxyResolver(
+          new ForwardingProxyResolverFactory(mock.get()), kNumThreads));
 
   int rv;
 
+  // Initialize the resolver.
+  TestCompletionCallback init_callback;
+  rv = resolver->SetPacScriptByData(ASCIIToUTF16("foo"), &init_callback);
+  EXPECT_EQ(OK, init_callback.WaitForResult());
+
   // Block the proxy resolver, so no request can complete.
   mock->Block();
 
   // Start 3 requests.
 
   TestCompletionCallback callback0;
   ProxyInfo results0;
   rv = resolver->GetProxyForURL(
       GURL("http://request0"), &results0, &callback0, NULL, BoundNetLog());
   EXPECT_EQ(ERR_IO_PENDING, rv);
 
   TestCompletionCallback callback1;
   ProxyInfo results1;
   rv = resolver->GetProxyForURL(
       GURL("http://request1"), &results1, &callback1, NULL, BoundNetLog());
   EXPECT_EQ(ERR_IO_PENDING, rv);
 
   TestCompletionCallback callback2;
   ProxyInfo results2;
   rv = resolver->GetProxyForURL(
       GURL("http://request2"), &results2, &callback2, NULL, BoundNetLog());
   EXPECT_EQ(ERR_IO_PENDING, rv);
 
   // Wait until request 0 reaches the worker thread.
   mock->WaitUntilBlocked();
 
   // Add some latency, to improve the chance that when
-  // SingleThreadedProxyResolver is deleted below we are still running inside
+  // MultiThreadedProxyResolver is deleted below we are still running inside
   // of the worker thread. The test will pass regardless, so this race doesn't
   // cause flakiness. However the destruction during execution is a more
   // interesting case to test.
   mock->SetResolveLatency(100);
 
   // Unblock the worker thread and delete the underlying
-  // SingleThreadedProxyResolver immediately.
+  // MultiThreadedProxyResolver immediately.
   mock->Unblock();
   resolver.reset();
 
   // Give any posted tasks a chance to run (in case there is badness).
   MessageLoop::current()->RunAllPending();
 
   // Check that none of the outstanding requests were completed.
   EXPECT_FALSE(callback0.have_result());
   EXPECT_FALSE(callback1.have_result());
   EXPECT_FALSE(callback2.have_result());
 }
 
 // Cancel an outstanding call to SetPacScriptByData().
-TEST(SingleThreadedProxyResolverTest, CancelSetPacScript) {
-  BlockableProxyResolver* mock = new BlockableProxyResolver;
-  SingleThreadedProxyResolver resolver(mock);
+TEST(MultiThreadedProxyResolverTest, SingleThread_CancelSetPacScript) {
+  const size_t kNumThreads = 1u;
+  scoped_ptr<BlockableProxyResolver> mock(new BlockableProxyResolver);
+  MultiThreadedProxyResolver resolver(
+      new ForwardingProxyResolverFactory(mock.get()), kNumThreads);
 
   int rv;
 
-  // Block the proxy resolver, so no request can complete.
-  mock->Block();
-
-  // Start request 0.
-  ProxyResolver::RequestHandle request0;
-  TestCompletionCallback callback0;
-  ProxyInfo results0;
-  rv = resolver.GetProxyForURL(
-      GURL("http://request0"), &results0, &callback0, &request0, BoundNetLog());
-  EXPECT_EQ(ERR_IO_PENDING, rv);
-
-  // Wait until requests 0 reaches the worker thread.
-  mock->WaitUntilBlocked();
-
   TestCompletionCallback set_pac_script_callback;
   rv = resolver.SetPacScriptByData(ASCIIToUTF16("data"),
                                    &set_pac_script_callback);
   EXPECT_EQ(ERR_IO_PENDING, rv);
 
-  // Cancel the SetPacScriptByData request (it can't have finished yet,
-  // since the single-thread is currently blocked).
+  // Cancel the SetPacScriptByData request.
   resolver.CancelSetPacScript();
 
-  // Start 1 more request.
-
-  TestCompletionCallback callback1;
-  ProxyInfo results1;
+  // Start another SetPacScript request
+  TestCompletionCallback set_pac_script_callback2;
+  rv = resolver.SetPacScriptByData(ASCIIToUTF16("data2"),
+                                   &set_pac_script_callback2);
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+
+  // Wait for the initialization to complete.
+
+  rv = set_pac_script_callback2.WaitForResult();
+  EXPECT_EQ(0, rv);
+  EXPECT_EQ(ASCIIToUTF16("data2"), mock->last_pac_script());
+
+  // The first SetPacScript callback should never have been completed.
+  EXPECT_FALSE(set_pac_script_callback.have_result());
+}
+
+// Tests setting the PAC script once, lazily creating new threads, and
+// cancelling requests.
+TEST(MultiThreadedProxyResolverTest, ThreeThreads_Basic) {
+  const size_t kNumThreads = 3u;
+  BlockableProxyResolverFactory* factory = new BlockableProxyResolverFactory;
+  MultiThreadedProxyResolver resolver(factory, kNumThreads);
+
+  int rv;
+
+  EXPECT_TRUE(resolver.expects_pac_bytes());
+
+  // Call SetPacScriptByData() -- verify that it reaches the synchronous
+  // resolver.
+  TestCompletionCallback set_script_callback;
+  rv = resolver.SetPacScriptByData(ASCIIToUTF16("pac script bytes"),
+                                   &set_script_callback);
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+  EXPECT_EQ(OK, set_script_callback.WaitForResult());
+  // One thread has been provisioned (i.e. one ProxyResolver was created).
+  ASSERT_EQ(1u, factory->resolvers().size());
+  EXPECT_EQ(ASCIIToUTF16("pac script bytes"),
+            factory->resolvers()[0]->last_pac_script());
+
+  const int kNumRequests = 9;
+  TestCompletionCallback callback[kNumRequests];
+  ProxyInfo results[kNumRequests];
+  ProxyResolver::RequestHandle request[kNumRequests];
+
+  // Start request 0 -- this should run on thread 0 as there is nothing else
+  // going on right now.
   rv = resolver.GetProxyForURL(
-      GURL("http://request1"), &results1, &callback1, NULL, BoundNetLog());
-  EXPECT_EQ(ERR_IO_PENDING, rv);
-
-  // Unblock the worker thread so the requests can continue running.
-  mock->Unblock();
-
-  // Wait for requests 0 and 1 to finish.
-
-  rv = callback0.WaitForResult();
+      GURL("http://request0"), &results[0], &callback[0], &request[0],
+      BoundNetLog());
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+
+  // Wait for request 0 to finish.
+  rv = callback[0].WaitForResult();
   EXPECT_EQ(0, rv);
-  EXPECT_EQ("PROXY request0:80", results0.ToPacString());
-
-  rv = callback1.WaitForResult();
-  EXPECT_EQ(1, rv);
-  EXPECT_EQ("PROXY request1:80", results1.ToPacString());
-
-  // The SetPacScript callback should never have been completed.
-  EXPECT_FALSE(set_pac_script_callback.have_result());
+  EXPECT_EQ("PROXY request0:80", results[0].ToPacString());
+  ASSERT_EQ(1u, factory->resolvers().size());
+  EXPECT_EQ(1, factory->resolvers()[0]->request_count());
+
+  MessageLoop::current()->RunAllPending();
+
+  // We now start 8 requests in parallel -- this will cause the maximum of
+  // three threads to be provisioned (an additional two from what we already
+  // have).
+
+  for (int i = 1; i < kNumRequests; ++i) {
+    rv = resolver.GetProxyForURL(
+        GURL(StringPrintf("http://request%d", i)), &results[i], &callback[i],
+        &request[i], BoundNetLog());
+    EXPECT_EQ(ERR_IO_PENDING, rv);
+  }
+
+  // We should now have a total of 3 threads, each with its own ProxyResolver
+  // that will get initialized with the same data. (We check this later since
+  // the assignment happens on the worker threads and may not have occurred
+  // yet.)
+  ASSERT_EQ(3u, factory->resolvers().size());
+
+  // Cancel 3 of the 8 oustanding requests.
+  resolver.CancelRequest(request[1]);
+  resolver.CancelRequest(request[3]);
+  resolver.CancelRequest(request[6]);
+
+  // Wait for the remaining requests to complete.
+  int kNonCancelledRequests[] = {2, 4, 5, 7, 8};
+  for (size_t i = 0; i < arraysize(kNonCancelledRequests); ++i) {
+    int request_index = kNonCancelledRequests[i];
+    EXPECT_GE(callback[request_index].WaitForResult(), 0);
+  }
+
+  // Check that the cancelled requests never invoked their callback.
+  EXPECT_FALSE(callback[1].have_result());
+  EXPECT_FALSE(callback[3].have_result());
+  EXPECT_FALSE(callback[6].have_result());
+
+  // We call SetPacScript again, solely to stop the current worker threads.
+  // (That way we can test to see the values observed by the synchronous
+  // resolvers in a non-racy manner).
+  TestCompletionCallback set_script_callback2;
+  rv = resolver.SetPacScriptByData(ASCIIToUTF16("xyz"), &set_script_callback2);
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+  EXPECT_EQ(OK, set_script_callback2.WaitForResult());
+  ASSERT_EQ(4u, factory->resolvers().size());
+
+  for (int i = 0; i < 3; ++i) {
+    EXPECT_EQ(ASCIIToUTF16("pac script bytes"),
+              factory->resolvers()[i]->last_pac_script()) << "i=" << i;
+  }
+
+  EXPECT_EQ(ASCIIToUTF16("xyz"),
+            factory->resolvers()[3]->last_pac_script());
+
+  // We don't know the exact ordering that requests ran on threads with,
+  // but we do know the total count that should have reached the threads.
+  // 8 total were submitted, and three were cancelled. Of the three that
+  // were cancelled, one of them (request 1) was cancelled after it had
+  // already been posted to the worker thread. So the resolvers will
+  // have seen 6 total (and 1 from the run prior).
+  ASSERT_EQ(4u, factory->resolvers().size());
+  int total_count = 0;
+  for (int i = 0; i < 3; ++i) {
+    total_count += factory->resolvers()[i]->request_count();
+  }
+  EXPECT_EQ(7, total_count);
+}
+
+// Tests using two threads. The first request hangs the first thread. Checks
+// that other requests are able to complete while this first request remains
+// stalled.
+TEST(MultiThreadedProxyResolverTest, OneThreadBlocked) {
+  const size_t kNumThreads = 2u;
+  BlockableProxyResolverFactory* factory = new BlockableProxyResolverFactory;
+  MultiThreadedProxyResolver resolver(factory, kNumThreads);
+
+  int rv;
+
+  EXPECT_TRUE(resolver.expects_pac_bytes());
+
+  // Initialize the resolver.
+  TestCompletionCallback set_script_callback;
+  rv = resolver.SetPacScriptByData(ASCIIToUTF16("pac script bytes"),
+                                   &set_script_callback);
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+  EXPECT_EQ(OK, set_script_callback.WaitForResult());
+  // One thread has been provisioned (i.e. one ProxyResolver was created).
+  ASSERT_EQ(1u, factory->resolvers().size());
+  EXPECT_EQ(ASCIIToUTF16("pac script bytes"),
+            factory->resolvers()[0]->last_pac_script());
+
+  const int kNumRequests = 4;
+  TestCompletionCallback callback[kNumRequests];
+  ProxyInfo results[kNumRequests];
+  ProxyResolver::RequestHandle request[kNumRequests];
+
+  // Start a request that will block the first thread.
+
+  factory->resolvers()[0]->Block();
+
+  rv = resolver.GetProxyForURL(
+      GURL("http://request0"), &results[0], &callback[0], &request[0],
+      BoundNetLog());
+
+  EXPECT_EQ(ERR_IO_PENDING, rv);
+  factory->resolvers()[0]->WaitUntilBlocked();
+
+  // Start 3 more requests -- they should all be serviced by thread #2
+  // since thread #1 is blocked.
+
+  for (int i = 1; i < kNumRequests; ++i) {
+    rv = resolver.GetProxyForURL(
+        GURL(StringPrintf("http://request%d", i)),
+        &results[i], &callback[i], &request[i], BoundNetLog());
+    EXPECT_EQ(ERR_IO_PENDING, rv);
+  }
+
+  // Wait for the three requests to complete (they should complete in FIFO
+  // order).
+  for (int i = 1; i < kNumRequests; ++i) {
+    EXPECT_EQ(i - 1, callback[i].WaitForResult());
+  }
+
+  // Unblock the first thread.
+  factory->resolvers()[0]->Unblock();
+  EXPECT_EQ(0, callback[0].WaitForResult());
+
+  // All in all, the first thread should have seen just 1 request. And the
+  // second thread 3 requests.
+  ASSERT_EQ(2u, factory->resolvers().size());
+  EXPECT_EQ(1, factory->resolvers()[0]->request_count());
+  EXPECT_EQ(3, factory->resolvers()[1]->request_count());
 }
 
 }  // namespace
+
 }  // namespace net