Disk cache: Switch the disk cache to use the cache_thread. ...

net/disk_cache/backend_unittest.cc

 // Copyright (c) 2006-2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/basictypes.h"
 #include "base/file_util.h"
 #include "base/path_service.h"
 #include "base/platform_thread.h"
 #include "base/string_util.h"
-#include "base/thread.h"
 #include "net/base/io_buffer.h"
 #include "net/base/net_errors.h"
 #include "net/base/test_completion_callback.h"
 #include "net/disk_cache/backend_impl.h"
 #include "net/disk_cache/disk_cache_test_base.h"
 #include "net/disk_cache/disk_cache_test_util.h"
 #include "net/disk_cache/histogram_macros.h"
 #include "net/disk_cache/mapped_file.h"
 #include "net/disk_cache/mem_backend_impl.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ skipping 226 matching lines @@
   ASSERT_EQ(net::OK, CreateEntry("key", &entry));
   ASSERT_EQ(0, entry->WriteData(0, 20000, buffer1, 0, NULL, false));
   entry->Close();
 
   // And verify that the first file is still there.
   scoped_refptr<net::IOBuffer> buffer2 = new net::IOBuffer(kSize);
   ASSERT_EQ(kSize, file_util::ReadFile(filename, buffer2->data(), kSize));
   EXPECT_EQ(0, memcmp(buffer1->data(), buffer2->data(), kSize));
 }
 
+// Tests that we deal with file-level pending operations at destruction time.
 TEST_F(DiskCacheTest, ShutdownWithPendingIO) {
-  TestCompletionCallback callback;
+  TestCompletionCallback cb;
 
   {
     FilePath path = GetCacheFilePath();
     ASSERT_TRUE(DeleteCache(path));
     base::Thread cache_thread("CacheThread");
     ASSERT_TRUE(cache_thread.StartWithOptions(
                     base::Thread::Options(MessageLoop::TYPE_IO, 0)));
 
     disk_cache::Backend* cache;
     int rv = disk_cache::BackendImpl::CreateBackend(
                  path, false, 0, net::DISK_CACHE, disk_cache::kNoRandom,
-                 cache_thread.message_loop_proxy(), &cache, &callback);
-    ASSERT_EQ(net::OK, callback.GetResult(rv));
+                 base::MessageLoopProxy::CreateForCurrentThread(), &cache, &cb);
+    ASSERT_EQ(net::OK, cb.GetResult(rv));
 
-    disk_cache::Entry* entry;
-    rv = cache->CreateEntry("some key", &entry, &callback);
-    ASSERT_EQ(net::OK, callback.GetResult(rv));
+    disk_cache::EntryImpl* entry;
+    rv = cache->CreateEntry("some key",
+                            reinterpret_cast<disk_cache::Entry**>(&entry), &cb);
+    ASSERT_EQ(net::OK, cb.GetResult(rv));
 
     const int kSize = 25000;
     scoped_refptr<net::IOBuffer> buffer = new net::IOBuffer(kSize);
     CacheTestFillBuffer(buffer->data(), kSize, false);
 
     for (int i = 0; i < 10 * 1024 * 1024; i += 64 * 1024) {
-      int rv = entry->WriteData(0, i, buffer, kSize, &callback, false);
+      // We are using the current thread as the cache thread because we want to
+      // be able to call directly this method to make sure that the OS (instead
+      // of us switching thread) is returning IO pending.
+      rv = entry->WriteDataImpl(0, i, buffer, kSize, &cb, false);
       if (rv == net::ERR_IO_PENDING)
         break;
       EXPECT_EQ(kSize, rv);
     }
 
+    // Don't call Close() to avoid going through the queue or we'll deadlock
+    // waiting for the operation to finish.
+    entry->Release();
+
+    // The cache destructor will see one pending operation here.
+    delete cache;
+
+    if (rv == net::ERR_IO_PENDING) {
+      EXPECT_TRUE(cb.have_result());
+    }
+  }
+
+  MessageLoop::current()->RunAllPending();
+}
+
+// Tests that we deal with background-thread pending operations.
+TEST_F(DiskCacheTest, ShutdownWithPendingIO2) {
+  TestCompletionCallback cb;
+
+  {
+    FilePath path = GetCacheFilePath();
+    ASSERT_TRUE(DeleteCache(path));
+    base::Thread cache_thread("CacheThread");
+    ASSERT_TRUE(cache_thread.StartWithOptions(
+                    base::Thread::Options(MessageLoop::TYPE_IO, 0)));
+
+    disk_cache::Backend* cache;
+    int rv = disk_cache::BackendImpl::CreateBackend(
+                 path, false, 0, net::DISK_CACHE, disk_cache::kNoRandom,
+                 cache_thread.message_loop_proxy(), &cache, &cb);
+    ASSERT_EQ(net::OK, cb.GetResult(rv));
+
+    disk_cache::Entry* entry;
+    rv = cache->CreateEntry("some key", &entry, &cb);
+    ASSERT_EQ(net::OK, cb.GetResult(rv));
+
+    const int kSize = 25000;
+    scoped_refptr<net::IOBuffer> buffer = new net::IOBuffer(kSize);
+    CacheTestFillBuffer(buffer->data(), kSize, false);
+
+    rv = entry->WriteData(0, 0, buffer, kSize, &cb, false);
+    EXPECT_EQ(net::ERR_IO_PENDING, rv);
+
     entry->Close();
 
-    // The cache destructor will see one pending operation here.
+    // The cache destructor will see two pending operations here.
     delete cache;
   }
 
   MessageLoop::current()->RunAllPending();
 }
 
 TEST_F(DiskCacheTest, TruncatedIndex) {
   FilePath path = GetCacheFilePath();
   ASSERT_TRUE(DeleteCache(path));
   FilePath index = path.AppendASCII("index");
@@ skipping 36 matching lines @@
   // By doubling the total size, we make this file cacheable.
   SetMaxSize(cache_size * 2);
   EXPECT_EQ(cache_size / 5, entry->WriteData(1, 0, buffer, cache_size / 5,
                                              NULL, false));
 
   // Let's fill up the cache!.
   SetMaxSize(cache_size * 10);
   EXPECT_EQ(cache_size * 3 / 4, entry->WriteData(0, 0, buffer,
                 cache_size * 3 / 4, NULL, false));
   entry->Close();
+  FlushQueueForTest();
 
   SetMaxSize(cache_size);
 
   // The cache is 95% full.
 
   ASSERT_EQ(net::OK, CreateEntry(second, &entry));
   EXPECT_EQ(cache_size / 10, entry->WriteData(0, 0, buffer, cache_size / 10,
                                               NULL, false)) << "trim the cache";
   entry->Close();
 
-  EXPECT_NE(net::OK, OpenEntry(first, &entry));
+  // On a slow system we may end up evicting everything not currently in use.

[gavinp, 2010/07/08 02:18:14]

Does reordering the assert for OpenEntry(second) and OpenEntry(first) make a
difference?  Why?

[rvargas (doing something else), 2010/07/08 17:30:50]

The problem with the old code is that it tests that only one entry (the oldest
one) is evicted. Eviction actually happens when the max_size limit is reached,
when an entry is charged by the space. In this case, it happens when "second" is
flushed from it's destructor... in other words, when the posted Close() executes
on the cache thread.

The issue with a slow system (valgrind) is that eviction of a single entry takes
more than 20 ms so we assume that we must keep working on that, and we post a
task to continue the eviction process... and given that we now have two threads,
that second task will actually run and evict the second entry so we'll fail to
open it.

By opening "second" right away, we don't give the eviction code a chance to
evict it, because when that posted task to evict runs, the entry is in use and
will not be evicted.

The sad thing is that now that we have two threads (and control for only one),
there are a lot of tests that are quite delicate in the order of events needed
to trigger the behavior under test :(

   ASSERT_EQ(net::OK, OpenEntry(second, &entry));
   EXPECT_EQ(cache_size / 10, entry->GetDataSize(0));
   entry->Close();
+
+  EXPECT_NE(net::OK, OpenEntry(first, &entry));
 }
 
 TEST_F(DiskCacheBackendTest, SetSize) {
   BackendSetSize();
 }
 
 TEST_F(DiskCacheBackendTest, NewEvictionSetSize) {
   SetNewEviction();
   BackendSetSize();
 }
@@ skipping 24 matching lines @@
   }
 
   for (int i = 0; i < 100; i++) {
     disk_cache::Entry* entry;
     ASSERT_EQ(net::OK, OpenEntry(entries[i]->GetKey(), &entry));
     EXPECT_TRUE(entry == entries[i]);
     entry->Close();
     entries[i]->Doom();
     entries[i]->Close();
   }
+  FlushQueueForTest();
   EXPECT_EQ(0, cache_->GetEntryCount());
 }
 
 TEST_F(DiskCacheBackendTest, Load) {
   // Work with a tiny index table (16 entries)
   SetMask(0xf);
   SetMaxSize(0x100000);
   BackendLoad();
 }
 
@@ skipping 203 matching lines @@
 
   // Simulate a crash.
   SimulateCrash();
 
   ASSERT_EQ(net::OK, CreateEntry(second, &entry));
   EXPECT_EQ(kSize, entry->WriteData(0, 0, buffer, kSize, NULL, false));
 
   EXPECT_EQ(2, cache_->GetEntryCount());
   SetMaxSize(kSize);
   entry->Close();  // Trim the cache.
+  FlushQueueForTest();
 
   // If we evicted the entry in less than 20mS, we have one entry in the cache;
   // if it took more than that, we posted a task and we'll delete the second
   // entry too.
   MessageLoop::current()->RunAllPending();
   EXPECT_GE(1, cache_->GetEntryCount());
   EXPECT_NE(net::OK, OpenEntry(first, &entry));
 }
 
 // We'll be leaking memory from this test.
@@ skipping 40 matching lines @@
   EXPECT_EQ(33, cache_->GetEntryCount());
   SetMaxSize(kSize);
 
   // For the new eviction code, all corrupt entries are on the second list so
   // they are not going away that easy.
   if (new_eviction_) {
     EXPECT_EQ(net::OK, DoomAllEntries());
   }
 
   entry->Close();  // Trim the cache.
+  FlushQueueForTest();
 
   // We may abort the eviction before cleaning up everything.
   MessageLoop::current()->RunAllPending();
   EXPECT_GE(30, cache_->GetEntryCount());
 }
 
 // We'll be leaking memory from this test.
 TEST_F(DiskCacheBackendTest, TrimInvalidEntry2) {
   BackendTrimInvalidEntry2();
 }
@@ skipping 554 matching lines @@
 
 // If the LRU is corrupt, we delete the cache.
 void DiskCacheBackendTest::BackendInvalidRankings() {
   disk_cache::Entry* entry;
   void* iter = NULL;
   ASSERT_EQ(net::OK, OpenNextEntry(&iter, &entry));
   entry->Close();
   EXPECT_EQ(2, cache_->GetEntryCount());
 
   EXPECT_NE(net::OK, OpenNextEntry(&iter, &entry));
-  MessageLoop::current()->RunAllPending();
+  FlushQueueForTest();  // Allow the restart to finish.
   EXPECT_EQ(0, cache_->GetEntryCount());
 }
 
 TEST_F(DiskCacheBackendTest, InvalidRankingsSuccess) {
   ASSERT_TRUE(CopyTestCache(L"bad_rankings"));
   DisableFirstCleanup();
   SetDirectMode();
   InitCache();
   BackendInvalidRankings();
 }
@@ skipping 30 matching lines @@
 void DiskCacheBackendTest::BackendDisable() {
   disk_cache::Entry *entry1, *entry2;
   void* iter = NULL;
   ASSERT_EQ(net::OK, OpenNextEntry(&iter, &entry1));
 
   EXPECT_NE(net::OK, OpenNextEntry(&iter, &entry2));
   EXPECT_EQ(2, cache_->GetEntryCount());
   EXPECT_NE(net::OK, CreateEntry("Something new", &entry2));
 
   entry1->Close();
-  MessageLoop::current()->RunAllPending();
+  FlushQueueForTest();  // Flushing the Close posts a task to restart the cache.
+  FlushQueueForTest();  // This one actually allows that task to complete.
 
   EXPECT_EQ(0, cache_->GetEntryCount());
 }
 
 TEST_F(DiskCacheBackendTest, DisableSuccess) {
   ASSERT_TRUE(CopyTestCache(L"bad_rankings"));
   DisableFirstCleanup();
   SetDirectMode();
   InitCache();
   BackendDisable();
@@ skipping 34 matching lines @@
   disk_cache::Entry* entry;
   void* iter = NULL;
   int count = 0;
   while (OpenNextEntry(&iter, &entry) == net::OK) {
     ASSERT_TRUE(NULL != entry);
     entry->Close();
     count++;
     ASSERT_LT(count, 9);
   };
 
-  MessageLoop::current()->RunAllPending();
+  FlushQueueForTest();
   EXPECT_EQ(0, cache_->GetEntryCount());
 }
 
 TEST_F(DiskCacheBackendTest, DisableSuccess2) {
   ASSERT_TRUE(CopyTestCache(L"list_loop"));
   DisableFirstCleanup();
   SetDirectMode();
   InitCache();
   BackendDisable2();
 }
@@ skipping 28 matching lines @@
 
 // If the index size changes when we disable the cache, we should not crash.
 void DiskCacheBackendTest::BackendDisable3() {
   disk_cache::Entry *entry1, *entry2;
   void* iter = NULL;
   EXPECT_EQ(2, cache_->GetEntryCount());
   ASSERT_EQ(net::OK, OpenNextEntry(&iter, &entry1));
   entry1->Close();
 
   EXPECT_NE(net::OK, OpenNextEntry(&iter, &entry2));
-  MessageLoop::current()->RunAllPending();
+  FlushQueueForTest();
 
   ASSERT_EQ(net::OK, CreateEntry("Something new", &entry2));
   entry2->Close();
 
   EXPECT_EQ(1, cache_->GetEntryCount());
 }
 
 TEST_F(DiskCacheBackendTest, DisableSuccess3) {
   ASSERT_TRUE(CopyTestCache(L"bad_rankings2"));
   DisableFirstCleanup();
@@ skipping 47 matching lines @@
   EXPECT_EQ(kBufSize, entry3->WriteData(1, 0, buf, kBufSize, NULL, false));
 
   std::string key = entry2->GetKey();
   EXPECT_EQ(sizeof(key2) - 1, key.size());
   key = entry3->GetKey();
   EXPECT_EQ(sizeof(key3) - 1, key.size());
 
   entry1->Close();
   entry2->Close();
   entry3->Close();
-  MessageLoop::current()->RunAllPending();
+  FlushQueueForTest();  // Flushing the Close posts a task to restart the cache.
+  FlushQueueForTest();  // This one actually allows that task to complete.
 
   EXPECT_EQ(0, cache_->GetEntryCount());
 }
 
 TEST_F(DiskCacheBackendTest, DisableSuccess4) {
   ASSERT_TRUE(CopyTestCache(L"bad_rankings"));
   DisableFirstCleanup();
   SetDirectMode();
   InitCache();
   BackendDisable4();
@@ skipping 201 matching lines @@
 // another.
 TEST_F(DiskCacheBackendTest, Histograms) {
   SetDirectMode();
   InitCache();
   disk_cache::BackendImpl* backend_ = cache_impl_;  // Needed be the macro.
 
   for (int i = 1; i < 3; i++) {
     CACHE_UMA(HOURS, "FillupTime", i, 28);
   }
 }