Disk cache: Reference CL for the implementation of file format version 3.

net/disk_cache/backend_unittest.cc

Index: net/disk_cache/backend_unittest.cc
===================================================================
--- net/disk_cache/backend_unittest.cc	(revision 199883)
+++ net/disk_cache/backend_unittest.cc	(working copy)
@@ -45,6 +45,8 @@
   void BackendSetSize();
   void BackendLoad();
   void BackendChain();
+  void BucketUse();
+  void BackendNewEvictionTrim();
   void BackendValidEntry();
   void BackendInvalidEntry();
   void BackendInvalidEntryRead();
@@ -83,6 +85,9 @@
   void BackendDisable2();
   void BackendDisable3();
   void BackendDisable4();
+  void BackendTotalBuffersSize1();
+  void BackendTotalBuffersSize2();
+  void BackendUpdateRankForExternalCacheHit();
   void TracingBackendBasics();
 };
 
@@ -99,7 +104,7 @@
   ASSERT_TRUE(NULL != entry1);
   entry1->Close();
   entry1 = NULL;
-
+  
   EXPECT_NE(net::OK, CreateEntry("the first key", &entry1));
   ASSERT_EQ(net::OK, OpenEntry("the first key", &entry1));
   EXPECT_NE(net::OK, OpenEntry("some other key", &entry2));
@@ -156,6 +161,11 @@
   BackendBasics();
 }
 
+TEST_F(DiskCacheBackendTest, V3Basics) {
+  UseVersion3();
+  BackendBasics();
+}
+
 void DiskCacheBackendTest::BackendKeying() {
   InitCache();
   const char* kName1 = "the first key";
@@ -201,11 +211,22 @@
   BackendKeying();
 }
 
+TEST_F(DiskCacheBackendTest, V3Keying) {
+  UseVersion3();
+  BackendKeying();
+}
+
 TEST_F(DiskCacheBackendTest, NewEvictionKeying) {
   SetNewEviction();
   BackendKeying();
 }
 
+TEST_F(DiskCacheBackendTest, V3NewEvictionKeying) {
+  UseVersion3();
+  SetNewEviction();
+  BackendKeying();
+}
+
 TEST_F(DiskCacheBackendTest, MemoryOnlyKeying) {
   SetMemoryOnlyMode();
   BackendKeying();
@@ -315,12 +336,12 @@
     ASSERT_TRUE(cache_thread.StartWithOptions(
                     base::Thread::Options(MessageLoop::TYPE_IO, 0)));
 
-    uint32 flags = disk_cache::kNoBuffering;
-    if (!fast)
-      flags |= disk_cache::kNoRandom;
+    if (fast)
+      AvoidTestFlag();
 
     UseCurrentThread();
-    CreateBackend(flags, NULL);
+    CreateBackend(NULL);
+    SetNoBuffering();
 
     disk_cache::EntryImpl* entry;
     rv = cache_->CreateEntry(
@@ -392,11 +413,11 @@
     ASSERT_TRUE(cache_thread.StartWithOptions(
                     base::Thread::Options(MessageLoop::TYPE_IO, 0)));
 
-    uint32 flags = disk_cache::kNoBuffering;
-    if (!fast)
-      flags |= disk_cache::kNoRandom;
+    if (fast)
+      AvoidTestFlag();
 
-    CreateBackend(flags, &cache_thread);
+    CreateBackend(&cache_thread);
+    SetNoBuffering();
 
     disk_cache::Entry* entry;
     int rv = cache_->CreateEntry("some key", &entry, cb.callback());
@@ -435,9 +456,9 @@
     ASSERT_TRUE(cache_thread.StartWithOptions(
                     base::Thread::Options(MessageLoop::TYPE_IO, 0)));
 
-    disk_cache::BackendFlags flags =
-      fast ? disk_cache::kNone : disk_cache::kNoRandom;
-    CreateBackend(flags, &cache_thread);
+    if (fast)
+      AvoidTestFlag();
+    CreateBackend(&cache_thread);
 
     disk_cache::Entry* entry;
     int rv = cache_->CreateEntry("some key", &entry, cb.callback());
@@ -523,17 +544,20 @@
                                        false));
 
   disk_cache::Entry* entry2;
-  ASSERT_EQ(net::OK, CreateEntry("an extra key", &entry2));
+  std::string extra_key("an extra key");
+  ASSERT_EQ(net::OK, CreateEntry(extra_key, &entry2));
   EXPECT_EQ(cache_size / 10, WriteData(entry2, 0, 0, buffer, cache_size / 10,
                                        false));
   entry2->Close();  // This will trigger the cache trim.
+  WaitForEntryToClose(extra_key);
 
-  EXPECT_NE(net::OK, OpenEntry(first, &entry2));
-
   FlushQueueForTest();  // Make sure that we are done trimming the cache.
   FlushQueueForTest();  // We may have posted two tasks to evict stuff.
 
   entry->Close();
+  WaitForEntryToClose(second);
+
+  EXPECT_NE(net::OK, OpenEntry(first, &entry2));
   ASSERT_EQ(net::OK, OpenEntry(second, &entry));
   EXPECT_EQ(cache_size / 10, entry->GetDataSize(0));
   entry->Close();
@@ -543,11 +567,24 @@
   BackendSetSize();
 }
 
+TEST_F(DiskCacheBackendTest, V3SetSize) {
+  UseVersion3();
+  UseVersion3();
+  SetNewEviction();
+  BackendSetSize();
+}
+
 TEST_F(DiskCacheBackendTest, NewEvictionSetSize) {
   SetNewEviction();
   BackendSetSize();
 }
 
+TEST_F(DiskCacheBackendTest, V3NewEvictionSetSize) {
+  UseVersion3();
+  SetNewEviction();
+  BackendSetSize();
+}
+
 TEST_F(DiskCacheBackendTest, MemoryOnlySetSize) {
   SetMemoryOnlyMode();
   BackendSetSize();
@@ -654,7 +691,102 @@
   BackendChain();
 }
 
-TEST_F(DiskCacheBackendTest, NewEvictionTrim) {
+// Tests the proper use of cell buckets while the index grow.
+void DiskCacheBackendTest::BucketUse() {
+  UseVersion3();
+  InitCache();
+  const int kNumEntries = 20;
+  disk_cache::Entry* entries[kNumEntries];
+  std::string key("The first key");
+
+  // This generates kNumEntries collisions so the extra table has to be used.
+  // There are only 2 extra buckets on the test setup, so the index has to grow.
+  for (int i = 0; i < kNumEntries; i++) {
+    ASSERT_EQ(net::OK, CreateEntry(key, &entries[i]));
+    entries[i]->Doom();
+    FlushQueueForTest();
+  }
+
+  for (int i = 0; i < kNumEntries / 2; i++)
+    entries[i]->Close();
+
+  for (int i = 0; i < kNumEntries / 2; i++) {
+    ASSERT_EQ(net::OK, CreateEntry(key, &entries[i]));
+    entries[i]->Doom();
+  }
+
+  for (int i = 0; i < kNumEntries; i++)
+    entries[i]->Close();
+}
+
+TEST_F(DiskCacheBackendTest, V3BucketUse) {
+  BucketUse();
+}
+
+TEST_F(DiskCacheBackendTest, V3DoubleIndexBucketUse) {
+  PresetTestMode();  // Doubles the index instead of a slow growth.
+  BucketUse();
+}
+
+TEST_F(DiskCacheBackendTest, V3DoubleIndex) {
+  UseVersion3();
+  PresetTestMode();
+  InitCache();
+
+  disk_cache::Entry* entry;
+  const int kNumEntries = 200;
+  for (int i = 0; i < kNumEntries; i++) {
+    std::string name(base::StringPrintf("Key %d", i));
+    ASSERT_EQ(net::OK, CreateEntry(name, &entry));
+    entry->Close();
+    FlushQueueForTest();
+  }
+
+  // Generate enough collisions to force growing the extra table (and with it,
+  // the index).
+  const int kNumSavedEntries = 5;
+  disk_cache::Entry* entries[kNumSavedEntries];
+  std::string key("The first key");
+  for (int i = 0; i < kNumSavedEntries; i++) {
+    ASSERT_EQ(net::OK, CreateEntry(key, &entries[i]));
+    entries[i]->Doom();
+    FlushQueueForTest();
+  }
+
+  for (int i = 0; i < kNumSavedEntries; i++)
+    entries[i]->Close();
+
+  // Verify that all entries are there.
+  for (int i = 0; i < kNumEntries; i++) {
+    std::string name(base::StringPrintf("Key %d", i));
+    ASSERT_EQ(net::OK, OpenEntry(name, &entry));
+    entry->Close();
+  }
+}
+
+// This test leaks memory because it simulates a crash.
+TEST_F(DiskCacheBackendTest, DISABLED_V3Backup) {
+  UseVersion3();
+  PresetTestMode();
+  InitCache();
+
+  disk_cache::Entry* entry;
+  std::string name1("First entry");
+  ASSERT_EQ(net::OK, CreateEntry(name1, &entry));
+  entry->Close();
+  WaitForEntryToClose(name1);
+  AddDelayForTest(40);
+
+  std::string name2("Another entry");
+  ASSERT_EQ(net::OK, CreateEntry(name2, &entry));
+
+  SimulateCrash();
+  EXPECT_EQ(net::OK, OpenEntry(name1, &entry));
+  entry->Close();
+  EXPECT_NE(net::OK, OpenEntry(name2, &entry));
+}
+
+void DiskCacheBackendTest::BackendNewEvictionTrim() {
   SetNewEviction();
   InitCache();
 
@@ -668,6 +800,13 @@
       ASSERT_EQ(net::OK, OpenEntry(name, &entry));
       entry->Close();
     }
+    if (!(i % 10))
+      FlushQueueForTest();
+
+    if (i == 0 || i == 90) {
+      WaitForEntryToClose(name);
+      AddDelayForTest(70);
+    }
   }
 
   // The first eviction must come from list 1 (10% limit), the second must come
@@ -684,6 +823,15 @@
   entry->Close();
 }
 
+TEST_F(DiskCacheBackendTest, NewEvictionTrim) {
+  BackendNewEvictionTrim();
+}
+
+TEST_F(DiskCacheBackendTest, V3NewEvictionTrim) {
+  UseVersion3();
+  BackendNewEvictionTrim();
+}
+
 // Before looking for invalid entries, let's check a valid entry.
 void DiskCacheBackendTest::BackendValidEntry() {
   InitCache();
@@ -1018,6 +1166,8 @@
     disk_cache::Entry* entry;
     ASSERT_EQ(net::OK, CreateEntry(key, &entry));
     entry->Close();
+    if (!(i % 10))
+      FlushQueueForTest();
   }
   EXPECT_EQ(kNumEntries, cache_->GetEntryCount());
   Time final = Time::Now();
@@ -1060,11 +1210,22 @@
   BackendEnumerations();
 }
 
+TEST_F(DiskCacheBackendTest, V3Enumerations) {
+  UseVersion3();
+  BackendEnumerations();
+}
+
 TEST_F(DiskCacheBackendTest, NewEvictionEnumerations) {
   SetNewEviction();
   BackendEnumerations();
 }
 
+TEST_F(DiskCacheBackendTest, V3NewEvictionEnumerations) {
+  UseVersion3();
+  SetNewEviction();
+  BackendEnumerations();
+}
+
 TEST_F(DiskCacheBackendTest, MemoryOnlyEnumerations) {
   SetMemoryOnlyMode();
   BackendEnumerations();
@@ -1093,7 +1254,7 @@
   FlushQueueForTest();
 
   // Make sure that the timestamp is not the same.
-  AddDelay();
+  AddDelayForTest(70);
   ASSERT_EQ(net::OK, OpenEntry(second, &entry1));
   void* iter = NULL;
   ASSERT_EQ(net::OK, OpenNextEntry(&iter, &entry2));
@@ -1129,11 +1290,22 @@
   BackendEnumerations2();
 }
 
+TEST_F(DiskCacheBackendTest, V3Enumerations2) {
+  UseVersion3();
+  BackendEnumerations2();
+}
+
 TEST_F(DiskCacheBackendTest, NewEvictionEnumerations2) {
   SetNewEviction();
   BackendEnumerations2();
 }
 
+TEST_F(DiskCacheBackendTest, V3NewEvictionEnumerations2) {
+  UseVersion3();
+  SetNewEviction();
+  BackendEnumerations2();
+}
+
 TEST_F(DiskCacheBackendTest, MemoryOnlyEnumerations2) {
   SetMemoryOnlyMode();
   BackendEnumerations2();
@@ -1307,8 +1479,8 @@
   entry->Close();
   FlushQueueForTest();
 
-  AddDelay();
-  Time middle = Time::Now();
+  AddDelayForTest(70);
+  Time middle = GetCurrentTime();
 
   ASSERT_EQ(net::OK, CreateEntry("third", &entry));
   entry->Close();
@@ -1316,8 +1488,8 @@
   entry->Close();
   FlushQueueForTest();
 
-  AddDelay();
-  Time final = Time::Now();
+  AddDelayForTest(70);
+  Time final = GetCurrentTime();
 
   ASSERT_EQ(4, cache_->GetEntryCount());
   EXPECT_EQ(net::OK, DoomEntriesSince(final));
@@ -1334,11 +1506,22 @@
   BackendDoomRecent();
 }
 
+TEST_F(DiskCacheBackendTest, V3DoomRecent) {
+  UseVersion3();
+  BackendDoomRecent();
+}
+
 TEST_F(DiskCacheBackendTest, NewEvictionDoomRecent) {
   SetNewEviction();
   BackendDoomRecent();
 }
 
+TEST_F(DiskCacheBackendTest, V3NewEvictionDoomRecent) {
+  UseVersion3();
+  SetNewEviction();
+  BackendDoomRecent();
+}
+
 TEST_F(DiskCacheBackendTest, MemoryOnlyDoomRecent) {
   SetMemoryOnlyMode();
   BackendDoomRecent();
@@ -1426,6 +1609,17 @@
   EXPECT_EQ(3, cache_->GetEntryCount());
 }
 
+TEST_F(DiskCacheBackendTest, V3DoomEntriesSinceSparse) {
+  base::Time start;
+  UseVersion3();
+  InitSparseCache(&start, NULL);
+  DoomEntriesSince(start);
+  // NOTE: BackendImpl counts child entries in its GetEntryCount(), while
+  // MemBackendImpl does not. Thats why expected value differs here from
+  // MemoryOnlyDoomEntriesSinceSparse.
+  EXPECT_EQ(3, cache_->GetEntryCount());
+}
+
 TEST_F(DiskCacheBackendTest, MemoryOnlyDoomAllSparse) {
   SetMemoryOnlyMode();
   InitSparseCache(NULL, NULL);
@@ -1439,6 +1633,13 @@
   EXPECT_EQ(0, cache_->GetEntryCount());
 }
 
+TEST_F(DiskCacheBackendTest, V3DoomAllSparse) {
+  UseVersion3();
+  InitSparseCache(NULL, NULL);
+  EXPECT_EQ(net::OK, DoomAllEntries());
+  EXPECT_EQ(0, cache_->GetEntryCount());
+}
+
 void DiskCacheBackendTest::BackendDoomBetween() {
   InitCache();
 
@@ -1486,11 +1687,22 @@
   BackendDoomBetween();
 }
 
+TEST_F(DiskCacheBackendTest, V3DoomBetween) {
+  UseVersion3();
+  BackendDoomBetween();
+}
+
 TEST_F(DiskCacheBackendTest, NewEvictionDoomBetween) {
   SetNewEviction();
   BackendDoomBetween();
 }
 
+TEST_F(DiskCacheBackendTest, V3NewEvictionDoomBetween) {
+  UseVersion3();
+  SetNewEviction();
+  BackendDoomBetween();
+}
+
 TEST_F(DiskCacheBackendTest, MemoryOnlyDoomBetween) {
   SetMemoryOnlyMode();
   BackendDoomBetween();
@@ -1521,6 +1733,19 @@
   EXPECT_EQ(3, cache_->GetEntryCount());
 }
 
+TEST_F(DiskCacheBackendTest, V3DoomEntriesBetweenSparse) {
+  base::Time start, end;
+  UseVersion3();
+  InitSparseCache(&start, &end);
+  DoomEntriesBetween(start, end);
+  EXPECT_EQ(9, cache_->GetEntryCount());
+
+  start = end;
+  end = base::Time::Now();
+  DoomEntriesBetween(start, end);
+  EXPECT_EQ(3, cache_->GetEntryCount());
+}
+
 void DiskCacheBackendTest::BackendTransaction(const std::string& name,
                                               int num_entries, bool load) {
   success_ = false;
@@ -2514,11 +2739,22 @@
   BackendDoomAll();
 }
 
+TEST_F(DiskCacheBackendTest, V3DoomAll) {
+  UseVersion3();
+  BackendDoomAll();
+}
+
 TEST_F(DiskCacheBackendTest, NewEvictionDoomAll) {
   SetNewEviction();
   BackendDoomAll();
 }
 
+TEST_F(DiskCacheBackendTest, V3NewEvictionDoomAll) {
+  UseVersion3();
+  SetNewEviction();
+  BackendDoomAll();
+}
+
 TEST_F(DiskCacheBackendTest, MemoryOnlyDoomAll) {
   SetMemoryOnlyMode();
   BackendDoomAll();
@@ -2662,7 +2898,7 @@
 
 // Make sure that we keep the total memory used by the internal buffers under
 // control.
-TEST_F(DiskCacheBackendTest, TotalBuffersSize1) {
+void DiskCacheBackendTest::BackendTotalBuffersSize1() {
   InitCache();
   std::string key("the first key");
   disk_cache::Entry* entry;
@@ -2690,35 +2926,53 @@
   }
 
   entry->Close();
-  EXPECT_EQ(0, cache_impl_->GetTotalBuffersSize());
+  EXPECT_EQ(0, GetTotalBuffersSize());
 }
 
+TEST_F(DiskCacheBackendTest, TotalBuffersSize1) {
+  BackendTotalBuffersSize1();
+}
+
+TEST_F(DiskCacheBackendTest, V3TotalBuffersSize1) {
+  UseVersion3();
+  BackendTotalBuffersSize1();
+}
+
 // This test assumes at least 150MB of system memory.
-TEST_F(DiskCacheBackendTest, TotalBuffersSize2) {
+void DiskCacheBackendTest::BackendTotalBuffersSize2() {
   InitCache();
 
   const int kOneMB = 1024 * 1024;
-  EXPECT_TRUE(cache_impl_->IsAllocAllowed(0, kOneMB));
-  EXPECT_EQ(kOneMB, cache_impl_->GetTotalBuffersSize());
+  EXPECT_TRUE(IsAllocAllowed(0, kOneMB));
+  EXPECT_EQ(kOneMB, GetTotalBuffersSize());
 
-  EXPECT_TRUE(cache_impl_->IsAllocAllowed(0, kOneMB));
-  EXPECT_EQ(kOneMB * 2, cache_impl_->GetTotalBuffersSize());
+  EXPECT_TRUE(IsAllocAllowed(0, kOneMB));
+  EXPECT_EQ(kOneMB * 2, GetTotalBuffersSize());
 
-  EXPECT_TRUE(cache_impl_->IsAllocAllowed(0, kOneMB));
-  EXPECT_EQ(kOneMB * 3, cache_impl_->GetTotalBuffersSize());
+  EXPECT_TRUE(IsAllocAllowed(0, kOneMB));
+  EXPECT_EQ(kOneMB * 3, GetTotalBuffersSize());
 
-  cache_impl_->BufferDeleted(kOneMB);
-  EXPECT_EQ(kOneMB * 2, cache_impl_->GetTotalBuffersSize());
+  BufferDeleted(kOneMB);
+  EXPECT_EQ(kOneMB * 2, GetTotalBuffersSize());
 
   // Check the upper limit.
-  EXPECT_FALSE(cache_impl_->IsAllocAllowed(0, 30 * kOneMB));
+  EXPECT_FALSE(IsAllocAllowed(0, 30 * kOneMB));
 
   for (int i = 0; i < 30; i++)
-    cache_impl_->IsAllocAllowed(0, kOneMB);  // Ignore the result.
+    IsAllocAllowed(0, kOneMB);  // Ignore the result.
 
-  EXPECT_FALSE(cache_impl_->IsAllocAllowed(0, kOneMB));
+  EXPECT_FALSE(IsAllocAllowed(0, kOneMB));
 }
 
+TEST_F(DiskCacheBackendTest, TotalBuffersSize2) {
+  BackendTotalBuffersSize2();
+}
+
+TEST_F(DiskCacheBackendTest, V3TotalBuffersSize2) {
+  UseVersion3();
+  BackendTotalBuffersSize2();
+}
+
 // Tests that sharing of external files works and we are able to delete the
 // files when we need to.
 TEST_F(DiskCacheBackendTest, FileSharing) {
@@ -2759,7 +3013,7 @@
   EXPECT_TRUE(disk_cache::DeleteCacheFile(name));
 }
 
-TEST_F(DiskCacheBackendTest, UpdateRankForExternalCacheHit) {
+void DiskCacheBackendTest::BackendUpdateRankForExternalCacheHit() {
   InitCache();
 
   disk_cache::Entry* entry;
@@ -2768,40 +3022,41 @@
     std::string key = base::StringPrintf("key%d", i);
     ASSERT_EQ(net::OK, CreateEntry(key, &entry));
     entry->Close();
+    AddDelayForTest(70);
   }
+  FlushQueueForTest();
 
   // Ping the oldest entry.
+  SetTestMode();
   cache_->OnExternalCacheHit("key0");
+  FlushQueueForTest();
 
   TrimForTest(false);
 
   // Make sure the older key remains.
-  EXPECT_EQ(1, cache_->GetEntryCount());
+  EXPECT_NE(net::OK, OpenEntry("key1", &entry));
   ASSERT_EQ(net::OK, OpenEntry("key0", &entry));
   entry->Close();
 }
 
+TEST_F(DiskCacheBackendTest, UpdateRankForExternalCacheHit) {
+  BackendUpdateRankForExternalCacheHit();
+}
+
+TEST_F(DiskCacheBackendTest, V3UpdateRankForExternalCacheHit) {
+  UseVersion3();
+  BackendUpdateRankForExternalCacheHit();
+}
+
 TEST_F(DiskCacheBackendTest, ShaderCacheUpdateRankForExternalCacheHit) {
   SetCacheType(net::SHADER_CACHE);
-  InitCache();
+  BackendUpdateRankForExternalCacheHit();
+}
 
-  disk_cache::Entry* entry;
-
-  for (int i = 0; i < 2; ++i) {
-    std::string key = base::StringPrintf("key%d", i);
-    ASSERT_EQ(net::OK, CreateEntry(key, &entry));
-    entry->Close();
-  }
-
-  // Ping the oldest entry.
-  cache_->OnExternalCacheHit("key0");
-
-  TrimForTest(false);
-
-  // Make sure the older key remains.
-  EXPECT_EQ(1, cache_->GetEntryCount());
-  ASSERT_EQ(net::OK, OpenEntry("key0", &entry));
-  entry->Close();
+TEST_F(DiskCacheBackendTest, V3ShaderCacheUpdateRankForExternalCacheHit) {
+  UseVersion3();
+  SetCacheType(net::SHADER_CACHE);
+  BackendUpdateRankForExternalCacheHit();
 }
 
 void DiskCacheBackendTest::TracingBackendBasics() {