Disk cache: Extend the internal buffering performed by each entry...

net/disk_cache/backend_unittest.cc

 // Copyright (c) 2006-2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "base/basictypes.h"
 #include "base/file_util.h"
 #include "base/platform_thread.h"
 #include "base/string_util.h"
 #include "net/base/io_buffer.h"
 #include "net/base/net_errors.h"
@@ skipping 206 matching lines @@
   FilePath filename = GetCacheFilePath().AppendASCII("f_000001");
 
   const int kSize = 50;
   scoped_refptr<net::IOBuffer> buffer1 = new net::IOBuffer(kSize);
   CacheTestFillBuffer(buffer1->data(), kSize, false);
   ASSERT_EQ(kSize, file_util::WriteFile(filename, buffer1->data(), kSize));
 
   // Now let's create a file with the cache.
   disk_cache::Entry* entry;
   ASSERT_EQ(net::OK, CreateEntry("key", &entry));
-  ASSERT_EQ(0, entry->WriteData(0, 20000, buffer1, 0, NULL, false));
+  ASSERT_EQ(0, WriteData(entry, 0, 20000, buffer1, 0, false));
   entry->Close();
 
   // And verify that the first file is still there.
   scoped_refptr<net::IOBuffer> buffer2 = new net::IOBuffer(kSize);
   ASSERT_EQ(kSize, file_util::ReadFile(filename, buffer2->data(), kSize));
   EXPECT_EQ(0, memcmp(buffer1->data(), buffer2->data(), kSize));
 }
 
 // Tests that we deal with file-level pending operations at destruction time.
 TEST_F(DiskCacheTest, ShutdownWithPendingIO) {
@@ skipping 110 matching lines @@
   SetMaxSize(cache_size);
   InitCache();
 
   std::string first("some key");
   std::string second("something else");
   disk_cache::Entry* entry;
   ASSERT_EQ(net::OK, CreateEntry(first, &entry));
 
   scoped_refptr<net::IOBuffer> buffer = new net::IOBuffer(cache_size);
   memset(buffer->data(), 0, cache_size);
-  EXPECT_EQ(cache_size / 10, entry->WriteData(0, 0, buffer, cache_size / 10,
-                                              NULL, false)) << "normal file";
+  EXPECT_EQ(cache_size / 10, WriteData(entry, 0, 0, buffer, cache_size / 10,
+                                       false)) << "normal file";
 
-  EXPECT_EQ(net::ERR_FAILED, entry->WriteData(1, 0, buffer, cache_size / 5,
-                NULL, false)) << "file size above the limit";
+  EXPECT_EQ(net::ERR_FAILED, WriteData(entry, 1, 0, buffer, cache_size / 5,
+                                       false)) << "file size above the limit";
 
   // By doubling the total size, we make this file cacheable.
   SetMaxSize(cache_size * 2);
-  EXPECT_EQ(cache_size / 5, entry->WriteData(1, 0, buffer, cache_size / 5,
-                                             NULL, false));
+  EXPECT_EQ(cache_size / 5, WriteData(entry, 1, 0, buffer, cache_size / 5,
+                                      false));
 
   // Let's fill up the cache!.
   SetMaxSize(cache_size * 10);
-  EXPECT_EQ(cache_size * 3 / 4, entry->WriteData(0, 0, buffer,
-                cache_size * 3 / 4, NULL, false));
+  EXPECT_EQ(cache_size * 3 / 4, WriteData(entry, 0, 0, buffer,
+                                          cache_size * 3 / 4, false));
   entry->Close();
   FlushQueueForTest();
 
   SetMaxSize(cache_size);
 
   // The cache is 95% full.
 
   ASSERT_EQ(net::OK, CreateEntry(second, &entry));
-  EXPECT_EQ(cache_size / 10, entry->WriteData(0, 0, buffer, cache_size / 10,
-                                              NULL, false));
+  EXPECT_EQ(cache_size / 10, WriteData(entry, 0, 0, buffer, cache_size / 10,
+                                       false));
 
   disk_cache::Entry* entry2;
   ASSERT_EQ(net::OK, CreateEntry("an extra key", &entry2));
-  EXPECT_EQ(cache_size / 10, entry2->WriteData(0, 0, buffer, cache_size / 10,
-                                               NULL, false));
+  EXPECT_EQ(cache_size / 10, WriteData(entry2, 0, 0, buffer, cache_size / 10,
+                                       false));
   entry2->Close();  // This will trigger the cache trim.
 
   EXPECT_NE(net::OK, OpenEntry(first, &entry2));
 
   FlushQueueForTest();  // Make sure that we are done trimming the cache.
   FlushQueueForTest();  // We may have posted two tasks to evict stuff.
 
   entry->Close();
   ASSERT_EQ(net::OK, OpenEntry(second, &entry));
   EXPECT_EQ(cache_size / 10, entry->GetDataSize(0));
@@ skipping 67 matching lines @@
   SetMemoryOnlyMode();
   BackendLoad();
 }
 
 // Before looking for invalid entries, let's check a valid entry.
 void DiskCacheBackendTest::BackendValidEntry() {
   SetDirectMode();
   InitCache();
 
   std::string key("Some key");
-  disk_cache::Entry* entry1;
-  ASSERT_EQ(net::OK, CreateEntry(key, &entry1));
+  disk_cache::Entry* entry;
+  ASSERT_EQ(net::OK, CreateEntry(key, &entry));
 
   const int kSize = 50;
   scoped_refptr<net::IOBuffer> buffer1 = new net::IOBuffer(kSize);
   memset(buffer1->data(), 0, kSize);
   base::strlcpy(buffer1->data(), "And the data to save", kSize);
-  EXPECT_EQ(kSize, entry1->WriteData(0, 0, buffer1, kSize, NULL, false));
-  entry1->Close();
+  EXPECT_EQ(kSize, WriteData(entry, 0, 0, buffer1, kSize, false));
+  entry->Close();
   SimulateCrash();
 
-  ASSERT_EQ(net::OK, OpenEntry(key, &entry1));
+  ASSERT_EQ(net::OK, OpenEntry(key, &entry));
 
   scoped_refptr<net::IOBuffer> buffer2 = new net::IOBuffer(kSize);
   memset(buffer2->data(), 0, kSize);
-  EXPECT_EQ(kSize, entry1->ReadData(0, 0, buffer2, kSize, NULL));
-  entry1->Close();
+  EXPECT_EQ(kSize, ReadData(entry, 0, 0, buffer2, kSize));
+  entry->Close();
   EXPECT_STREQ(buffer1->data(), buffer2->data());
 }
 
 TEST_F(DiskCacheBackendTest, ValidEntry) {
   BackendValidEntry();
 }
 
 TEST_F(DiskCacheBackendTest, NewEvictionValidEntry) {
   SetNewEviction();
   BackendValidEntry();
 }
 
 // The same logic of the previous test (ValidEntry), but this time force the
 // entry to be invalid, simulating a crash in the middle.
 // We'll be leaking memory from this test.
 void DiskCacheBackendTest::BackendInvalidEntry() {
   // Use the implementation directly... we need to simulate a crash.
   SetDirectMode();
   InitCache();
 
   std::string key("Some key");
-  disk_cache::Entry* entry1;
-  ASSERT_EQ(net::OK, CreateEntry(key, &entry1));
+  disk_cache::Entry* entry;
+  ASSERT_EQ(net::OK, CreateEntry(key, &entry));
 
   const int kSize = 50;
-  scoped_refptr<net::IOBuffer> buffer1 = new net::IOBuffer(kSize);
-  memset(buffer1->data(), 0, kSize);
-  base::strlcpy(buffer1->data(), "And the data to save", kSize);
-  EXPECT_EQ(kSize, entry1->WriteData(0, 0, buffer1, kSize, NULL, false));
+  scoped_refptr<net::IOBuffer> buffer = new net::IOBuffer(kSize);
+  memset(buffer->data(), 0, kSize);
+  base::strlcpy(buffer->data(), "And the data to save", kSize);
+  EXPECT_EQ(kSize, WriteData(entry, 0, 0, buffer, kSize, false));
   SimulateCrash();
 
-  EXPECT_NE(net::OK, OpenEntry(key, &entry1));
+  EXPECT_NE(net::OK, OpenEntry(key, &entry));
   EXPECT_EQ(0, cache_->GetEntryCount());
 }
 
 // This and the other intentionally leaky tests below are excluded from
 // purify and valgrind runs by naming them in the files
 //   net/data/purify/net_unittests.exe.gtest.txt and
 //   net/data/valgrind/net_unittests.gtest.txt
 // The scripts tools/{purify,valgrind}/chrome_tests.sh
 // read those files and pass the appropriate --gtest_filter to net_unittests.
 TEST_F(DiskCacheBackendTest, InvalidEntry) {
   BackendInvalidEntry();
 }
 
 // We'll be leaking memory from this test.
 TEST_F(DiskCacheBackendTest, NewEvictionInvalidEntry) {
   SetNewEviction();
   BackendInvalidEntry();
 }
 
 // Almost the same test, but this time crash the cache after reading an entry.
 // We'll be leaking memory from this test.
 void DiskCacheBackendTest::BackendInvalidEntryRead() {
   // Use the implementation directly... we need to simulate a crash.
   SetDirectMode();
   InitCache();
 
   std::string key("Some key");
-  disk_cache::Entry* entry1;
-  ASSERT_EQ(net::OK, CreateEntry(key, &entry1));
+  disk_cache::Entry* entry;
+  ASSERT_EQ(net::OK, CreateEntry(key, &entry));
 
   const int kSize = 50;
-  scoped_refptr<net::IOBuffer> buffer1 = new net::IOBuffer(kSize);
-  memset(buffer1->data(), 0, kSize);
-  base::strlcpy(buffer1->data(), "And the data to save", kSize);
-  EXPECT_EQ(kSize, entry1->WriteData(0, 0, buffer1, kSize, NULL, false));
-  entry1->Close();
-  ASSERT_EQ(net::OK, OpenEntry(key, &entry1));
-  EXPECT_EQ(kSize, entry1->ReadData(0, 0, buffer1, kSize, NULL));
+  scoped_refptr<net::IOBuffer> buffer = new net::IOBuffer(kSize);
+  memset(buffer->data(), 0, kSize);
+  base::strlcpy(buffer->data(), "And the data to save", kSize);
+  EXPECT_EQ(kSize, WriteData(entry, 0, 0, buffer, kSize, false));
+  entry->Close();
+  ASSERT_EQ(net::OK, OpenEntry(key, &entry));
+  EXPECT_EQ(kSize, ReadData(entry, 0, 0, buffer, kSize));
 
   SimulateCrash();
 
-  EXPECT_NE(net::OK, OpenEntry(key, &entry1));
+  EXPECT_NE(net::OK, OpenEntry(key, &entry));
   EXPECT_EQ(0, cache_->GetEntryCount());
 }
 
 // We'll be leaking memory from this test.
 TEST_F(DiskCacheBackendTest, InvalidEntryRead) {
   BackendInvalidEntryRead();
 }
 
 // We'll be leaking memory from this test.
 TEST_F(DiskCacheBackendTest, NewEvictionInvalidEntryRead) {
@@ skipping 70 matching lines @@
   SetMaxSize(kSize * 10);
   InitCache();
 
   std::string first("some key");
   std::string second("something else");
   disk_cache::Entry* entry;
   ASSERT_EQ(net::OK, CreateEntry(first, &entry));
 
   scoped_refptr<net::IOBuffer> buffer = new net::IOBuffer(kSize);
   memset(buffer->data(), 0, kSize);
-  EXPECT_EQ(kSize, entry->WriteData(0, 0, buffer, kSize, NULL, false));
+  EXPECT_EQ(kSize, WriteData(entry, 0, 0, buffer, kSize, false));
 
   // Simulate a crash.
   SimulateCrash();
 
   ASSERT_EQ(net::OK, CreateEntry(second, &entry));
-  EXPECT_EQ(kSize, entry->WriteData(0, 0, buffer, kSize, NULL, false));
+  EXPECT_EQ(kSize, WriteData(entry, 0, 0, buffer, kSize, false));
 
   EXPECT_EQ(2, cache_->GetEntryCount());
   SetMaxSize(kSize);
   entry->Close();  // Trim the cache.
   FlushQueueForTest();
 
   // If we evicted the entry in less than 20mS, we have one entry in the cache;
   // if it took more than that, we posted a task and we'll delete the second
   // entry too.
   MessageLoop::current()->RunAllPending();
@@ skipping 23 matching lines @@
   InitCache();
 
   scoped_refptr<net::IOBuffer> buffer = new net::IOBuffer(kSize);
   memset(buffer->data(), 0, kSize);
   disk_cache::Entry* entry;
 
   // Writing 32 entries to this cache chains most of them.
   for (int i = 0; i < 32; i++) {
     std::string key(StringPrintf("some key %d", i));
     ASSERT_EQ(net::OK, CreateEntry(key, &entry));
-    EXPECT_EQ(kSize, entry->WriteData(0, 0, buffer, kSize, NULL, false));
+    EXPECT_EQ(kSize, WriteData(entry, 0, 0, buffer, kSize, false));
     entry->Close();
     ASSERT_EQ(net::OK, OpenEntry(key, &entry));
     // Note that we are not closing the entries.
   }
 
   // Simulate a crash.
   SimulateCrash();
 
   ASSERT_EQ(net::OK, CreateEntry("Something else", &entry));
-  EXPECT_EQ(kSize, entry->WriteData(0, 0, buffer, kSize, NULL, false));
+  EXPECT_EQ(kSize, WriteData(entry, 0, 0, buffer, kSize, false));
 
   EXPECT_EQ(33, cache_->GetEntryCount());
   SetMaxSize(kSize);
 
   // For the new eviction code, all corrupt entries are on the second list so
   // they are not going away that easy.
   if (new_eviction_) {
     EXPECT_EQ(net::OK, DoomAllEntries());
   }
 
@@ skipping 132 matching lines @@
   InitCache();
 
   std::string key("Some key");
   disk_cache::Entry *entry, *entry1, *entry2;
   ASSERT_EQ(net::OK, CreateEntry(key, &entry1));
 
   const int kSize = 50;
   scoped_refptr<net::IOBuffer> buffer1 = new net::IOBuffer(kSize);
   memset(buffer1->data(), 0, kSize);
   base::strlcpy(buffer1->data(), "And the data to save", kSize);
-  EXPECT_EQ(kSize, entry1->WriteData(0, 0, buffer1, kSize, NULL, false));
+  EXPECT_EQ(kSize, WriteData(entry1, 0, 0, buffer1, kSize, false));
   entry1->Close();
   ASSERT_EQ(net::OK, OpenEntry(key, &entry1));
-  EXPECT_EQ(kSize, entry1->ReadData(0, 0, buffer1, kSize, NULL));
+  EXPECT_EQ(kSize, ReadData(entry1, 0, 0, buffer1, kSize));
 
   std::string key2("Another key");
   ASSERT_EQ(net::OK, CreateEntry(key2, &entry2));
   entry2->Close();
   ASSERT_EQ(2, cache_->GetEntryCount());
 
   SimulateCrash();
 
   void* iter = NULL;
   int count = 0;
@@ skipping 609 matching lines @@
   CacheTestFillBuffer(key2, sizeof(key2), true);
   CacheTestFillBuffer(key3, sizeof(key3), true);
   key2[sizeof(key2) - 1] = '\0';
   key3[sizeof(key3) - 1] = '\0';
   ASSERT_EQ(net::OK, CreateEntry(key2, &entry2));
   ASSERT_EQ(net::OK, CreateEntry(key3, &entry3));
 
   const int kBufSize = 20000;
   scoped_refptr<net::IOBuffer> buf = new net::IOBuffer(kBufSize);
   memset(buf->data(), 0, kBufSize);
-  EXPECT_EQ(100, entry2->WriteData(0, 0, buf, 100, NULL, false));
-  EXPECT_EQ(kBufSize, entry3->WriteData(0, 0, buf, kBufSize, NULL, false));
+  EXPECT_EQ(100, WriteData(entry2, 0, 0, buf, 100, false));
+  EXPECT_EQ(kBufSize, WriteData(entry3, 0, 0, buf, kBufSize, false));
 
   // This line should disable the cache but not delete it.
   EXPECT_NE(net::OK, OpenNextEntry(&iter, &entry4));
   EXPECT_EQ(4, cache_->GetEntryCount());
 
   EXPECT_NE(net::OK, CreateEntry("cache is disabled", &entry4));
 
-  EXPECT_EQ(100, entry2->ReadData(0, 0, buf, 100, NULL));
-  EXPECT_EQ(100, entry2->WriteData(0, 0, buf, 100, NULL, false));
-  EXPECT_EQ(100, entry2->WriteData(1, 0, buf, 100, NULL, false));
+  EXPECT_EQ(100, ReadData(entry2, 0, 0, buf, 100));
+  EXPECT_EQ(100, WriteData(entry2, 0, 0, buf, 100, false));
+  EXPECT_EQ(100, WriteData(entry2, 1, 0, buf, 100, false));
 
-  EXPECT_EQ(kBufSize, entry3->ReadData(0, 0, buf, kBufSize, NULL));
-  EXPECT_EQ(kBufSize, entry3->WriteData(0, 0, buf, kBufSize, NULL, false));
-  EXPECT_EQ(kBufSize, entry3->WriteData(1, 0, buf, kBufSize, NULL, false));
+  EXPECT_EQ(kBufSize, ReadData(entry3, 0, 0, buf, kBufSize));
+  EXPECT_EQ(kBufSize, WriteData(entry3, 0, 0, buf, kBufSize, false));
+  EXPECT_EQ(kBufSize, WriteData(entry3, 1, 0, buf, kBufSize, false));
 
   std::string key = entry2->GetKey();
   EXPECT_EQ(sizeof(key2) - 1, key.size());
   key = entry3->GetKey();
   EXPECT_EQ(sizeof(key3) - 1, key.size());
 
   entry1->Close();
   entry2->Close();
   entry3->Close();
   FlushQueueForTest();  // Flushing the Close posts a task to restart the cache.
@@ skipping 212 matching lines @@
 // another.
 TEST_F(DiskCacheBackendTest, Histograms) {
   SetDirectMode();
   InitCache();
   disk_cache::BackendImpl* backend_ = cache_impl_;  // Needed be the macro.
 
   for (int i = 1; i < 3; i++) {
     CACHE_UMA(HOURS, "FillupTime", i, 28);
   }
 }
+
+// Make sure that we keep the total memory used by the internal buffers under
+// control.
+TEST_F(DiskCacheBackendTest, TotalBuffersSize1) {
+  SetDirectMode();
+  InitCache();
+  std::string key("the first key");
+  disk_cache::Entry* entry;
+  ASSERT_EQ(net::OK, CreateEntry(key, &entry));
+
+  const int kSize = 200;
+  scoped_refptr<net::IOBuffer> buffer = new net::IOBuffer(kSize);
+  CacheTestFillBuffer(buffer->data(), kSize, true);
+
+  for (int i = 0; i < 10; i++) {
+    SCOPED_TRACE(i);
+    // Allocate 2MB for this entry.
+    EXPECT_EQ(kSize, WriteData(entry, 0, 0, buffer, kSize, true));
+    EXPECT_EQ(kSize, WriteData(entry, 1, 0, buffer, kSize, true));
+    EXPECT_EQ(kSize, WriteData(entry, 0, 1024 * 1024, buffer, kSize, false));
+    EXPECT_EQ(kSize, WriteData(entry, 1, 1024 * 1024, buffer, kSize, false));
+
+    // Delete one of the buffers and truncate the other.
+    EXPECT_EQ(0, WriteData(entry, 0, 0, buffer, 0, true));
+    EXPECT_EQ(0, WriteData(entry, 1, 10, buffer, 0, true));
+
+    // Delete the second buffer, writing 10 bytes to disk.
+    entry->Close();
+    ASSERT_EQ(net::OK, OpenEntry(key, &entry));
+  }
+
+  entry->Close();
+  EXPECT_EQ(0, cache_impl_->GetTotalBuffersSize());
+}
+
+// This test assumes at least 150MB of system memory.
+TEST_F(DiskCacheBackendTest, TotalBuffersSize2) {
+  SetDirectMode();
+  InitCache();
+
+  const int kOneMB = 1024 * 1024;
+  EXPECT_TRUE(cache_impl_->IsAllocAllowed(0, kOneMB));
+  EXPECT_EQ(kOneMB, cache_impl_->GetTotalBuffersSize());
+
+  EXPECT_TRUE(cache_impl_->IsAllocAllowed(0, kOneMB));
+  EXPECT_EQ(kOneMB * 2, cache_impl_->GetTotalBuffersSize());
+
+  EXPECT_TRUE(cache_impl_->IsAllocAllowed(0, kOneMB));
+  EXPECT_EQ(kOneMB * 3, cache_impl_->GetTotalBuffersSize());
+
+  cache_impl_->BufferDeleted(kOneMB);
+  EXPECT_EQ(kOneMB * 2, cache_impl_->GetTotalBuffersSize());
+
+  // Check the upper limit.
+  EXPECT_FALSE(cache_impl_->IsAllocAllowed(0, 30 * kOneMB));
+
+  for (int i = 0; i < 30; i++)
+    cache_impl_->IsAllocAllowed(0, kOneMB);  // Ignore the result.
+
+  EXPECT_FALSE(cache_impl_->IsAllocAllowed(0, kOneMB));
+}

[gavinp, 2010/08/19 18:17:22]

NIT: add a newline here