Implement cache counting for the simple and memory backends.

net/disk_cache/backend_unittest.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <stdint.h>
 
 #include "base/basictypes.h"
 #include "base/files/file_util.h"
 #include "base/metrics/field_trial.h"
 #include "base/run_loop.h"
@@ skipping 71 matching lines @@
   // will be filled with times, used by DoomEntriesSince and DoomEntriesBetween.
   // There are 4 entries after doomed_start and 2 after doomed_end.
   void InitSparseCache(base::Time* doomed_start, base::Time* doomed_end);
 
   bool CreateSetOfRandomEntries(std::set<std::string>* key_pool);
   bool EnumerateAndMatchKeys(int max_to_open,
                              TestIterator* iter,
                              std::set<std::string>* keys_to_match,
                              size_t* count);
 
+  // Computes the expected size of entry metadata, i.e. the total size without
+  // the actual data stored. This depends only on the entry's |key| size.
+  int GetEntryMetadataSize(std::string key);
+
   // Actual tests:
   void BackendBasics();
   void BackendKeying();
   void BackendShutdownWithPendingFileIO(bool fast);
   void BackendShutdownWithPendingIO(bool fast);
   void BackendShutdownWithPendingCreate(bool fast);
   void BackendSetSize();
   void BackendLoad();
   void BackendChain();
   void BackendValidEntry();
@@ skipping 173 matching lines @@
     EXPECT_EQ(1U, keys_to_match->erase(entry->GetKey()));
     entry->Close();
     ++(*count);
     if (max_to_open >= 0 && static_cast<int>(*count) >= max_to_open)
       break;
   };
 
   return true;
 }
 
+int DiskCacheBackendTest::GetEntryMetadataSize(std::string key) {
+  // For blockfile and memory backends, it is just the key size.
+  if (!simple_cache_mode_)
+    return key.size();
+
+  // For the simple cache, we must add the file header and EOF, and that for
+  // every stream.

[gavinp, 2015/10/13 15:50:12]

Is this right? We now store stream 0 and stream 1 in the same file, yes? And we
don't store stream 2 if there's no data in it. Tricky.

I realize you're just copy/pasting SimpleEntryImpl::GetDiskUsage() here.

How important is it to be byte-perfect here?

[msramek, 2015/10/13 17:25:50]

In DiskCacheBackendTest::BackendCalculateSizeOfAllEntries() where I alternate
between writing to streams 0 and 1, I tried writing to all three streams
instead, and the tests still pass. Which means that GetEntryMetadataSize() is
correct in the sense that

sum for entry in cache
  GetEntryMetadataSize() + sum for stream in entry
                             stream data size

equals CalculateSizeOfAllEntries(), which is SimpleIndex::cache_size_ in the
case of simple cache.

Now if I understand your concern correctly, GetDiskUsage() does not check
whether stream 2 is empty, and we use GetDiskUsage() in UpdateEntrySize(). This
means that the cache pretends that the file for stream 2 always exists?

It would be nice to be more precise with this computation, to give the user the
exact disk size freed when deleting the cache, but if the cache itself thinks
that it's larger than it really is, well.. I think with ~250 bytes per entry the
relative error should be low enough that it's OK.

+  return disk_cache::kSimpleEntryStreamCount * (
+      sizeof(disk_cache::SimpleFileHeader) +
+      sizeof(disk_cache::SimpleFileEOF) +
+      key.size());
+}
+
 void DiskCacheBackendTest::BackendBasics() {
   InitCache();
   disk_cache::Entry *entry1 = NULL, *entry2 = NULL;
   EXPECT_NE(net::OK, OpenEntry("the first key", &entry1));
   ASSERT_EQ(net::OK, CreateEntry("the first key", &entry1));
   ASSERT_TRUE(NULL != entry1);
   entry1->Close();
   entry1 = NULL;
 
   ASSERT_EQ(net::OK, OpenEntry("the first key", &entry1));
@@ skipping 1377 matching lines @@
   start = end;
   end = base::Time::Now();
   DoomEntriesBetween(start, end);
   EXPECT_EQ(3, cache_->GetEntryCount());
 }
 
 void DiskCacheBackendTest::BackendCalculateSizeOfAllEntries() {
   InitCache();
 
   // The cache is initially empty.
-  if (memory_only_ || simple_cache_mode_) {
-    // TODO(msramek): Implement.
-    EXPECT_EQ(net::ERR_NOT_IMPLEMENTED, CalculateSizeOfAllEntries());
-  } else {
-    EXPECT_EQ(0, CalculateSizeOfAllEntries());
-  }
+  EXPECT_EQ(0, CalculateSizeOfAllEntries());
 
   // Generate random entries and populate them with data of respective
   // sizes 0, 1, ..., count - 1 bytes.
   std::set<std::string> key_pool;
   CreateSetOfRandomEntries(&key_pool);
 
   int count = 0;
   for (std::string key : key_pool) {
     std::string data(count, ' ');
     scoped_refptr<net::StringIOBuffer> buffer = new net::StringIOBuffer(data);
 
     // Alternate between writing to the first and second stream to test that
-    // we are not taking just the first stream into account.
+    // we are not taking just the first stream into account. For convenience,
+    // the last written stream should be 0. This is because writing to
+    // the stream 1 in simple cache triggers a write to the stream 0 as well.
+    // This will happen asynchronously and possibly later than our call to
+    // |CalculateSizeOfAllEntries|.
     disk_cache::Entry* entry;
     ASSERT_EQ(net::OK, OpenEntry(key, &entry));
-    ASSERT_EQ(count, WriteData(entry, count % 2, 0, buffer.get(), count, true));
+    ASSERT_EQ(count,
+              WriteData(entry, (count + 1) % 2, 0, buffer.get(), count, true));
     entry->Close();
 
     ++count;
   }
 
   // The resulting size should be (0 + 1 + ... + count - 1) plus keys.
   int result = CalculateSizeOfAllEntries();
-  if (memory_only_ || simple_cache_mode_) {
-    // TODO(msramek): Implement.
-    EXPECT_EQ(net::ERR_NOT_IMPLEMENTED, result);
-  } else {
-    int total_key_size = 0;
-    for (std::string key : key_pool)
-      total_key_size += key.size();
-
-    EXPECT_EQ((count - 1) * count / 2 + total_key_size, result);
-  }
+  int total_metadata_size = 0;
+  for (std::string key : key_pool)
+    total_metadata_size += GetEntryMetadataSize(key);
+  EXPECT_EQ((count - 1) * count / 2 + total_metadata_size, result);
 
   // Add another entry and test if the size is updated. Then remove it and test
   // if the size is back to original value.
   {
     const int last_entry_size = 47;
     std::string data(last_entry_size, ' ');
     scoped_refptr<net::StringIOBuffer> buffer = new net::StringIOBuffer(data);
 
     disk_cache::Entry* entry;
     std::string key = GenerateKey(true);
     ASSERT_EQ(net::OK, CreateEntry(key, &entry));
     ASSERT_EQ(last_entry_size,
               WriteData(entry, 0, 0, buffer.get(), last_entry_size, true));
     entry->Close();
 
     int new_result = CalculateSizeOfAllEntries();
-    if (memory_only_ || simple_cache_mode_) {
-      // TODO(msramek): Implement.
-      EXPECT_EQ(net::ERR_NOT_IMPLEMENTED, new_result);
-    } else {
-      EXPECT_EQ(result + last_entry_size + static_cast<int>(key.size()),
-                new_result);
-    }
+    EXPECT_EQ(result + last_entry_size + GetEntryMetadataSize(key),
+              new_result);
 
     DoomEntry(key);
     new_result = CalculateSizeOfAllEntries();
-    if (memory_only_ || simple_cache_mode_) {
-      // TODO(msramek): Implement.
-      EXPECT_EQ(net::ERR_NOT_IMPLEMENTED, new_result);
-    } else {
-      EXPECT_EQ(result, new_result);
-    }
+    EXPECT_EQ(result, new_result);
   }
 
   // After dooming the entries, the size should be back to zero.
   ASSERT_EQ(net::OK, DoomAllEntries());
-  if (memory_only_ || simple_cache_mode_) {
-    // TODO(msramek): Implement.
-    EXPECT_EQ(net::ERR_NOT_IMPLEMENTED, CalculateSizeOfAllEntries());
-  } else {
-    EXPECT_EQ(0, CalculateSizeOfAllEntries());
-  }
+  EXPECT_EQ(0, CalculateSizeOfAllEntries());
 }
 
 TEST_F(DiskCacheBackendTest, CalculateSizeOfAllEntries) {
   BackendCalculateSizeOfAllEntries();
 }
 
 TEST_F(DiskCacheBackendTest, MemoryOnlyCalculateSizeOfAllEntries) {
   SetMemoryOnlyMode();
   BackendCalculateSizeOfAllEntries();
 }
@@ skipping 1852 matching lines @@
 // after closing.
 // NOTE: IF THIS TEST IS FLAKY THEN IT IS FAILING. See https://crbug.com/416940
 TEST_F(DiskCacheBackendTest, SimpleCacheDeleteQuickly) {
   SetSimpleCacheMode();
   for (int i = 0; i < 100; ++i) {
     InitCache();
     cache_.reset();
     EXPECT_TRUE(CleanupCacheDir());
   }
 }