Write out CacheStorageCache size to index file.

content/browser/cache_storage/cache_storage_manager_unittest.cc

Index: content/browser/cache_storage/cache_storage_manager_unittest.cc
diff --git a/content/browser/cache_storage/cache_storage_manager_unittest.cc b/content/browser/cache_storage/cache_storage_manager_unittest.cc
index 90601a61139287f68e553ee359318b311596a198..e7e09b530da9fde021f7c41b5b332cb9907ea255 100644
--- a/content/browser/cache_storage/cache_storage_manager_unittest.cc
+++ b/content/browser/cache_storage/cache_storage_manager_unittest.cc
@@ -7,9 +7,11 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <list>
 #include <set>
 #include <utility>
 
+#include "base/files/file_enumerator.h"
 #include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/files/scoped_temp_dir.h"
@@ -22,8 +24,10 @@
 #include "base/strings/string_number_conversions.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "content/browser/blob_storage/chrome_blob_storage_context.h"
+#include "content/browser/cache_storage/cache_storage.h"
 #include "content/browser/cache_storage/cache_storage.pb.h"
 #include "content/browser/cache_storage/cache_storage_cache_handle.h"
+#include "content/browser/cache_storage/cache_storage_index.h"
 #include "content/browser/cache_storage/cache_storage_quota_client.h"
 #include "content/browser/quota/mock_quota_manager_proxy.h"
 #include "content/public/browser/browser_thread.h"
@@ -44,6 +48,31 @@
 
 namespace content {
 
+namespace {
+
+bool IsIndexFileCurrent(const base::FilePath& cache_dir) {
+  base::File::Info info;
+  const base::FilePath index_path =
+      cache_dir.AppendASCII(CacheStorage::kIndexFileName);
+  if (!GetFileInfo(index_path, &info))
+    return false;
+  base::Time index_last_modified = info.last_modified;
+
+  base::FileEnumerator enumerator(cache_dir, false,
+                                  base::FileEnumerator::DIRECTORIES);
+  for (base::FilePath file_path = enumerator.Next(); !file_path.empty();
+       file_path = enumerator.Next()) {
+    if (!GetFileInfo(file_path, &info))
+      return false;
+    if (index_last_modified < info.last_modified)
+      return false;
+  }
+
+  return true;
+}
+
+}  // anonymous namespace
+
 // Returns a BlobProtocolHandler that uses |blob_storage_context|. Caller owns
 // the memory.
 std::unique_ptr<storage::BlobProtocolHandler> CreateMockBlobProtocolHandler(
@@ -65,6 +94,69 @@ class CacheStorageManagerTest : public testing::Test {
         origin2_("http://example2.com") {}
 
   void SetUp() override {
+    base::FilePath temp_dir_path;
+    const bool is_incognito = MemoryOnly();

[jkarlin, 2016/12/14 14:19:56]

why this variable?

[cmumford, 2016/12/15 22:29:15]

It was in origin/master (and refactored). Removed it.

+    if (!is_incognito)
+      ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
+
+    CreateStorageManager();
+  }
+
+  void TearDown() override { DestroyStorageManager(); }
+
+  virtual bool MemoryOnly() { return false; }
+
+  void BoolAndErrorCallback(base::RunLoop* run_loop,
+                            bool value,
+                            CacheStorageError error) {
+    callback_bool_ = value;
+    callback_error_ = error;
+    run_loop->Quit();
+  }
+
+  void CacheAndErrorCallback(
+      base::RunLoop* run_loop,
+      std::unique_ptr<CacheStorageCacheHandle> cache_handle,
+      CacheStorageError error) {
+    callback_cache_handle_ = std::move(cache_handle);
+    callback_error_ = error;
+    run_loop->Quit();
+  }
+
+  void CacheMetadataCallback(base::RunLoop* run_loop,
+                             const CacheStorageIndex& cache_index) {
+    callback_cache_index_ = cache_index;
+    run_loop->Quit();
+  }
+
+  const std::string& GetFirstIndexName() const {
+    return callback_cache_index_.ordered_cache_metadata().front().name;
+  }
+
+  std::vector<std::string> GetIndexNames() const {
+    std::vector<std::string> cache_names;
+    for (const auto& metadata : callback_cache_index_.ordered_cache_metadata())
+      cache_names.push_back(metadata.name);
+    return cache_names;
+  }
+
+  void CachePutCallback(base::RunLoop* run_loop, CacheStorageError error) {
+    callback_error_ = error;
+    run_loop->Quit();
+  }
+
+  void CacheMatchCallback(
+      base::RunLoop* run_loop,
+      CacheStorageError error,
+      std::unique_ptr<ServiceWorkerResponse> response,
+      std::unique_ptr<storage::BlobDataHandle> blob_data_handle) {
+    callback_error_ = error;
+    callback_cache_handle_response_ = std::move(response);
+    callback_data_handle_ = std::move(blob_data_handle);
+    run_loop->Quit();
+  }
+
+  void CreateStorageManager() {
     ChromeBlobStorageContext* blob_storage_context(
         ChromeBlobStorageContext::GetFor(&browser_context_));
     // Wait for ChromeBlobStorageContext to finish initializing.
@@ -77,17 +169,16 @@ class CacheStorageManagerTest : public testing::Test {
         "blob", CreateMockBlobProtocolHandler(blob_storage_context->context()));
 
     net::URLRequestContext* url_request_context =
-        BrowserContext::GetDefaultStoragePartition(&browser_context_)->
-              GetURLRequestContext()->GetURLRequestContext();
+        BrowserContext::GetDefaultStoragePartition(&browser_context_)
+            ->GetURLRequestContext()
+            ->GetURLRequestContext();

[jkarlin, 2016/12/14 14:19:56]

Huh, it seems strange that 'git cl format' would do this.

[cmumford, 2016/12/15 22:29:15]

Agreed - I manually reverted prior version.

 
     url_request_context->set_job_factory(url_request_job_factory_.get());
 
-    const bool is_incognito = MemoryOnly();
     base::FilePath temp_dir_path;
-    if (!is_incognito) {
-      ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
+    const bool is_incognito = MemoryOnly();

[jkarlin, 2016/12/14 14:19:56]

Ah, that's why. Please remove "is_incognito" here as well.

[cmumford, 2016/12/15 22:29:15]

Done.

+    if (!is_incognito)
       temp_dir_path = temp_dir_.GetPath();
-    }
 
     quota_policy_ = new MockSpecialStoragePolicy;
     mock_quota_manager_ = new MockQuotaManager(
@@ -106,55 +197,39 @@ class CacheStorageManagerTest : public testing::Test {
         quota_manager_proxy_);
 
     cache_manager_->SetBlobParametersForCache(
-        BrowserContext::GetDefaultStoragePartition(&browser_context_)->
-            GetURLRequestContext(),
+        BrowserContext::GetDefaultStoragePartition(&browser_context_)
+            ->GetURLRequestContext(),
         blob_storage_context->context()->AsWeakPtr());
   }
 
-  void TearDown() override {
-    quota_manager_proxy_->SimulateQuotaManagerDestroyed();
+  bool FlushCacheStorageIndex(const GURL& origin) {
+    bool write_scheduled =
+        CacheStorageForOrigin(origin)->InitiateScheduledIndexWriteForTest();
+    // Wait for write to complete.
     base::RunLoop().RunUntilIdle();
+    return write_scheduled;
   }
 
-  virtual bool MemoryOnly() { return false; }
-
-  void BoolAndErrorCallback(base::RunLoop* run_loop,
-                            bool value,
-                            CacheStorageError error) {
-    callback_bool_ = value;
-    callback_error_ = error;
-    run_loop->Quit();
-  }
+  void DestroyStorageManager() {
+    quota_manager_proxy_->SimulateQuotaManagerDestroyed();
+    base::RunLoop().RunUntilIdle();
+    quota_manager_proxy_ = nullptr;
 
-  void CacheAndErrorCallback(
-      base::RunLoop* run_loop,
-      std::unique_ptr<CacheStorageCacheHandle> cache_handle,
-      CacheStorageError error) {
-    callback_cache_handle_ = std::move(cache_handle);
-    callback_error_ = error;
-    run_loop->Quit();
-  }
+    url_request_job_factory_.reset();
+    blob_storage_context_ = nullptr;
 
-  void StringsCallback(base::RunLoop* run_loop,
-                       const std::vector<std::string>& strings) {
-    callback_strings_ = strings;
-    run_loop->Quit();
-  }
+    quota_policy_ = nullptr;
+    mock_quota_manager_ = nullptr;
+    quota_manager_proxy_ = nullptr;
 
-  void CachePutCallback(base::RunLoop* run_loop, CacheStorageError error) {
-    callback_error_ = error;
-    run_loop->Quit();
-  }
+    callback_cache_handle_ = nullptr;
+    callback_bool_ = false;
+    callback_cache_handle_response_ = nullptr;
+    callback_data_handle_ = nullptr;
+    callback_cache_index_ = CacheStorageIndex();
+    callback_all_origins_usage_.clear();
 
-  void CacheMatchCallback(
-      base::RunLoop* run_loop,
-      CacheStorageError error,
-      std::unique_ptr<ServiceWorkerResponse> response,
-      std::unique_ptr<storage::BlobDataHandle> blob_data_handle) {
-    callback_error_ = error;
-    callback_cache_handle_response_ = std::move(response);
-    callback_data_handle_ = std::move(blob_data_handle);
-    run_loop->Quit();
+    cache_manager_ = nullptr;
   }
 
   bool Open(const GURL& origin, const std::string& cache_name) {
@@ -198,10 +273,10 @@ class CacheStorageManagerTest : public testing::Test {
   size_t Keys(const GURL& origin) {
     base::RunLoop loop;
     cache_manager_->EnumerateCaches(
-        origin, base::Bind(&CacheStorageManagerTest::StringsCallback,
+        origin, base::Bind(&CacheStorageManagerTest::CacheMetadataCallback,
                            base::Unretained(this), base::Unretained(&loop)));
     loop.Run();
-    return callback_strings_.size();
+    return callback_cache_index_.num_entries();
   }
 
   bool StorageMatch(const GURL& origin,
@@ -393,7 +468,7 @@ class CacheStorageManagerTest : public testing::Test {
   CacheStorageError callback_error_;
   std::unique_ptr<ServiceWorkerResponse> callback_cache_handle_response_;
   std::unique_ptr<storage::BlobDataHandle> callback_data_handle_;
-  std::vector<std::string> callback_strings_;
+  CacheStorageIndex callback_cache_index_;
 
   const GURL origin1_;
   const GURL origin2_;
@@ -506,9 +581,9 @@ TEST_P(CacheStorageManagerTestP, SomeKeys) {
   std::vector<std::string> expected_keys;
   expected_keys.push_back("foo");
   expected_keys.push_back("bar");
-  EXPECT_EQ(expected_keys, callback_strings_);
+  EXPECT_EQ(expected_keys, GetIndexNames());
   EXPECT_EQ(1u, Keys(origin2_));
-  EXPECT_STREQ("baz", callback_strings_[0].c_str());
+  EXPECT_STREQ("baz", GetFirstIndexName().c_str());
 }
 
 TEST_P(CacheStorageManagerTestP, DeletedKeysGone) {
@@ -517,7 +592,7 @@ TEST_P(CacheStorageManagerTestP, DeletedKeysGone) {
   EXPECT_TRUE(Open(origin2_, "baz"));
   EXPECT_TRUE(Delete(origin1_, "bar"));
   EXPECT_EQ(1u, Keys(origin1_));
-  EXPECT_STREQ("foo", callback_strings_[0].c_str());
+  EXPECT_STREQ("foo", GetFirstIndexName().c_str());
 }
 
 TEST_P(CacheStorageManagerTestP, StorageMatchEntryExists) {
@@ -663,7 +738,7 @@ TEST_P(CacheStorageManagerTestP, Chinese) {
   EXPECT_TRUE(Open(origin1_, "你好"));
   EXPECT_EQ(callback_cache_handle_->value(), cache_handle->value());
   EXPECT_EQ(1u, Keys(origin1_));
-  EXPECT_STREQ("你好", callback_strings_[0].c_str());
+  EXPECT_STREQ("你好", GetFirstIndexName().c_str());
 }
 
 TEST_F(CacheStorageManagerTest, EmptyKey) {
@@ -673,7 +748,7 @@ TEST_F(CacheStorageManagerTest, EmptyKey) {
   EXPECT_TRUE(Open(origin1_, ""));
   EXPECT_EQ(cache_handle->value(), callback_cache_handle_->value());
   EXPECT_EQ(1u, Keys(origin1_));
-  EXPECT_STREQ("", callback_strings_[0].c_str());
+  EXPECT_STREQ("", GetFirstIndexName().c_str());
   EXPECT_TRUE(Has(origin1_, ""));
   EXPECT_TRUE(Delete(origin1_, ""));
   EXPECT_EQ(0u, Keys(origin1_));
@@ -691,7 +766,7 @@ TEST_F(CacheStorageManagerTest, DataPersists) {
   std::vector<std::string> expected_keys;
   expected_keys.push_back("foo");
   expected_keys.push_back("baz");
-  EXPECT_EQ(expected_keys, callback_strings_);
+  EXPECT_EQ(expected_keys, GetIndexNames());
 }
 
 TEST_F(CacheStorageManagerMemoryOnlyTest, DataLostWhenMemoryOnly) {
@@ -708,7 +783,7 @@ TEST_F(CacheStorageManagerTest, BadCacheName) {
   const std::string bad_name = "../../../../../../../../../../../../../../foo";
   EXPECT_TRUE(Open(origin1_, bad_name));
   EXPECT_EQ(1u, Keys(origin1_));
-  EXPECT_STREQ(bad_name.c_str(), callback_strings_[0].c_str());
+  EXPECT_STREQ(bad_name.c_str(), GetFirstIndexName().c_str());
 }
 
 TEST_F(CacheStorageManagerTest, BadOriginName) {
@@ -717,7 +792,7 @@ TEST_F(CacheStorageManagerTest, BadOriginName) {
   GURL bad_origin("http://../../../../../../../../../../../../../../foo");
   EXPECT_TRUE(Open(bad_origin, "foo"));
   EXPECT_EQ(1u, Keys(bad_origin));
-  EXPECT_STREQ("foo", callback_strings_[0].c_str());
+  EXPECT_STREQ("foo", GetFirstIndexName().c_str());
 }
 
 // With a persistent cache if the client drops its reference to a
@@ -860,6 +935,125 @@ TEST_P(CacheStorageManagerTestP, GetAllOriginsUsage) {
   }
 }
 
+TEST_F(CacheStorageManagerTest, GetAllOriginsUsageWithOldIndex) {

[jkarlin, 2016/12/14 14:19:56]

Add a simple test to show the expected case, e.g., add a resource, get its size,
close the manager, reopen the manager, and the size is correct.

[cmumford, 2016/12/15 22:29:15]

Done - added CacheSizeCorrectAfterReopen.

+  // Write a single value (V1) to the cache.
+  const GURL kFooURL = origin1_.Resolve("foo");
+  const std::string kCacheName = "foo";
+  EXPECT_TRUE(Open(origin1_, kCacheName));
+  std::unique_ptr<CacheStorageCacheHandle> original_handle =
+      std::move(callback_cache_handle_);
+
+  EXPECT_TRUE(CachePut(original_handle->value(), kFooURL));
+  int64_t cache_size_v1 = Size(origin1_);
+  base::FilePath storage_dir = original_handle->value()->path().DirName();
+  original_handle.release();
+  EXPECT_GE(cache_size_v1, 0);
+
+  // Close the caches and cache manager.
+  EXPECT_TRUE(FlushCacheStorageIndex(origin1_));
+  DestroyStorageManager();
+
+  // Save a copy of the V1 index.
+  EXPECT_TRUE(IsIndexFileCurrent(storage_dir));
+  base::FilePath index_path = storage_dir.AppendASCII("index.txt");
+  EXPECT_TRUE(base::PathExists(index_path));
+  base::FilePath backup_index_path = storage_dir.AppendASCII("index.txt.bak");
+  EXPECT_TRUE(base::CopyFile(index_path, backup_index_path));
+
+  // Create a new CacheStorageManager that hasn't yet loaded the origin.
+  CreateStorageManager();
+  quota_manager_proxy_->SimulateQuotaManagerDestroyed();
+  cache_manager_ = CacheStorageManager::Create(cache_manager_.get());
+
+  // Create a second value (V2) in the cache.
+  EXPECT_TRUE(Open(origin1_, kCacheName));
+  original_handle = std::move(callback_cache_handle_);
+  const GURL kBarURL = origin1_.Resolve("bar");
+  EXPECT_TRUE(CachePut(original_handle->value(), kBarURL));
+  original_handle.release();
+
+  std::vector<CacheStorageUsageInfo> usage = GetAllOriginsUsage();
+  ASSERT_EQ(1ULL, usage.size());
+  int64_t usage_before_close = usage[0].total_size_bytes;
+  EXPECT_GT(usage_before_close, 0);
+
+  // Close the caches and cache manager.
+  DestroyStorageManager();
+
+  // Restore the index to the V1 state. Make the access/mod times of index file
+  // older than the other directories in the store to trigger size
+  // recalculation.
+  EXPECT_TRUE(base::CopyFile(backup_index_path, index_path));
+  base::Time t = base::Time::Now() - base::TimeDelta::FromHours(1);
+  EXPECT_TRUE(base::TouchFile(index_path, t, t));
+  EXPECT_FALSE(IsIndexFileCurrent(storage_dir));
+
+  CreateStorageManager();
+  usage = GetAllOriginsUsage();
+  ASSERT_EQ(1ULL, usage.size());
+
+  EXPECT_EQ(usage_before_close, usage[0].total_size_bytes);
+
+  EXPECT_FALSE(usage[0].last_modified.is_null());
+}
+
+TEST_F(CacheStorageManagerTest, GetOriginSizeWithOldIndex) {
+  // Write a single value (V1) to the cache.
+  const GURL kFooURL = origin1_.Resolve("foo");
+  const std::string kCacheName = "foo";
+  EXPECT_TRUE(Open(origin1_, kCacheName));
+  std::unique_ptr<CacheStorageCacheHandle> original_handle =
+      std::move(callback_cache_handle_);
+
+  EXPECT_TRUE(CachePut(original_handle->value(), kFooURL));
+  int64_t cache_size_v1 = Size(origin1_);
+  base::FilePath storage_dir = original_handle->value()->path().DirName();
+  original_handle.release();
+  EXPECT_GE(cache_size_v1, 0);
+
+  // Close the caches and cache manager.
+  EXPECT_TRUE(FlushCacheStorageIndex(origin1_));
+  DestroyStorageManager();
+
+  // Save a copy of the V1 index.
+  EXPECT_TRUE(IsIndexFileCurrent(storage_dir));
+  base::FilePath index_path = storage_dir.AppendASCII("index.txt");
+  EXPECT_TRUE(base::PathExists(index_path));
+  base::FilePath backup_index_path = storage_dir.AppendASCII("index.txt.bak");
+  EXPECT_TRUE(base::CopyFile(index_path, backup_index_path));
+
+  // Create a new CacheStorageManager that hasn't yet loaded the origin.
+  CreateStorageManager();
+  quota_manager_proxy_->SimulateQuotaManagerDestroyed();
+  cache_manager_ = CacheStorageManager::Create(cache_manager_.get());
+
+  // Reopen the cache and write a second value (V2).
+  EXPECT_TRUE(Open(origin1_, kCacheName));
+  original_handle = std::move(callback_cache_handle_);
+  const GURL kBarURL = origin1_.Resolve("bar");
+  EXPECT_TRUE(CachePut(original_handle->value(), kBarURL));
+  original_handle.release();
+  int64_t cache_size_v2 = Size(origin1_);
+  EXPECT_GE(cache_size_v2, 0);
+
+  // Close the caches and cache manager.
+  DestroyStorageManager();
+
+  // Restore the index to the V1 state.
+  EXPECT_TRUE(base::CopyFile(backup_index_path, index_path));
+
+  // Make the access/mod times of index file older than the other files in the
+  // cache to trigger size recalculation.
+  base::Time t = base::Time::Now() - base::TimeDelta::FromHours(1);
+  EXPECT_TRUE(base::TouchFile(index_path, t, t));
+  EXPECT_FALSE(IsIndexFileCurrent(storage_dir));
+
+  // Reopen the cache and ensure the size is correct for the V2 value.
+  CreateStorageManager();
+  EXPECT_TRUE(Open(origin1_, kCacheName));
+  EXPECT_EQ(cache_size_v2, Size(origin1_));
+}
+
 TEST_P(CacheStorageManagerTestP, GetSizeThenCloseAllCaches) {
   EXPECT_TRUE(Open(origin1_, "foo"));
   EXPECT_TRUE(CachePut(callback_cache_handle_->value(),