Write out CacheStorageCache size to index file.

content/browser/cache_storage/cache_storage.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/cache_storage/cache_storage.h"
 
 #include <stddef.h>
 
 #include <set>
 #include <string>
@@ skipping 10 matching lines @@
 #include "base/numerics/safe_conversions.h"
 #include "base/sha1.h"
 #include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "content/browser/cache_storage/cache_storage.pb.h"
 #include "content/browser/cache_storage/cache_storage_cache.h"
 #include "content/browser/cache_storage/cache_storage_cache_handle.h"
+#include "content/browser/cache_storage/cache_storage_index.h"
 #include "content/browser/cache_storage/cache_storage_scheduler.h"
 #include "content/public/browser/browser_thread.h"
 #include "net/base/directory_lister.h"
 #include "net/base/net_errors.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "storage/browser/blob/blob_storage_context.h"
 #include "storage/browser/quota/quota_manager_proxy.h"
 
 namespace content {
 
 namespace {
 
 std::string HexedHash(const std::string& value) {
   std::string value_hash = base::SHA1HashString(value);
   std::string valued_hexed_hash = base::ToLowerASCII(
       base::HexEncode(value_hash.c_str(), value_hash.length()));
   return valued_hexed_hash;
 }
 
-void SizeRetrievedFromCache(
-    std::unique_ptr<CacheStorageCacheHandle> cache_handle,
-    const base::Closure& closure,
-    int64_t* accumulator,
-    int64_t size) {
-  *accumulator += size;
-  closure.Run();
-}
-
 void SizeRetrievedFromAllCaches(std::unique_ptr<int64_t> accumulator,
                                 const CacheStorage::SizeCallback& callback) {
   base::ThreadTaskRunnerHandle::Get()->PostTask(
       FROM_HERE, base::Bind(callback, *accumulator));
 }
 
+void DoNothingWithBool(bool success) {}
+
 }  // namespace
 
 const char CacheStorage::kIndexFileName[] = "index.txt";
+const int64_t CacheStorage::kSizeUnknown;
 
 struct CacheStorage::CacheMatchResponse {
   CacheMatchResponse() = default;
   ~CacheMatchResponse() = default;
 
   CacheStorageError error;
   std::unique_ptr<ServiceWorkerResponse> service_worker_response;
   std::unique_ptr<storage::BlobDataHandle> blob_data_handle;
 };
 
 // Handles the loading and clean up of CacheStorageCache objects.
 class CacheStorage::CacheLoader {
  public:
   typedef base::Callback<void(std::unique_ptr<CacheStorageCache>)>
       CacheCallback;
   typedef base::Callback<void(bool)> BoolCallback;
-  typedef base::Callback<void(std::unique_ptr<std::vector<std::string>>)>
-      StringVectorCallback;
+  using CacheStorageIndexLoadCallback =
+      base::Callback<void(std::unique_ptr<CacheStorageIndex>)>;
 
   CacheLoader(
       base::SequencedTaskRunner* cache_task_runner,
       scoped_refptr<net::URLRequestContextGetter> request_context_getter,
       storage::QuotaManagerProxy* quota_manager_proxy,
       base::WeakPtr<storage::BlobStorageContext> blob_context,
       CacheStorage* cache_storage,
       const GURL& origin)
       : cache_task_runner_(cache_task_runner),
         request_context_getter_(request_context_getter),
         quota_manager_proxy_(quota_manager_proxy),
         blob_context_(blob_context),
         cache_storage_(cache_storage),
         origin_(origin) {
     DCHECK(!origin_.is_empty());
   }
 
   virtual ~CacheLoader() {}
 
   // Creates a CacheStorageCache with the given name. It does not attempt to
   // load the backend, that happens lazily when the cache is used.
   virtual std::unique_ptr<CacheStorageCache> CreateCache(
-      const std::string& cache_name) = 0;
+      const std::string& cache_name,
+      int64_t cache_size) = 0;
 
   // Deletes any pre-existing cache of the same name and then loads it.
   virtual void PrepareNewCacheDestination(const std::string& cache_name,
                                           const CacheCallback& callback) = 0;
 
   // After the backend has been deleted, do any extra house keeping such as
   // removing the cache's directory.
   virtual void CleanUpDeletedCache(CacheStorageCache* cache) = 0;
 
-  // Writes the cache names (and sizes) to disk if applicable.
-  virtual void WriteIndex(const StringVector& cache_names,
+  // Writes the cache index to disk if applicable.
+  virtual void WriteIndex(const CacheStorageIndex& index,
                           const BoolCallback& callback) = 0;
 
-  // Loads the cache names from disk if applicable.
-  virtual void LoadIndex(std::unique_ptr<std::vector<std::string>> cache_names,
-                         const StringVectorCallback& callback) = 0;
+  // Loads the cache index from disk if applicable.
+  virtual void LoadIndex(const CacheStorageIndexLoadCallback& callback) = 0;
 
   // Called when CacheStorage has created a cache. Used to hold onto a handle to
   // the cache if necessary.
   virtual void NotifyCacheCreated(
       const std::string& cache_name,
       std::unique_ptr<CacheStorageCacheHandle> cache_handle) {}
 
   // Notification that the cache for |cache_handle| has been doomed. If the
   // loader is holding a handle to the cache, it should drop it now.
   virtual void NotifyCacheDoomed(
@@ skipping 26 matching lines @@
                base::WeakPtr<storage::BlobStorageContext> blob_context,
                CacheStorage* cache_storage,
                const GURL& origin)
       : CacheLoader(cache_task_runner,
                     request_context,
                     quota_manager_proxy,
                     blob_context,
                     cache_storage,
                     origin) {}
 
-  std::unique_ptr<CacheStorageCache> CreateCache(
-      const std::string& cache_name) override {
+  std::unique_ptr<CacheStorageCache> CreateCache(const std::string& cache_name,
+                                                 int64_t cache_size) override {
     return CacheStorageCache::CreateMemoryCache(
         origin_, cache_name, cache_storage_, request_context_getter_,
         quota_manager_proxy_, blob_context_);
   }
 
   void PrepareNewCacheDestination(const std::string& cache_name,
                                   const CacheCallback& callback) override {
-    std::unique_ptr<CacheStorageCache> cache = CreateCache(cache_name);
+    std::unique_ptr<CacheStorageCache> cache =
+        CreateCache(cache_name, 0 /*cache_size*/);
     callback.Run(std::move(cache));
   }
 
   void CleanUpDeletedCache(CacheStorageCache* cache) override {}
 
-  void WriteIndex(const StringVector& cache_names,
+  void WriteIndex(const CacheStorageIndex& index,
                   const BoolCallback& callback) override {
     callback.Run(true);
   }
 
-  void LoadIndex(std::unique_ptr<std::vector<std::string>> cache_names,
-                 const StringVectorCallback& callback) override {
-    callback.Run(std::move(cache_names));
+  void LoadIndex(const CacheStorageIndexLoadCallback& callback) override {
+    callback.Run(base::MakeUnique<CacheStorageIndex>());
   }
 
   void NotifyCacheCreated(
       const std::string& cache_name,
       std::unique_ptr<CacheStorageCacheHandle> cache_handle) override {
     DCHECK(!base::ContainsKey(cache_handles_, cache_name));
     cache_handles_.insert(std::make_pair(cache_name, std::move(cache_handle)));
   };
 
   void NotifyCacheDoomed(
@@ skipping 25 matching lines @@
                     const GURL& origin)
       : CacheLoader(cache_task_runner,
                     request_context,
                     quota_manager_proxy,
                     blob_context,
                     cache_storage,
                     origin),
         origin_path_(origin_path),
         weak_ptr_factory_(this) {}
 
-  std::unique_ptr<CacheStorageCache> CreateCache(
-      const std::string& cache_name) override {
+  std::unique_ptr<CacheStorageCache> CreateCache(const std::string& cache_name,
+                                                 int64_t cache_size) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
     DCHECK(base::ContainsKey(cache_name_to_cache_dir_, cache_name));
 
     std::string cache_dir = cache_name_to_cache_dir_[cache_name];
     base::FilePath cache_path = origin_path_.AppendASCII(cache_dir);
     return CacheStorageCache::CreatePersistentCache(
         origin_, cache_name, cache_storage_, cache_path,
-        request_context_getter_, quota_manager_proxy_, blob_context_);
+        request_context_getter_, quota_manager_proxy_, blob_context_,
+        cache_size);
   }
 
   void PrepareNewCacheDestination(const std::string& cache_name,
                                   const CacheCallback& callback) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
     PostTaskAndReplyWithResult(
         cache_task_runner_.get(), FROM_HERE,
         base::Bind(&SimpleCacheLoader::PrepareNewCacheDirectoryInPool,
                    origin_path_),
@@ skipping 16 matching lines @@
 
   void PrepareNewCacheCreateCache(const std::string& cache_name,
                                   const CacheCallback& callback,
                                   const std::string& cache_dir) {
     if (cache_dir.empty()) {
       callback.Run(std::unique_ptr<CacheStorageCache>());
       return;
     }
 
     cache_name_to_cache_dir_[cache_name] = cache_dir;
-    callback.Run(CreateCache(cache_name));
+    callback.Run(CreateCache(cache_name, CacheStorage::kSizeUnknown));
   }
 
   void CleanUpDeletedCache(CacheStorageCache* cache) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
     DCHECK(base::ContainsKey(doomed_cache_to_path_, cache));
 
     base::FilePath cache_path =
         origin_path_.AppendASCII(doomed_cache_to_path_[cache]);
     doomed_cache_to_path_.erase(cache);
 
     cache_task_runner_->PostTask(
         FROM_HERE, base::Bind(&SimpleCacheLoader::CleanUpDeleteCacheDirInPool,
                               cache_path));
   }
 
   static void CleanUpDeleteCacheDirInPool(const base::FilePath& cache_path) {
     base::DeleteFile(cache_path, true /* recursive */);
   }
 
-  void WriteIndex(const StringVector& cache_names,
+  void WriteIndex(const CacheStorageIndex& index,
                   const BoolCallback& callback) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
     // 1. Create the index file as a string. (WriteIndex)
     // 2. Write the file to disk. (WriteIndexWriteToFileInPool)
 
-    proto::CacheStorageIndex index;
-    index.set_origin(origin_.spec());
+    proto::CacheStorageIndex protobuf_index;
+    protobuf_index.set_origin(origin_.spec());
 
-    for (size_t i = 0u, max = cache_names.size(); i < max; ++i) {
-      DCHECK(base::ContainsKey(cache_name_to_cache_dir_, cache_names[i]));
+    for (const auto& cache_metadata : index.ordered_cache_metadata()) {
+      DCHECK(base::ContainsKey(cache_name_to_cache_dir_, cache_metadata.name));
 
-      proto::CacheStorageIndex::Cache* index_cache = index.add_cache();
-      index_cache->set_name(cache_names[i]);
-      index_cache->set_cache_dir(cache_name_to_cache_dir_[cache_names[i]]);
+      proto::CacheStorageIndex::Cache* index_cache = protobuf_index.add_cache();
+      index_cache->set_name(cache_metadata.name);
+      index_cache->set_cache_dir(cache_name_to_cache_dir_[cache_metadata.name]);
+      if (cache_metadata.size == CacheStorage::kSizeUnknown)
+        index_cache->clear_size();
+      else
+        index_cache->set_size(cache_metadata.size);
     }
 
     std::string serialized;
-    bool success = index.SerializeToString(&serialized);
+    bool success = protobuf_index.SerializeToString(&serialized);
     DCHECK(success);
 
     base::FilePath tmp_path = origin_path_.AppendASCII("index.txt.tmp");
     base::FilePath index_path =
         origin_path_.AppendASCII(CacheStorage::kIndexFileName);
 
     PostTaskAndReplyWithResult(
         cache_task_runner_.get(), FROM_HERE,
         base::Bind(&SimpleCacheLoader::WriteIndexWriteToFileInPool, tmp_path,
                    index_path, serialized),
         callback);
   }
 
   static bool WriteIndexWriteToFileInPool(const base::FilePath& tmp_path,
                                           const base::FilePath& index_path,
                                           const std::string& data) {
     int bytes_written = base::WriteFile(tmp_path, data.c_str(), data.size());
     if (bytes_written != base::checked_cast<int>(data.size())) {
       base::DeleteFile(tmp_path, /* recursive */ false);
       return false;
     }
 
     // Atomically rename the temporary index file to become the real one.
     return base::ReplaceFile(tmp_path, index_path, NULL);
   }
 
-  void LoadIndex(std::unique_ptr<std::vector<std::string>> names,
-                 const StringVectorCallback& callback) override {
+  void LoadIndex(const CacheStorageIndexLoadCallback& callback) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
-    // 1. Read the file from disk. (LoadIndexReadFileInPool)
-    // 2. Parse file and return the names of the caches (LoadIndexDidReadFile)
-
-    base::FilePath index_path =
-        origin_path_.AppendASCII(CacheStorage::kIndexFileName);
-
     PostTaskAndReplyWithResult(
         cache_task_runner_.get(), FROM_HERE,
-        base::Bind(&SimpleCacheLoader::ReadAndMigrateIndexInPool, index_path),
-        base::Bind(&SimpleCacheLoader::LoadIndexDidReadFile,
-                   weak_ptr_factory_.GetWeakPtr(), base::Passed(&names),
-                   callback));
+        base::Bind(&SimpleCacheLoader::ReadAndMigrateIndexInPool, origin_path_),
+        base::Bind(&SimpleCacheLoader::LoadIndexDidReadIndex,
+                   weak_ptr_factory_.GetWeakPtr(), callback));
   }
 
-  void LoadIndexDidReadFile(std::unique_ptr<std::vector<std::string>> names,
-                            const StringVectorCallback& callback,
-                            const std::string& serialized) {
+  void LoadIndexDidReadIndex(const CacheStorageIndexLoadCallback& callback,
+                             proto::CacheStorageIndex protobuf_index) {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
     std::unique_ptr<std::set<std::string>> cache_dirs(
         new std::set<std::string>);
 
-    proto::CacheStorageIndex index;
-    if (index.ParseFromString(serialized)) {
-      for (int i = 0, max = index.cache_size(); i < max; ++i) {
-        const proto::CacheStorageIndex::Cache& cache = index.cache(i);
-        DCHECK(cache.has_cache_dir());
-        names->push_back(cache.name());
-        cache_name_to_cache_dir_[cache.name()] = cache.cache_dir();
-        cache_dirs->insert(cache.cache_dir());
-      }
+    auto index = base::MakeUnique<CacheStorageIndex>();
+    for (int i = 0, max = protobuf_index.cache_size(); i < max; ++i) {
+      const proto::CacheStorageIndex::Cache& cache = protobuf_index.cache(i);
+      DCHECK(cache.has_cache_dir());
+      int64_t cache_size =
+          cache.has_size() ? cache.size() : CacheStorage::kSizeUnknown;
+      index->Insert(CacheStorageIndex::CacheMetadata(cache.name(), cache_size));
+      cache_name_to_cache_dir_[cache.name()] = cache.cache_dir();
+      cache_dirs->insert(cache.cache_dir());
     }
 
     cache_task_runner_->PostTask(
         FROM_HERE, base::Bind(&DeleteUnreferencedCachesInPool, origin_path_,
                               base::Passed(&cache_dirs)));
-    callback.Run(std::move(names));
+    callback.Run(std::move(index));
   }
 
   void NotifyCacheDoomed(
       std::unique_ptr<CacheStorageCacheHandle> cache_handle) override {
     DCHECK(base::ContainsKey(cache_name_to_cache_dir_,
                              cache_handle->value()->cache_name()));
     auto iter =
         cache_name_to_cache_dir_.find(cache_handle->value()->cache_name());
     doomed_cache_to_path_[cache_handle->value()] = iter->second;
     cache_name_to_cache_dir_.erase(iter);
@@ skipping 15 matching lines @@
     while (!(cache_path = file_enum.Next()).empty()) {
       if (!base::ContainsKey(*cache_dirs, cache_path.BaseName().AsUTF8Unsafe()))
         dirs_to_delete.push_back(cache_path);
     }
 
     for (const base::FilePath& cache_path : dirs_to_delete)
       base::DeleteFile(cache_path, true /* recursive */);
   }
 
   // Runs on cache_task_runner_
-  static std::string MigrateCachesIfNecessaryInPool(
-      const std::string& body,
-      const base::FilePath& index_path) {
+  static proto::CacheStorageIndex ReadAndMigrateIndexInPool(
+      const base::FilePath& origin_path) {
+    const base::FilePath index_path =
+        origin_path.AppendASCII(CacheStorage::kIndexFileName);
+
     proto::CacheStorageIndex index;
-    if (!index.ParseFromString(body))
-      return body;
+    std::string body;
+    if (!base::ReadFileToString(index_path, &body) ||
+        !index.ParseFromString(body))
+      return proto::CacheStorageIndex();
+    body.clear();
 
-    base::FilePath origin_path = index_path.DirName();
-    bool index_is_dirty = false;
-    const std::string kBadIndexState("");
+    base::File::Info file_info;
+    base::Time index_last_modified;
+    if (GetFileInfo(index_path, &file_info))
+      index_last_modified = file_info.last_modified;
+    bool index_modified = false;
 
     // Look for caches that have no cache_dir. Give any such caches a directory
     // with a random name and move them there. Then, rewrite the index file.
+    // Additionally invalidate any index entries where the cache was modified

[jkarlin, 2016/12/09 19:49:19]

We're just invalidating the size of the index entry right? Not the whole index
entry.

[cmumford, 2016/12/15 22:29:14]

Done.

+    // after the index (making it out-of-date)

[jkarlin, 2016/12/09 19:49:19]

Needs a period.

[cmumford, 2016/12/15 22:29:14]

Done.

     for (int i = 0, max = index.cache_size(); i < max; ++i) {
       const proto::CacheStorageIndex::Cache& cache = index.cache(i);
-      if (!cache.has_cache_dir()) {
+      if (cache.has_cache_dir()) {
+        if (cache.has_size()) {
+          base::FilePath cache_dir = origin_path.AppendASCII(cache.cache_dir());
+          if (!GetFileInfo(cache_dir, &file_info) ||
+              index_last_modified <= file_info.last_modified) {
+            // Index is older than this cache, so invalidate index entries that
+            // may change as a result of cache operations.
+            index.mutable_cache(i)->clear_size();
+          }
+        }
+      } else {
         // Find a new home for the cache.
         base::FilePath legacy_cache_path =
             origin_path.AppendASCII(HexedHash(cache.name()));
         std::string cache_dir;
         base::FilePath cache_path;
         do {
           cache_dir = base::GenerateGUID();
           cache_path = origin_path.AppendASCII(cache_dir);
         } while (base::PathExists(cache_path));
 
         if (!base::Move(legacy_cache_path, cache_path)) {
           // If the move fails then the cache is in a bad state. Return an empty
           // index so that the CacheStorage can start fresh. The unreferenced
           // caches will be discarded later in initialization.
-          return kBadIndexState;
+          return proto::CacheStorageIndex();
         }
 
         index.mutable_cache(i)->set_cache_dir(cache_dir);
-        index_is_dirty = true;
+        index.mutable_cache(i)->clear_size();
+        index_modified = true;
       }
     }
 
-    if (index_is_dirty) {
-      std::string new_body;
-      if (!index.SerializeToString(&new_body))
-        return kBadIndexState;
-      if (base::WriteFile(index_path, new_body.c_str(), new_body.size()) !=
-          base::checked_cast<int>(new_body.size()))
-        return kBadIndexState;
-      return new_body;
+    if (index_modified) {
+      if (!index.SerializeToString(&body))
+        return proto::CacheStorageIndex();
+      if (base::WriteFile(index_path, body.c_str(), body.size()) !=
+          base::checked_cast<int>(body.size()))
+        return proto::CacheStorageIndex();
     }
 
-    return body;
-  }
-
-  // Runs on cache_task_runner_
-  static std::string ReadAndMigrateIndexInPool(
-      const base::FilePath& index_path) {
-    std::string body;
-    base::ReadFileToString(index_path, &body);
-
-    return MigrateCachesIfNecessaryInPool(body, index_path);
+    return index;
   }
 
   const base::FilePath origin_path_;
   std::map<std::string, std::string> cache_name_to_cache_dir_;
   std::map<CacheStorageCache*, std::string> doomed_cache_to_path_;
 
   base::WeakPtrFactory<SimpleCacheLoader> weak_ptr_factory_;
 };
 
 CacheStorage::CacheStorage(
     const base::FilePath& path,
     bool memory_only,
     base::SequencedTaskRunner* cache_task_runner,
     scoped_refptr<net::URLRequestContextGetter> request_context,
     scoped_refptr<storage::QuotaManagerProxy> quota_manager_proxy,
     base::WeakPtr<storage::BlobStorageContext> blob_context,
     const GURL& origin)
     : initialized_(false),
       initializing_(false),
       memory_only_(memory_only),
       scheduler_(new CacheStorageScheduler(
           CacheStorageSchedulerClient::CLIENT_STORAGE)),
+      cache_index_(base::MakeUnique<CacheStorageIndex>()),

[jkarlin, 2016/12/09 19:49:19]

Why create this in the constructor? It's a bug if something tries to use
cache_index_ before its been loaded in init.

[cmumford, 2016/12/15 22:29:14]

Done.

       origin_path_(path),
       cache_task_runner_(cache_task_runner),
       quota_manager_proxy_(quota_manager_proxy),
       origin_(origin),
+      index_write_pending_(false),
       weak_factory_(this) {
   if (memory_only)
     cache_loader_.reset(new MemoryLoader(
         cache_task_runner_.get(), std::move(request_context),
         quota_manager_proxy.get(), blob_context, this, origin));
   else
     cache_loader_.reset(new SimpleCacheLoader(
         origin_path_, cache_task_runner_.get(), std::move(request_context),
         quota_manager_proxy.get(), blob_context, this, origin));
 }
@@ skipping 42 matching lines @@
 
   quota_manager_proxy_->NotifyStorageAccessed(
       storage::QuotaClient::kServiceWorkerCache, origin_,
       storage::kStorageTypeTemporary);
 
   scheduler_->ScheduleOperation(
       base::Bind(&CacheStorage::DeleteCacheImpl, weak_factory_.GetWeakPtr(),
                  cache_name, scheduler_->WrapCallbackToRunNext(callback)));
 }
 
-void CacheStorage::EnumerateCaches(const StringsCallback& callback) {
+void CacheStorage::EnumerateCaches(const IndexCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   if (!initialized_)
     LazyInit();
 
   quota_manager_proxy_->NotifyStorageAccessed(
       storage::QuotaClient::kServiceWorkerCache, origin_,
       storage::kStorageTypeTemporary);
 
   scheduler_->ScheduleOperation(
@@ skipping 55 matching lines @@
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   if (!initialized_)
     LazyInit();
 
   scheduler_->ScheduleOperation(
       base::Bind(&CacheStorage::SizeImpl, weak_factory_.GetWeakPtr(),
                  scheduler_->WrapCallbackToRunNext(callback)));
 }
 
+void CacheStorage::ScheduleWriteIndex() {
+  static const int64_t kWriteIndexDelay = 5;

[jkarlin, 2016/12/09 19:49:19]

kWriteIndexDelaySecs

[cmumford, 2016/12/15 22:29:14]

Done.

+  index_write_task_.Reset(
+      base::Bind(&CacheStorage::WriteIndex, weak_factory_.GetWeakPtr()));
+  cache_task_runner_->PostDelayedTask(
+      FROM_HERE, index_write_task_.callback(),
+      base::TimeDelta::FromSeconds(kWriteIndexDelay));
+  index_write_pending_ = true;
+}
+
+void CacheStorage::WriteIndex() {
+  scheduler_->ScheduleOperation(
+      base::Bind(&CacheStorage::WriteIndexImpl, weak_factory_.GetWeakPtr()));
+}
+
+void CacheStorage::WriteIndexImpl() {
+  index_write_pending_ = false;
+  cache_loader_->WriteIndex(*cache_index_, base::Bind(&DoNothingWithBool));

[jkarlin, 2016/12/09 19:49:19]

The callback needs to call RunNext on the scheduler else the scheduler won't
advance until the next Schedule operation is called. You could use:

scheduler_->WrapCallbackToRunNext(base::Bind(&DoNothingWithBool))

[cmumford, 2016/12/15 22:29:14]

Done. I also tweaked the method names slightly since WriteIndex didn't actually
do the write - it only scheduled one.

+}
+
+bool CacheStorage::InitiateScheduledIndexWriteForTest() {
+  if (index_write_pending_) {
+    index_write_task_.Cancel();
+    WriteIndex();
+    return true;
+  }
+  return false;
+}
+
+void CacheStorage::CacheSizeUpdated(const CacheStorageCache* cache,
+                                    int64_t size) {
+  // If the cache is doomed (but still running), ignore any size updates.
+  if (base::ContainsKey(doomed_caches_, const_cast<CacheStorageCache*>(cache)))
+    return;
+  cache_index_->SetCacheSize(cache->cache_name(), size);
+  ScheduleWriteIndex();
+}
+
 void CacheStorage::StartAsyncOperationForTesting() {
   scheduler_->ScheduleOperation(base::Bind(&base::DoNothing));
 }
 
 void CacheStorage::CompleteAsyncOperationForTesting() {
   scheduler_->CompleteOperationAndRunNext();
 }
 
 // Init is run lazily so that it is called on the proper MessageLoop.
 void CacheStorage::LazyInit() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(!initialized_);
 
   if (initializing_)
     return;
 
   DCHECK(!scheduler_->ScheduledOperations());
 
   initializing_ = true;
   scheduler_->ScheduleOperation(
       base::Bind(&CacheStorage::LazyInitImpl, weak_factory_.GetWeakPtr()));
 }
 
 void CacheStorage::LazyInitImpl() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(!initialized_);
   DCHECK(initializing_);
 
-  // 1. Get the list of cache names (async call)
+  // 1. Get the cache index (async call)
   // 2. For each cache name, load the cache (async call)
   // 3. Once each load is complete, update the map variables.
   // 4. Call the list of waiting callbacks.
 
-  std::unique_ptr<std::vector<std::string>> indexed_cache_names(
-      new std::vector<std::string>());
-
-  cache_loader_->LoadIndex(std::move(indexed_cache_names),
-                           base::Bind(&CacheStorage::LazyInitDidLoadIndex,
+  cache_loader_->LoadIndex(base::Bind(&CacheStorage::LazyInitDidLoadIndex,
                                       weak_factory_.GetWeakPtr()));
 }
 
 void CacheStorage::LazyInitDidLoadIndex(
-    std::unique_ptr<std::vector<std::string>> indexed_cache_names) {
+    std::unique_ptr<CacheStorageIndex> index) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(cache_map_.empty());
 
-  for (size_t i = 0u, max = indexed_cache_names->size(); i < max; ++i) {
-    cache_map_.insert(std::make_pair(indexed_cache_names->at(i),
+  for (const auto& cache_metadata : index->ordered_cache_metadata()) {
+    cache_map_.insert(std::make_pair(cache_metadata.name,
                                      std::unique_ptr<CacheStorageCache>()));
-    ordered_cache_names_.push_back(indexed_cache_names->at(i));
   }
 
+  cache_index_ = std::move(index);
+
   initializing_ = false;
   initialized_ = true;
 
   scheduler_->CompleteOperationAndRunNext();
 }
 
 void CacheStorage::OpenCacheImpl(const std::string& cache_name,
                                  const CacheAndErrorCallback& callback) {
   std::unique_ptr<CacheStorageCacheHandle> cache_handle =
       GetLoadedCache(cache_name);
@@ skipping 18 matching lines @@
 
   if (!cache) {
     callback.Run(std::unique_ptr<CacheStorageCacheHandle>(),
                  CACHE_STORAGE_ERROR_STORAGE);
     return;
   }
 
   CacheStorageCache* cache_ptr = cache.get();
 
   cache_map_.insert(std::make_pair(cache_name, std::move(cache)));
-  ordered_cache_names_.push_back(cache_name);
+  cache_index_->Insert(
+      CacheStorageIndex::CacheMetadata(cache_name, cache_ptr->cache_size()));
 
   cache_loader_->WriteIndex(
-      ordered_cache_names_,
-      base::Bind(&CacheStorage::CreateCacheDidWriteIndex,
-                 weak_factory_.GetWeakPtr(), callback,
-                 base::Passed(CreateCacheHandle(cache_ptr))));
+      *cache_index_, base::Bind(&CacheStorage::CreateCacheDidWriteIndex,
+                                weak_factory_.GetWeakPtr(), callback,
+                                base::Passed(CreateCacheHandle(cache_ptr))));
 
   cache_loader_->NotifyCacheCreated(cache_name, CreateCacheHandle(cache_ptr));
 }
 
 void CacheStorage::CreateCacheDidWriteIndex(
     const CacheAndErrorCallback& callback,
     std::unique_ptr<CacheStorageCacheHandle> cache_handle,
     bool success) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(cache_handle);
@@ skipping 10 matching lines @@
 }
 
 void CacheStorage::DeleteCacheImpl(const std::string& cache_name,
                                    const BoolAndErrorCallback& callback) {
   if (!GetLoadedCache(cache_name)) {
     base::ThreadTaskRunnerHandle::Get()->PostTask(
         FROM_HERE, base::Bind(callback, false, CACHE_STORAGE_ERROR_NOT_FOUND));
     return;
   }
 
-  // Delete the name from ordered_cache_names_.
-  StringVector original_ordered_cache_names = ordered_cache_names_;
-  StringVector::iterator iter = std::find(
-      ordered_cache_names_.begin(), ordered_cache_names_.end(), cache_name);
-  DCHECK(iter != ordered_cache_names_.end());
-  ordered_cache_names_.erase(iter);
+  // Save the current index so that it can be restored if the write fails.
+  // TODO(cmumford): Avoid creating a copy, and make CacheStorageCacheIndex
+  // DISALLOW_COPY_AND_ASSIGN.
+  auto prior_index = base::MakeUnique<CacheStorageIndex>(*cache_index_);
+  cache_index_->Delete(cache_name);
 
-  cache_loader_->WriteIndex(ordered_cache_names_,
-                            base::Bind(&CacheStorage::DeleteCacheDidWriteIndex,
-                                       weak_factory_.GetWeakPtr(), cache_name,
-                                       original_ordered_cache_names, callback));
+  cache_loader_->WriteIndex(
+      *cache_index_,
+      base::Bind(&CacheStorage::DeleteCacheDidWriteIndex,
+                 weak_factory_.GetWeakPtr(), cache_name,
+                 base::Passed(std::move(prior_index)), callback));
 }
 
 void CacheStorage::DeleteCacheDidWriteIndex(
     const std::string& cache_name,
-    const StringVector& original_ordered_cache_names,
+    std::unique_ptr<CacheStorageIndex> index_before_delete,
     const BoolAndErrorCallback& callback,
     bool success) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   if (!success) {
     // Undo any changes if the change couldn't be written to disk.
-    ordered_cache_names_ = original_ordered_cache_names;
+    cache_index_ = std::move(index_before_delete);
     callback.Run(false, CACHE_STORAGE_ERROR_STORAGE);
     return;
   }
 
   // Make sure that a cache handle exists for the doomed cache to ensure that
   // DeleteCacheFinalize is called.
   std::unique_ptr<CacheStorageCacheHandle> cache_handle =
       GetLoadedCache(cache_name);
 
   CacheMap::iterator map_iter = cache_map_.find(cache_name);
   doomed_caches_.insert(
       std::make_pair(map_iter->second.get(), std::move(map_iter->second)));
   cache_map_.erase(map_iter);
 
   cache_loader_->NotifyCacheDoomed(std::move(cache_handle));
 
   callback.Run(true, CACHE_STORAGE_OK);
 }
 
 // Call this once the last handle to a doomed cache is gone. It's okay if this
 // doesn't get to complete before shutdown, the cache will be removed from disk
 // on next startup in that case.
 void CacheStorage::DeleteCacheFinalize(
     std::unique_ptr<CacheStorageCache> doomed_cache) {
   CacheStorageCache* cache = doomed_cache.get();
+  cache->SetObserver(nullptr);

[jkarlin, 2016/12/09 19:49:19]

You could call this as soon as a cache is doomed, then you won't get those
problematic observer updates.

[cmumford, 2016/12/15 22:29:14]

Done.

   cache->Size(base::Bind(&CacheStorage::DeleteCacheDidGetSize,
                          weak_factory_.GetWeakPtr(),
                          base::Passed(std::move(doomed_cache))));
 }
 
 void CacheStorage::DeleteCacheDidGetSize(
     std::unique_ptr<CacheStorageCache> cache,
     int64_t cache_size) {
   quota_manager_proxy_->NotifyStorageModified(
       storage::QuotaClient::kServiceWorkerCache, origin_,
       storage::kStorageTypeTemporary, -1 * cache_size);
 
   cache_loader_->CleanUpDeletedCache(cache.get());
 }
 
-void CacheStorage::EnumerateCachesImpl(const StringsCallback& callback) {
-  callback.Run(ordered_cache_names_);
+void CacheStorage::EnumerateCachesImpl(const IndexCallback& callback) {
+  callback.Run(*cache_index_);
 }
 
 void CacheStorage::MatchCacheImpl(
     const std::string& cache_name,
     std::unique_ptr<ServiceWorkerFetchRequest> request,
     const CacheStorageCacheQueryParams& match_params,
     const CacheStorageCache::ResponseCallback& callback) {
   std::unique_ptr<CacheStorageCacheHandle> cache_handle =
       GetLoadedCache(cache_name);
 
@@ skipping 20 matching lines @@
     std::unique_ptr<ServiceWorkerResponse> response,
     std::unique_ptr<storage::BlobDataHandle> handle) {
   callback.Run(error, std::move(response), std::move(handle));
 }
 
 void CacheStorage::MatchAllCachesImpl(
     std::unique_ptr<ServiceWorkerFetchRequest> request,
     const CacheStorageCacheQueryParams& match_params,
     const CacheStorageCache::ResponseCallback& callback) {
   std::vector<CacheMatchResponse>* match_responses =
-      new std::vector<CacheMatchResponse>(ordered_cache_names_.size());
+      new std::vector<CacheMatchResponse>(cache_index_->num_entries());
 
   base::Closure barrier_closure = base::BarrierClosure(
-      ordered_cache_names_.size(),
+      cache_index_->num_entries(),
       base::Bind(&CacheStorage::MatchAllCachesDidMatchAll,
                  weak_factory_.GetWeakPtr(),
                  base::Passed(base::WrapUnique(match_responses)), callback));
 
-  for (size_t i = 0, max = ordered_cache_names_.size(); i < max; ++i) {
+  size_t idx = 0;
+  for (const auto& cache_metadata : cache_index_->ordered_cache_metadata()) {
     std::unique_ptr<CacheStorageCacheHandle> cache_handle =
-        GetLoadedCache(ordered_cache_names_[i]);
+        GetLoadedCache(cache_metadata.name);
     DCHECK(cache_handle);
 
     CacheStorageCache* cache_ptr = cache_handle->value();
     cache_ptr->Match(base::MakeUnique<ServiceWorkerFetchRequest>(*request),
                      match_params,
                      base::Bind(&CacheStorage::MatchAllCachesDidMatch,
                                 weak_factory_.GetWeakPtr(),
                                 base::Passed(std::move(cache_handle)),
-                                &match_responses->at(i), barrier_closure));
+                                &match_responses->at(idx), barrier_closure));
+    idx++;
   }
 }
 
 void CacheStorage::MatchAllCachesDidMatch(
     std::unique_ptr<CacheStorageCacheHandle> cache_handle,
     CacheMatchResponse* out_match_response,
     const base::Closure& barrier_closure,
     CacheStorageError error,
     std::unique_ptr<ServiceWorkerResponse> service_worker_response,
     std::unique_ptr<storage::BlobDataHandle> handle) {
@@ skipping 67 matching lines @@
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(initialized_);
 
   CacheMap::iterator map_iter = cache_map_.find(cache_name);
   if (map_iter == cache_map_.end())
     return std::unique_ptr<CacheStorageCacheHandle>();
 
   CacheStorageCache* cache = map_iter->second.get();
 
   if (!cache) {
-    std::unique_ptr<CacheStorageCache> new_cache =
-        cache_loader_->CreateCache(cache_name);
+    std::unique_ptr<CacheStorageCache> new_cache = cache_loader_->CreateCache(
+        cache_name, cache_index_->GetCacheSize(cache_name));
     CacheStorageCache* cache_ptr = new_cache.get();
     map_iter->second = std::move(new_cache);
 
     return CreateCacheHandle(cache_ptr);
   }
 
   return CreateCacheHandle(cache);
 }
 
+void CacheStorage::SizeRetrievedFromCache(
+    std::unique_ptr<CacheStorageCacheHandle> cache_handle,
+    const base::Closure& closure,
+    int64_t* accumulator,
+    int64_t size) {
+  cache_index_->SetCacheSize(cache_handle->value()->cache_name(), size);
+  *accumulator += size;
+  closure.Run();
+}
+
 void CacheStorage::GetSizeThenCloseAllCachesImpl(const SizeCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(initialized_);
 
   std::unique_ptr<int64_t> accumulator(new int64_t(0));
   int64_t* accumulator_ptr = accumulator.get();
 
   base::Closure barrier_closure = base::BarrierClosure(
-      ordered_cache_names_.size(),
+      cache_index_->num_entries(),
       base::Bind(&SizeRetrievedFromAllCaches,
                  base::Passed(std::move(accumulator)), callback));
 
-  for (const std::string& cache_name : ordered_cache_names_) {
-    std::unique_ptr<CacheStorageCacheHandle> cache_handle =
-        GetLoadedCache(cache_name);
+  for (const auto& cache_metadata : cache_index_->ordered_cache_metadata()) {
+    auto cache_handle = GetLoadedCache(cache_metadata.name);
     CacheStorageCache* cache = cache_handle->value();
-    cache->GetSizeThenClose(base::Bind(&SizeRetrievedFromCache,
+    cache->GetSizeThenClose(base::Bind(&CacheStorage::SizeRetrievedFromCache,
+                                       weak_factory_.GetWeakPtr(),
                                        base::Passed(std::move(cache_handle)),
                                        barrier_closure, accumulator_ptr));
   }
 }
 
 void CacheStorage::SizeImpl(const SizeCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(initialized_);
 
+  if (cache_index_->GetStorageSize() != kSizeUnknown) {
+    base::ThreadTaskRunnerHandle::Get()->PostTask(
+        FROM_HERE, base::Bind(callback, cache_index_->GetStorageSize()));
+    return;
+  }
+
   std::unique_ptr<int64_t> accumulator(new int64_t(0));
   int64_t* accumulator_ptr = accumulator.get();
 
   base::Closure barrier_closure = base::BarrierClosure(
-      ordered_cache_names_.size(),
+      cache_index_->num_entries(),
       base::Bind(&SizeRetrievedFromAllCaches,
                  base::Passed(std::move(accumulator)), callback));
 
-  for (const std::string& cache_name : ordered_cache_names_) {
+  for (const auto& cache_metadata : cache_index_->ordered_cache_metadata()) {
+    if (cache_metadata.size != CacheStorage::kSizeUnknown) {
+      *accumulator_ptr += cache_metadata.size;
+      barrier_closure.Run();
+      continue;
+    }
     std::unique_ptr<CacheStorageCacheHandle> cache_handle =
-        GetLoadedCache(cache_name);
+        GetLoadedCache(cache_metadata.name);
     CacheStorageCache* cache = cache_handle->value();
-    cache->Size(base::Bind(&SizeRetrievedFromCache,
+    cache->Size(base::Bind(&CacheStorage::SizeRetrievedFromCache,
+                           weak_factory_.GetWeakPtr(),
                            base::Passed(std::move(cache_handle)),
                            barrier_closure, accumulator_ptr));
   }
 }
 
 }  // namespace content