Write out CacheStorageCache size to index file.

content/browser/cache_storage/cache_storage.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/cache_storage/cache_storage.h"
 
 #include <stddef.h>
 
 #include <set>
 #include <string>
+#include <unordered_map>

[jkarlin, 2016/11/23 16:02:17]

Unused

[cmumford, 2016/11/29 18:10:20]

Done.

 #include <utility>
 
 #include "base/barrier_closure.h"
 #include "base/files/file_util.h"
 #include "base/files/memory_mapped_file.h"
 #include "base/guid.h"
 #include "base/location.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/sha1.h"
 #include "base/single_thread_task_runner.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_util.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "content/browser/cache_storage/cache_storage.pb.h"
 #include "content/browser/cache_storage/cache_storage_cache.h"
 #include "content/browser/cache_storage/cache_storage_cache_handle.h"
+#include "content/browser/cache_storage/cache_storage_index.h"
 #include "content/browser/cache_storage/cache_storage_scheduler.h"
 #include "content/public/browser/browser_thread.h"
 #include "net/base/directory_lister.h"
 #include "net/base/net_errors.h"
 #include "net/url_request/url_request_context_getter.h"
 #include "storage/browser/blob/blob_storage_context.h"
 #include "storage/browser/quota/quota_manager_proxy.h"
 
 namespace content {
 
 namespace {
 
 std::string HexedHash(const std::string& value) {
   std::string value_hash = base::SHA1HashString(value);
   std::string valued_hexed_hash = base::ToLowerASCII(
       base::HexEncode(value_hash.c_str(), value_hash.length()));
   return valued_hexed_hash;
 }
 
-void SizeRetrievedFromCache(
-    std::unique_ptr<CacheStorageCacheHandle> cache_handle,
-    const base::Closure& closure,
-    int64_t* accumulator,
-    int64_t size) {
-  *accumulator += size;
-  closure.Run();
-}
-
 void SizeRetrievedFromAllCaches(std::unique_ptr<int64_t> accumulator,
                                 const CacheStorage::SizeCallback& callback) {
   base::ThreadTaskRunnerHandle::Get()->PostTask(
       FROM_HERE, base::Bind(callback, *accumulator));
 }
 
+void DoNothingWithBool(bool success) {}
+
 }  // namespace
 
 const char CacheStorage::kIndexFileName[] = "index.txt";
+const int64_t CacheStorage::kSizeUnknown;
 
 struct CacheStorage::CacheMatchResponse {
   CacheMatchResponse() = default;
   ~CacheMatchResponse() = default;
 
   CacheStorageError error;
   std::unique_ptr<ServiceWorkerResponse> service_worker_response;
   std::unique_ptr<storage::BlobDataHandle> blob_data_handle;
 };
 
 // Handles the loading and clean up of CacheStorageCache objects.
 class CacheStorage::CacheLoader {
  public:
   typedef base::Callback<void(std::unique_ptr<CacheStorageCache>)>
       CacheCallback;
   typedef base::Callback<void(bool)> BoolCallback;
-  typedef base::Callback<void(std::unique_ptr<std::vector<std::string>>)>
-      StringVectorCallback;
+  using CacheStorageIndexCallback =
+      base::Callback<void(std::unique_ptr<CacheStorageIndex>)>;
 
   CacheLoader(
       base::SequencedTaskRunner* cache_task_runner,
       scoped_refptr<net::URLRequestContextGetter> request_context_getter,
       storage::QuotaManagerProxy* quota_manager_proxy,
       base::WeakPtr<storage::BlobStorageContext> blob_context,
       CacheStorage* cache_storage,
       const GURL& origin)
       : cache_task_runner_(cache_task_runner),
         request_context_getter_(request_context_getter),
         quota_manager_proxy_(quota_manager_proxy),
         blob_context_(blob_context),
         cache_storage_(cache_storage),
         origin_(origin) {
     DCHECK(!origin_.is_empty());
   }
 
   virtual ~CacheLoader() {}
 
   // Creates a CacheStorageCache with the given name. It does not attempt to
   // load the backend, that happens lazily when the cache is used.
   virtual std::unique_ptr<CacheStorageCache> CreateCache(
-      const std::string& cache_name) = 0;
+      const std::string& cache_name,
+      int64_t cache_size) = 0;
 
   // Deletes any pre-existing cache of the same name and then loads it.
   virtual void PrepareNewCacheDestination(const std::string& cache_name,
                                           const CacheCallback& callback) = 0;
 
   // After the backend has been deleted, do any extra house keeping such as
   // removing the cache's directory.
   virtual void CleanUpDeletedCache(CacheStorageCache* cache) = 0;
 
-  // Writes the cache names (and sizes) to disk if applicable.
-  virtual void WriteIndex(const StringVector& cache_names,
+  // Writes the cache index to disk if applicable.
+  virtual void WriteIndex(const CacheStorageIndex& index,
                           const BoolCallback& callback) = 0;
 
-  // Loads the cache names from disk if applicable.
-  virtual void LoadIndex(std::unique_ptr<std::vector<std::string>> cache_names,
-                         const StringVectorCallback& callback) = 0;
+  // Loads the cache index from disk if applicable.
+  virtual void LoadIndex(const CacheStorageIndexCallback& callback) = 0;
 
   // Called when CacheStorage has created a cache. Used to hold onto a handle to
   // the cache if necessary.
   virtual void NotifyCacheCreated(
       const std::string& cache_name,
       std::unique_ptr<CacheStorageCacheHandle> cache_handle) {}
 
   // Notification that the cache for |cache_handle| has been doomed. If the
   // loader is holding a handle to the cache, it should drop it now.
   virtual void NotifyCacheDoomed(
@@ skipping 26 matching lines @@
                base::WeakPtr<storage::BlobStorageContext> blob_context,
                CacheStorage* cache_storage,
                const GURL& origin)
       : CacheLoader(cache_task_runner,
                     request_context,
                     quota_manager_proxy,
                     blob_context,
                     cache_storage,
                     origin) {}
 
-  std::unique_ptr<CacheStorageCache> CreateCache(
-      const std::string& cache_name) override {
+  std::unique_ptr<CacheStorageCache> CreateCache(const std::string& cache_name,
+                                                 int64_t cache_size) override {
     return CacheStorageCache::CreateMemoryCache(
         origin_, cache_name, cache_storage_, request_context_getter_,
         quota_manager_proxy_, blob_context_);
   }
 
   void PrepareNewCacheDestination(const std::string& cache_name,
                                   const CacheCallback& callback) override {
-    std::unique_ptr<CacheStorageCache> cache = CreateCache(cache_name);
+    std::unique_ptr<CacheStorageCache> cache =
+        CreateCache(cache_name, 0 /*cache_size*/);
     callback.Run(std::move(cache));
   }
 
   void CleanUpDeletedCache(CacheStorageCache* cache) override {}
 
-  void WriteIndex(const StringVector& cache_names,
+  void WriteIndex(const CacheStorageIndex& index,
                   const BoolCallback& callback) override {
     callback.Run(true);
   }
 
-  void LoadIndex(std::unique_ptr<std::vector<std::string>> cache_names,
-                 const StringVectorCallback& callback) override {
-    callback.Run(std::move(cache_names));
+  void LoadIndex(const CacheStorageIndexCallback& callback) override {
+    callback.Run(base::MakeUnique<CacheStorageIndex>());
   }
 
   void NotifyCacheCreated(
       const std::string& cache_name,
       std::unique_ptr<CacheStorageCacheHandle> cache_handle) override {
     DCHECK(!base::ContainsKey(cache_handles_, cache_name));
     cache_handles_.insert(std::make_pair(cache_name, std::move(cache_handle)));
   };
 
   void NotifyCacheDoomed(
@@ skipping 25 matching lines @@
                     const GURL& origin)
       : CacheLoader(cache_task_runner,
                     request_context,
                     quota_manager_proxy,
                     blob_context,
                     cache_storage,
                     origin),
         origin_path_(origin_path),
         weak_ptr_factory_(this) {}
 
-  std::unique_ptr<CacheStorageCache> CreateCache(
-      const std::string& cache_name) override {
+  std::unique_ptr<CacheStorageCache> CreateCache(const std::string& cache_name,
+                                                 int64_t cache_size) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
     DCHECK(base::ContainsKey(cache_name_to_cache_dir_, cache_name));
 
     std::string cache_dir = cache_name_to_cache_dir_[cache_name];
     base::FilePath cache_path = origin_path_.AppendASCII(cache_dir);
     return CacheStorageCache::CreatePersistentCache(
         origin_, cache_name, cache_storage_, cache_path,
-        request_context_getter_, quota_manager_proxy_, blob_context_);
+        request_context_getter_, quota_manager_proxy_, blob_context_,
+        cache_size);
   }
 
   void PrepareNewCacheDestination(const std::string& cache_name,
                                   const CacheCallback& callback) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
     PostTaskAndReplyWithResult(
         cache_task_runner_.get(), FROM_HERE,
         base::Bind(&SimpleCacheLoader::PrepareNewCacheDirectoryInPool,
                    origin_path_),
@@ skipping 16 matching lines @@
 
   void PrepareNewCacheCreateCache(const std::string& cache_name,
                                   const CacheCallback& callback,
                                   const std::string& cache_dir) {
     if (cache_dir.empty()) {
       callback.Run(std::unique_ptr<CacheStorageCache>());
       return;
     }
 
     cache_name_to_cache_dir_[cache_name] = cache_dir;
-    callback.Run(CreateCache(cache_name));
+    callback.Run(CreateCache(cache_name, CacheStorage::kSizeUnknown));
   }
 
   void CleanUpDeletedCache(CacheStorageCache* cache) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
     DCHECK(base::ContainsKey(doomed_cache_to_path_, cache));
 
     base::FilePath cache_path =
         origin_path_.AppendASCII(doomed_cache_to_path_[cache]);
     doomed_cache_to_path_.erase(cache);
 
     cache_task_runner_->PostTask(
         FROM_HERE, base::Bind(&SimpleCacheLoader::CleanUpDeleteCacheDirInPool,
                               cache_path));
   }
 
   static void CleanUpDeleteCacheDirInPool(const base::FilePath& cache_path) {
     base::DeleteFile(cache_path, true /* recursive */);
   }
 
-  void WriteIndex(const StringVector& cache_names,
+  void WriteIndex(const CacheStorageIndex& index,
                   const BoolCallback& callback) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
     // 1. Create the index file as a string. (WriteIndex)
     // 2. Write the file to disk. (WriteIndexWriteToFileInPool)
 
-    proto::CacheStorageIndex index;
-    index.set_origin(origin_.spec());
+    proto::CacheStorageIndex protobuf_index;
+    protobuf_index.set_origin(origin_.spec());
 
-    for (size_t i = 0u, max = cache_names.size(); i < max; ++i) {
-      DCHECK(base::ContainsKey(cache_name_to_cache_dir_, cache_names[i]));
+    for (const auto& cache_metadata : index.ordered_cache_metadata()) {
+      DCHECK(base::ContainsKey(cache_name_to_cache_dir_, cache_metadata.name));
 
-      proto::CacheStorageIndex::Cache* index_cache = index.add_cache();
-      index_cache->set_name(cache_names[i]);
-      index_cache->set_cache_dir(cache_name_to_cache_dir_[cache_names[i]]);
+      proto::CacheStorageIndex::Cache* index_cache = protobuf_index.add_cache();
+      index_cache->set_name(cache_metadata.name);
+      index_cache->set_cache_dir(cache_name_to_cache_dir_[cache_metadata.name]);
+      if (cache_metadata.size == CacheStorage::kSizeUnknown)
+        index_cache->clear_size();
+      else
+        index_cache->set_size(cache_metadata.size);
     }
 
     std::string serialized;
-    bool success = index.SerializeToString(&serialized);
+    bool success = protobuf_index.SerializeToString(&serialized);
     DCHECK(success);
 
     base::FilePath tmp_path = origin_path_.AppendASCII("index.txt.tmp");
     base::FilePath index_path =
         origin_path_.AppendASCII(CacheStorage::kIndexFileName);
 
     PostTaskAndReplyWithResult(
         cache_task_runner_.get(), FROM_HERE,
         base::Bind(&SimpleCacheLoader::WriteIndexWriteToFileInPool, tmp_path,
                    index_path, serialized),
         callback);
   }
 
   static bool WriteIndexWriteToFileInPool(const base::FilePath& tmp_path,
                                           const base::FilePath& index_path,
                                           const std::string& data) {
     int bytes_written = base::WriteFile(tmp_path, data.c_str(), data.size());
     if (bytes_written != base::checked_cast<int>(data.size())) {
       base::DeleteFile(tmp_path, /* recursive */ false);
       return false;
     }
 
     // Atomically rename the temporary index file to become the real one.
     return base::ReplaceFile(tmp_path, index_path, NULL);
   }
 
-  void LoadIndex(std::unique_ptr<std::vector<std::string>> names,
-                 const StringVectorCallback& callback) override {
+  void LoadIndex(const CacheStorageIndexCallback& callback) override {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
-    // 1. Read the file from disk. (LoadIndexReadFileInPool)
-    // 2. Parse file and return the names of the caches (LoadIndexDidReadFile)
-
-    base::FilePath index_path =
-        origin_path_.AppendASCII(CacheStorage::kIndexFileName);
-
     PostTaskAndReplyWithResult(
         cache_task_runner_.get(), FROM_HERE,
-        base::Bind(&SimpleCacheLoader::ReadAndMigrateIndexInPool, index_path),
-        base::Bind(&SimpleCacheLoader::LoadIndexDidReadFile,
-                   weak_ptr_factory_.GetWeakPtr(), base::Passed(&names),
-                   callback));
+        base::Bind(&SimpleCacheLoader::ReadAndMigrateIndexInPool, origin_path_),
+        base::Bind(&SimpleCacheLoader::LoadIndexDidReadIndex,
+                   weak_ptr_factory_.GetWeakPtr(), callback));
   }
 
-  void LoadIndexDidReadFile(std::unique_ptr<std::vector<std::string>> names,
-                            const StringVectorCallback& callback,
-                            const std::string& serialized) {
+  void LoadIndexDidReadIndex(const CacheStorageIndexCallback& callback,
+                             proto::CacheStorageIndex protobuf_index) {
     DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
     std::unique_ptr<std::set<std::string>> cache_dirs(
         new std::set<std::string>);
 
-    proto::CacheStorageIndex index;
-    if (index.ParseFromString(serialized)) {
-      for (int i = 0, max = index.cache_size(); i < max; ++i) {
-        const proto::CacheStorageIndex::Cache& cache = index.cache(i);
-        DCHECK(cache.has_cache_dir());
-        names->push_back(cache.name());
-        cache_name_to_cache_dir_[cache.name()] = cache.cache_dir();
-        cache_dirs->insert(cache.cache_dir());
-      }
+    auto index = base::MakeUnique<CacheStorageIndex>();
+    for (int i = 0, max = protobuf_index.cache_size(); i < max; ++i) {
+      const proto::CacheStorageIndex::Cache& cache = protobuf_index.cache(i);
+      DCHECK(cache.has_cache_dir());
+      int64_t cache_size =
+          cache.has_size() ? cache.size() : CacheStorage::kSizeUnknown;
+      index->Insert(CacheStorageIndex::CacheMetadata(cache.name(), cache_size));
+      cache_name_to_cache_dir_[cache.name()] = cache.cache_dir();
+      cache_dirs->insert(cache.cache_dir());
     }
 
     cache_task_runner_->PostTask(
         FROM_HERE, base::Bind(&DeleteUnreferencedCachesInPool, origin_path_,
                               base::Passed(&cache_dirs)));
-    callback.Run(std::move(names));
+    callback.Run(std::move(index));
   }
 
   void NotifyCacheDoomed(
       std::unique_ptr<CacheStorageCacheHandle> cache_handle) override {
     DCHECK(base::ContainsKey(cache_name_to_cache_dir_,
                              cache_handle->value()->cache_name()));
     auto iter =
         cache_name_to_cache_dir_.find(cache_handle->value()->cache_name());
     doomed_cache_to_path_[cache_handle->value()] = iter->second;
     cache_name_to_cache_dir_.erase(iter);
@@ skipping 15 matching lines @@
     while (!(cache_path = file_enum.Next()).empty()) {
       if (!base::ContainsKey(*cache_dirs, cache_path.BaseName().AsUTF8Unsafe()))
         dirs_to_delete.push_back(cache_path);
     }
 
     for (const base::FilePath& cache_path : dirs_to_delete)
       base::DeleteFile(cache_path, true /* recursive */);
   }
 
   // Runs on cache_task_runner_
-  static std::string MigrateCachesIfNecessaryInPool(
-      const std::string& body,
-      const base::FilePath& index_path) {
+  static proto::CacheStorageIndex ReadAndMigrateIndexInPool(
+      const base::FilePath& origin_path) {
+    const base::FilePath index_path =
+        origin_path.AppendASCII(CacheStorage::kIndexFileName);
+
     proto::CacheStorageIndex index;
-    if (!index.ParseFromString(body))
-      return body;
+    std::string body;
+    if (!base::ReadFileToString(index_path, &body) ||
+        !index.ParseFromString(body))
+      return proto::CacheStorageIndex();
+    body.clear();
 
-    base::FilePath origin_path = index_path.DirName();
-    bool index_is_dirty = false;
-    const std::string kBadIndexState("");
+    base::File::Info file_info;
+    base::Time index_last_modified;
+    if (GetFileInfo(index_path, &file_info))
+      index_last_modified = file_info.last_modified;
+    bool index_modified = false;
 
     // Look for caches that have no cache_dir. Give any such caches a directory
     // with a random name and move them there. Then, rewrite the index file.
+    // Additionally invalidate any index entries where the cache was modified
+    // after the index (making it out-of-date)
     for (int i = 0, max = index.cache_size(); i < max; ++i) {
       const proto::CacheStorageIndex::Cache& cache = index.cache(i);
-      if (!cache.has_cache_dir()) {
+      if (cache.has_cache_dir()) {
+        if (cache.has_size()) {
+          base::FilePath cache_dir = origin_path.AppendASCII(cache.cache_dir());
+          if (!GetFileInfo(cache_dir, &file_info) ||
+              index_last_modified <= file_info.last_modified) {
+            // Index is older than this cache, so invalidate index entries that
+            // may change as a result of cache operations.
+            index.mutable_cache(i)->clear_size();
+          }
+        }
+      } else {
         // Find a new home for the cache.
         base::FilePath legacy_cache_path =
             origin_path.AppendASCII(HexedHash(cache.name()));
         std::string cache_dir;
         base::FilePath cache_path;
         do {
           cache_dir = base::GenerateGUID();
           cache_path = origin_path.AppendASCII(cache_dir);
         } while (base::PathExists(cache_path));
 
         if (!base::Move(legacy_cache_path, cache_path)) {
           // If the move fails then the cache is in a bad state. Return an empty
           // index so that the CacheStorage can start fresh. The unreferenced
           // caches will be discarded later in initialization.
-          return kBadIndexState;
+          return proto::CacheStorageIndex();
         }
 
         index.mutable_cache(i)->set_cache_dir(cache_dir);
-        index_is_dirty = true;
+        index.mutable_cache(i)->clear_size();
+        index_modified = true;
       }
     }
 
-    if (index_is_dirty) {
-      std::string new_body;
-      if (!index.SerializeToString(&new_body))
-        return kBadIndexState;
-      if (base::WriteFile(index_path, new_body.c_str(), new_body.size()) !=
-          base::checked_cast<int>(new_body.size()))
-        return kBadIndexState;
-      return new_body;
+    if (index_modified) {

[jkarlin, 2016/11/23 16:02:18]

Since only one place marks index_modified as true, we can get rid of the
index_modified variable and move this logic to the relevant spot.

[cmumford, 2016/11/29 18:10:20]

index_modified_ is being set when looping all cache entries - more than one of
which may be modified. We only want to write this out once.

Or am I missing something?

[jkarlin, 2016/12/01 17:43:30]

Quite right. Never mind me :)

+      if (!index.SerializeToString(&body))
+        return proto::CacheStorageIndex();
+      if (base::WriteFile(index_path, body.c_str(), body.size()) !=
+          base::checked_cast<int>(body.size()))
+        return proto::CacheStorageIndex();
     }
 
-    return body;
-  }
-
-  // Runs on cache_task_runner_
-  static std::string ReadAndMigrateIndexInPool(
-      const base::FilePath& index_path) {
-    std::string body;
-    base::ReadFileToString(index_path, &body);
-
-    return MigrateCachesIfNecessaryInPool(body, index_path);
+    return index;
   }
 
   const base::FilePath origin_path_;
   std::map<std::string, std::string> cache_name_to_cache_dir_;
   std::map<CacheStorageCache*, std::string> doomed_cache_to_path_;
 
   base::WeakPtrFactory<SimpleCacheLoader> weak_ptr_factory_;
 };
 
 CacheStorage::CacheStorage(
     const base::FilePath& path,
     bool memory_only,
     base::SequencedTaskRunner* cache_task_runner,
     scoped_refptr<net::URLRequestContextGetter> request_context,
     scoped_refptr<storage::QuotaManagerProxy> quota_manager_proxy,
     base::WeakPtr<storage::BlobStorageContext> blob_context,
     const GURL& origin)
     : initialized_(false),
       initializing_(false),
       memory_only_(memory_only),
       scheduler_(new CacheStorageScheduler(
           CacheStorageSchedulerClient::CLIENT_STORAGE)),
+      index_(base::MakeUnique<CacheStorageIndex>()),
       origin_path_(path),
       cache_task_runner_(cache_task_runner),
       quota_manager_proxy_(quota_manager_proxy),
       origin_(origin),
       weak_factory_(this) {
   if (memory_only)
     cache_loader_.reset(new MemoryLoader(
         cache_task_runner_.get(), std::move(request_context),
         quota_manager_proxy.get(), blob_context, this, origin));
   else
@@ skipping 46 matching lines @@
 
   quota_manager_proxy_->NotifyStorageAccessed(
       storage::QuotaClient::kServiceWorkerCache, origin_,
       storage::kStorageTypeTemporary);
 
   scheduler_->ScheduleOperation(
       base::Bind(&CacheStorage::DeleteCacheImpl, weak_factory_.GetWeakPtr(),
                  cache_name, scheduler_->WrapCallbackToRunNext(callback)));
 }
 
-void CacheStorage::EnumerateCaches(const StringsCallback& callback) {
+void CacheStorage::EnumerateCaches(const CacheMetadataCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   if (!initialized_)
     LazyInit();
 
   quota_manager_proxy_->NotifyStorageAccessed(
       storage::QuotaClient::kServiceWorkerCache, origin_,
       storage::kStorageTypeTemporary);
 
   scheduler_->ScheduleOperation(
@@ skipping 55 matching lines @@
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   if (!initialized_)
     LazyInit();
 
   scheduler_->ScheduleOperation(
       base::Bind(&CacheStorage::SizeImpl, weak_factory_.GetWeakPtr(),
                  scheduler_->WrapCallbackToRunNext(callback)));
 }
 
+void CacheStorage::ScheduleWriteIndex() {
+  // TODO(cmumford): Write in a lazy fashion to minimize writes.
+  // TODO(cmumford): Write the index before/after other cache writes because
+  //                 modification-times are used for out-of-date checks.

[jkarlin, 2016/11/23 16:02:17]

I'd say post a cancelable (with 5 second delay) task to write the index. If
another change occurs before the task runs, cancel the current task and start a
new one.

[cmumford, 2016/11/29 18:10:20]

Done.

+  cache_loader_->WriteIndex(*index_, base::Bind(&DoNothingWithBool));
+}
+
+void CacheStorage::CacheSizeSet(const CacheStorageCache* cache,
+                                Whence whence,
+                                int64_t size) {
+  switch (whence) {

[jkarlin, 2016/11/23 16:02:18]

This complexity (deltas vs total size) seems unnecessary. Why not just always
send the cache size? It looks like CacheStorageIndex could just loop through its
map of caches on demand to get the total size.

[cmumford, 2016/11/29 18:10:20]

You're totally right - fixed.

+    case Whence::SET:
+      index_->SetCacheSize(cache->cache_name(), size);
+      break;
+    case Whence::DELTA:
+      if (!size)
+        return;
+      index_->SetCacheSizeModified(cache->cache_name(), size);
+      break;
+  }
+  ScheduleWriteIndex();
+}
+
 void CacheStorage::StartAsyncOperationForTesting() {
   scheduler_->ScheduleOperation(base::Bind(&base::DoNothing));
 }
 
 void CacheStorage::CompleteAsyncOperationForTesting() {
   scheduler_->CompleteOperationAndRunNext();
 }
 
 // Init is run lazily so that it is called on the proper MessageLoop.
 void CacheStorage::LazyInit() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(!initialized_);
 
   if (initializing_)
     return;
 
   DCHECK(!scheduler_->ScheduledOperations());
 
   initializing_ = true;
   scheduler_->ScheduleOperation(
       base::Bind(&CacheStorage::LazyInitImpl, weak_factory_.GetWeakPtr()));
 }
 
 void CacheStorage::LazyInitImpl() {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(!initialized_);
   DCHECK(initializing_);
 
-  // 1. Get the list of cache names (async call)
+  // 1. Get the cache index (async call)
   // 2. For each cache name, load the cache (async call)
   // 3. Once each load is complete, update the map variables.
   // 4. Call the list of waiting callbacks.
 
-  std::unique_ptr<std::vector<std::string>> indexed_cache_names(
-      new std::vector<std::string>());
-
-  cache_loader_->LoadIndex(std::move(indexed_cache_names),
-                           base::Bind(&CacheStorage::LazyInitDidLoadIndex,
+  cache_loader_->LoadIndex(base::Bind(&CacheStorage::LazyInitDidLoadIndex,
                                       weak_factory_.GetWeakPtr()));
 }
 
 void CacheStorage::LazyInitDidLoadIndex(
-    std::unique_ptr<std::vector<std::string>> indexed_cache_names) {
+    std::unique_ptr<CacheStorageIndex> index) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
+  DCHECK(cache_map_.empty());
 
-  for (size_t i = 0u, max = indexed_cache_names->size(); i < max; ++i) {
-    cache_map_.insert(std::make_pair(indexed_cache_names->at(i),
+  for (const auto& cache_metadata : index->ordered_cache_metadata()) {
+    cache_map_.insert(std::make_pair(cache_metadata.name,
                                      std::unique_ptr<CacheStorageCache>()));
-    ordered_cache_names_.push_back(indexed_cache_names->at(i));
   }
 
+  index_.swap(index);

[jkarlin, 2016/11/23 16:02:17]

index_ = std::move(index);

[cmumford, 2016/11/29 18:10:20]

Done.

+
   initializing_ = false;
   initialized_ = true;
 
   scheduler_->CompleteOperationAndRunNext();
 }
 
 void CacheStorage::OpenCacheImpl(const std::string& cache_name,
                                  const CacheAndErrorCallback& callback) {
   std::unique_ptr<CacheStorageCacheHandle> cache_handle =
       GetLoadedCache(cache_name);
@@ skipping 18 matching lines @@
 
   if (!cache) {
     callback.Run(std::unique_ptr<CacheStorageCacheHandle>(),
                  CACHE_STORAGE_ERROR_STORAGE);
     return;
   }
 
   CacheStorageCache* cache_ptr = cache.get();
 
   cache_map_.insert(std::make_pair(cache_name, std::move(cache)));
-  ordered_cache_names_.push_back(cache_name);
+  index_->Insert(
+      CacheStorageIndex::CacheMetadata(cache_name, cache_ptr->cache_size()));
 
   cache_loader_->WriteIndex(
-      ordered_cache_names_,
-      base::Bind(&CacheStorage::CreateCacheDidWriteIndex,
-                 weak_factory_.GetWeakPtr(), callback,
-                 base::Passed(CreateCacheHandle(cache_ptr))));
+      *index_, base::Bind(&CacheStorage::CreateCacheDidWriteIndex,
+                          weak_factory_.GetWeakPtr(), callback,
+                          base::Passed(CreateCacheHandle(cache_ptr))));
 
   cache_loader_->NotifyCacheCreated(cache_name, CreateCacheHandle(cache_ptr));
 }
 
 void CacheStorage::CreateCacheDidWriteIndex(
     const CacheAndErrorCallback& callback,
     std::unique_ptr<CacheStorageCacheHandle> cache_handle,
     bool success) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(cache_handle);
@@ skipping 10 matching lines @@
 }
 
 void CacheStorage::DeleteCacheImpl(const std::string& cache_name,
                                    const BoolAndErrorCallback& callback) {
   if (!GetLoadedCache(cache_name)) {
     base::ThreadTaskRunnerHandle::Get()->PostTask(
         FROM_HERE, base::Bind(callback, false, CACHE_STORAGE_ERROR_NOT_FOUND));
     return;
   }
 
-  // Delete the name from ordered_cache_names_.
-  StringVector original_ordered_cache_names = ordered_cache_names_;
-  StringVector::iterator iter = std::find(
-      ordered_cache_names_.begin(), ordered_cache_names_.end(), cache_name);
-  DCHECK(iter != ordered_cache_names_.end());
-  ordered_cache_names_.erase(iter);
+  // Save the current index so that it can be restored if the write fails.
+  // TODO(cmumford): Avoid creating a copy, and make CacheStorageCacheIndex
+  // DISALLOW_COPY_AND_ASSIGN.
+  auto prior_index = base::MakeUnique<CacheStorageIndex>(*index_);
+  index_->Delete(cache_name);
 
-  cache_loader_->WriteIndex(ordered_cache_names_,
-                            base::Bind(&CacheStorage::DeleteCacheDidWriteIndex,
-                                       weak_factory_.GetWeakPtr(), cache_name,
-                                       original_ordered_cache_names, callback));
+  cache_loader_->WriteIndex(
+      *index_, base::Bind(&CacheStorage::DeleteCacheDidWriteIndex,
+                          weak_factory_.GetWeakPtr(), cache_name,
+                          base::Passed(std::move(prior_index)), callback));
 }
 
 void CacheStorage::DeleteCacheDidWriteIndex(
     const std::string& cache_name,
-    const StringVector& original_ordered_cache_names,
+    std::unique_ptr<CacheStorageIndex> index_before_delete,
     const BoolAndErrorCallback& callback,
     bool success) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
 
   if (!success) {
     // Undo any changes if the change couldn't be written to disk.
-    ordered_cache_names_ = original_ordered_cache_names;
+    index_.swap(index_before_delete);

[jkarlin, 2016/11/23 16:02:18]

index_ = std::move(index_before_delete);

[cmumford, 2016/11/29 18:10:20]

Done.

     callback.Run(false, CACHE_STORAGE_ERROR_STORAGE);
     return;
   }
 
   // Make sure that a cache handle exists for the doomed cache to ensure that
   // DeleteCacheFinalize is called.
   std::unique_ptr<CacheStorageCacheHandle> cache_handle =
       GetLoadedCache(cache_name);
 
   CacheMap::iterator map_iter = cache_map_.find(cache_name);
   doomed_caches_.insert(
       std::make_pair(map_iter->second.get(), std::move(map_iter->second)));
   cache_map_.erase(map_iter);
 
   cache_loader_->NotifyCacheDoomed(std::move(cache_handle));
 
   callback.Run(true, CACHE_STORAGE_OK);
 }
 
 // Call this once the last handle to a doomed cache is gone. It's okay if this
 // doesn't get to complete before shutdown, the cache will be removed from disk
 // on next startup in that case.
 void CacheStorage::DeleteCacheFinalize(
     std::unique_ptr<CacheStorageCache> doomed_cache) {
   CacheStorageCache* cache = doomed_cache.get();
+  cache->SetObserver(nullptr);
   cache->Size(base::Bind(&CacheStorage::DeleteCacheDidGetSize,
                          weak_factory_.GetWeakPtr(),
                          base::Passed(std::move(doomed_cache))));
 }
 
 void CacheStorage::DeleteCacheDidGetSize(
     std::unique_ptr<CacheStorageCache> cache,
     int64_t cache_size) {
   quota_manager_proxy_->NotifyStorageModified(
       storage::QuotaClient::kServiceWorkerCache, origin_,
       storage::kStorageTypeTemporary, -1 * cache_size);
 
   cache_loader_->CleanUpDeletedCache(cache.get());
 }
 
-void CacheStorage::EnumerateCachesImpl(const StringsCallback& callback) {
-  callback.Run(ordered_cache_names_);
+void CacheStorage::EnumerateCachesImpl(const CacheMetadataCallback& callback) {
+  callback.Run(*index_);
 }
 
 void CacheStorage::MatchCacheImpl(
     const std::string& cache_name,
     std::unique_ptr<ServiceWorkerFetchRequest> request,
     const CacheStorageCacheQueryParams& match_params,
     const CacheStorageCache::ResponseCallback& callback) {
   std::unique_ptr<CacheStorageCacheHandle> cache_handle =
       GetLoadedCache(cache_name);
 
@@ skipping 20 matching lines @@
     std::unique_ptr<ServiceWorkerResponse> response,
     std::unique_ptr<storage::BlobDataHandle> handle) {
   callback.Run(error, std::move(response), std::move(handle));
 }
 
 void CacheStorage::MatchAllCachesImpl(
     std::unique_ptr<ServiceWorkerFetchRequest> request,
     const CacheStorageCacheQueryParams& match_params,
     const CacheStorageCache::ResponseCallback& callback) {
   std::vector<CacheMatchResponse>* match_responses =
-      new std::vector<CacheMatchResponse>(ordered_cache_names_.size());
+      new std::vector<CacheMatchResponse>(index_->num_entries());
 
   base::Closure barrier_closure = base::BarrierClosure(
-      ordered_cache_names_.size(),
+      index_->num_entries(),
       base::Bind(&CacheStorage::MatchAllCachesDidMatchAll,
                  weak_factory_.GetWeakPtr(),
                  base::Passed(base::WrapUnique(match_responses)), callback));
 
-  for (size_t i = 0, max = ordered_cache_names_.size(); i < max; ++i) {
+  size_t idx = 0;
+  for (const auto& cache_metadata : index_->ordered_cache_metadata()) {
     std::unique_ptr<CacheStorageCacheHandle> cache_handle =
-        GetLoadedCache(ordered_cache_names_[i]);
+        GetLoadedCache(cache_metadata.name);
     DCHECK(cache_handle);
 
     CacheStorageCache* cache_ptr = cache_handle->value();
     cache_ptr->Match(base::MakeUnique<ServiceWorkerFetchRequest>(*request),
                      match_params,
                      base::Bind(&CacheStorage::MatchAllCachesDidMatch,
                                 weak_factory_.GetWeakPtr(),
                                 base::Passed(std::move(cache_handle)),
-                                &match_responses->at(i), barrier_closure));
+                                &match_responses->at(idx), barrier_closure));
+    idx++;
   }
 }
 
 void CacheStorage::MatchAllCachesDidMatch(
     std::unique_ptr<CacheStorageCacheHandle> cache_handle,
     CacheMatchResponse* out_match_response,
     const base::Closure& barrier_closure,
     CacheStorageError error,
     std::unique_ptr<ServiceWorkerResponse> service_worker_response,
     std::unique_ptr<storage::BlobDataHandle> handle) {
@@ skipping 67 matching lines @@
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(initialized_);
 
   CacheMap::iterator map_iter = cache_map_.find(cache_name);
   if (map_iter == cache_map_.end())
     return std::unique_ptr<CacheStorageCacheHandle>();
 
   CacheStorageCache* cache = map_iter->second.get();
 
   if (!cache) {
-    std::unique_ptr<CacheStorageCache> new_cache =
-        cache_loader_->CreateCache(cache_name);
+    std::unique_ptr<CacheStorageCache> new_cache = cache_loader_->CreateCache(
+        cache_name, index_->GetCacheSize(cache_name));
     CacheStorageCache* cache_ptr = new_cache.get();
     map_iter->second = std::move(new_cache);
 
     return CreateCacheHandle(cache_ptr);
   }
 
   return CreateCacheHandle(cache);
 }
 
+void CacheStorage::SizeRetrievedFromCache(
+    std::unique_ptr<CacheStorageCacheHandle> cache_handle,
+    const base::Closure& closure,
+    int64_t* accumulator,
+    int64_t size) {
+  index_->SetCacheSize(cache_handle->value()->cache_name(), size);
+  *accumulator += size;
+  closure.Run();
+}
+
 void CacheStorage::GetSizeThenCloseAllCachesImpl(const SizeCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(initialized_);
 
   std::unique_ptr<int64_t> accumulator(new int64_t(0));
   int64_t* accumulator_ptr = accumulator.get();
 
   base::Closure barrier_closure = base::BarrierClosure(
-      ordered_cache_names_.size(),
+      index_->num_entries(),
       base::Bind(&SizeRetrievedFromAllCaches,
                  base::Passed(std::move(accumulator)), callback));
 
-  for (const std::string& cache_name : ordered_cache_names_) {
-    std::unique_ptr<CacheStorageCacheHandle> cache_handle =
-        GetLoadedCache(cache_name);
+  for (const auto& cache_metadata : index_->ordered_cache_metadata()) {
+    auto cache_handle = GetLoadedCache(cache_metadata.name);
     CacheStorageCache* cache = cache_handle->value();
-    cache->GetSizeThenClose(base::Bind(&SizeRetrievedFromCache,
+    cache->GetSizeThenClose(base::Bind(&CacheStorage::SizeRetrievedFromCache,
+                                       weak_factory_.GetWeakPtr(),
                                        base::Passed(std::move(cache_handle)),
                                        barrier_closure, accumulator_ptr));
   }
 }
 
 void CacheStorage::SizeImpl(const SizeCallback& callback) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK(initialized_);
 
+  if (index_->GetStorageSize() != kSizeUnknown) {
+    base::ThreadTaskRunnerHandle::Get()->PostTask(
+        FROM_HERE, base::Bind(callback, index_->GetStorageSize()));
+    return;
+  }
+
   std::unique_ptr<int64_t> accumulator(new int64_t(0));
   int64_t* accumulator_ptr = accumulator.get();
 
   base::Closure barrier_closure = base::BarrierClosure(
-      ordered_cache_names_.size(),
+      index_->num_entries(),
       base::Bind(&SizeRetrievedFromAllCaches,
                  base::Passed(std::move(accumulator)), callback));
 
-  for (const std::string& cache_name : ordered_cache_names_) {
+  for (const auto& cache_metadata : index_->ordered_cache_metadata()) {
+    if (cache_metadata.size != CacheStorage::kSizeUnknown) {
+      *accumulator_ptr += cache_metadata.size;
+      barrier_closure.Run();
+      continue;
+    }
     std::unique_ptr<CacheStorageCacheHandle> cache_handle =
-        GetLoadedCache(cache_name);
+        GetLoadedCache(cache_metadata.name);
     CacheStorageCache* cache = cache_handle->value();
-    cache->Size(base::Bind(&SizeRetrievedFromCache,
+    cache->Size(base::Bind(&CacheStorage::SizeRetrievedFromCache,
+                           weak_factory_.GetWeakPtr(),
                            base::Passed(std::move(cache_handle)),
                            barrier_closure, accumulator_ptr));
   }
 }
 
 }  // namespace content