Asynchronous initialization in Simple Index.

net/disk_cache/simple/simple_index.cc

Index: net/disk_cache/simple/simple_index.cc
diff --git a/net/disk_cache/simple/simple_index.cc b/net/disk_cache/simple/simple_index.cc
index 31009f9d4130e95d7fd3a6471e347ac330770180..7b18068ab027d2faf01c190dffd6021f19e0f71a 100644
--- a/net/disk_cache/simple/simple_index.cc
+++ b/net/disk_cache/simple/simple_index.cc
@@ -43,13 +43,23 @@ namespace disk_cache {
 
 SimpleIndex::SimpleIndex(
     const scoped_refptr<base::TaskRunner>& cache_thread,
+    const scoped_refptr<base::TaskRunner>& io_thread,
     const base::FilePath& path)
     : path_(path),
-      cache_thread_(cache_thread) {
-  index_filename_ = path_.AppendASCII("simple-index");
+      cache_size_(0),
+      initialized_(false),
+      index_filename_(path_.AppendASCII("simple-index")),
+      cache_thread_(cache_thread),
+      io_thread_(io_thread) { }
+
+void SimpleIndex::Initialize() {
+  base::WorkerPool::PostTask(FROM_HERE,
+                             base::Bind(&SimpleIndex::LoadFromDisk,
+                                        base::Unretained(this)),

[gavinp, 2013/04/10 11:45:44]

I don't think this use of base::Unretained is safe. Can't the destructor be
called before the cache is done initializing?

[felipeg, 2013/04/10 14:21:45]

Done.

+                             true);
 }
 
-bool SimpleIndex::Initialize() {
+void SimpleIndex::LoadFromDisk() {
   if (!OpenIndexFile())
     return RestoreFromDisk();
   uLong incremental_crc = crc32(0L, Z_NULL, 0);
@@ -103,35 +113,49 @@ bool SimpleIndex::Initialize() {
     SimpleIndexFile::EntryMetadata entry_metadata;
     SimpleIndexFile::EntryMetadata::DeSerialize(
         &entries_buffer.get()[entries_buffer_offset], &entry_metadata);
-    InsertInternal(entry_metadata);
+    InsertInternal(&initializing_set_, entry_metadata);
     entries_buffer_offset += SimpleIndexFile::kEntryMetadataSize;
   }
-  DCHECK_EQ(header.number_of_entries, entries_set_.size());
+  DCHECK_EQ(header.number_of_entries, initializing_set_.size());
   CloseIndexFile();
-  return true;
+
+  io_thread_->PostTask(FROM_HERE,
+                       base::Bind(&SimpleIndex::MergeInitializingSet,
+                                  base::Unretained(this)));
 }
 
 void SimpleIndex::Insert(const std::string& key) {
   // Upon insert we don't know yet the size of the entry.
   // It will be updated later when the SynchronousEntryImpl finish and the
   // UpdateEntrySize will be called.
-  InsertInternal(SimpleIndexFile::EntryMetadata(GetEntryHashForKey(key),
+  const std::string hash_key = GetEntryHashForKey(key);
+  InsertInternal(&entries_set_, SimpleIndexFile::EntryMetadata(hash_key,
                                                 base::Time::Now(), 0));
+  if (!initialized_)
+    removals_set_.erase(hash_key);
 }
 
 void SimpleIndex::Remove(const std::string& key) {
   UpdateEntrySize(key, 0);
-  entries_set_.erase(GetEntryHashForKey(key));
+  const std::string hash_key = GetEntryHashForKey(key);
+  entries_set_.erase(hash_key);
+
+  if (!initialized_)
+    removals_set_.insert(hash_key);
 }
 
 bool SimpleIndex::Has(const std::string& key) const {
-  return entries_set_.count(GetEntryHashForKey(key)) != 0;
+  // If not initialized, always return true, forcing it to go to the disk.
+  return !initialized_ || entries_set_.count(GetEntryHashForKey(key)) != 0;
 }
 
 bool SimpleIndex::UseIfExists(const std::string& key) {
+  // Always update the last used time, even if it is during initialization.
+  // It will be merged later.
   EntrySet::iterator it = entries_set_.find(GetEntryHashForKey(key));
   if (it == entries_set_.end())
-    return false;
+    // If not initialized, always return true, forcing it to go to the disk.
+    return !initialized_;
   it->second.SetLastUsedTime(base::Time::Now());
   return true;
 }
@@ -149,17 +173,21 @@ bool SimpleIndex::UpdateEntrySize(const std::string& key, uint64 entry_size) {
   return true;
 }
 
+// static
 void SimpleIndex::InsertInternal(
+    EntrySet* entry_set,
     const SimpleIndexFile::EntryMetadata& entry_metadata) {
-  entries_set_.insert(make_pair(entry_metadata.GetHashKey(), entry_metadata));
+  DCHECK(entry_set);
+  entry_set->insert(make_pair(entry_metadata.GetHashKey(), entry_metadata));
 }
 
-bool SimpleIndex::RestoreFromDisk() {
+void SimpleIndex::RestoreFromDisk() {
   using file_util::FileEnumerator;
   LOG(INFO) << "Simple Cache Index is being restored from disk.";
   CloseIndexFile();
   file_util::Delete(index_filename_, /* recursive = */ false);
-  entries_set_.clear();
+  DCHECK_EQ(0U, initializing_set_.size());
+
   // TODO(felipeg,gavinp): Fix this once we have a one-file per entry format.
   const base::FilePath::StringType file_pattern = FILE_PATH_LITERAL("*_[0-2]");
   FileEnumerator enumerator(path_,
@@ -187,9 +215,9 @@ bool SimpleIndex::RestoreFromDisk() {
       last_used_time = FileEnumerator::GetLastModifiedTime(find_info);
 
     int64 file_size = FileEnumerator::GetFilesize(find_info);
-    EntrySet::iterator it = entries_set_.find(hash_key);
-    if (it == entries_set_.end()) {
-      InsertInternal(SimpleIndexFile::EntryMetadata(
+    EntrySet::iterator it = initializing_set_.find(hash_key);
+    if (it == initializing_set_.end()) {
+      InsertInternal(&initializing_set_, SimpleIndexFile::EntryMetadata(
           hash_key, last_used_time, file_size));
     } else {
       // Summing up the total size of the entry through all the *_[0-2] files
@@ -197,8 +225,39 @@ bool SimpleIndex::RestoreFromDisk() {
     }
   }
 
-  // TODO(felipeg): Detect unrecoverable problems and return false here.
-  return true;
+  io_thread_->PostTask(FROM_HERE,
+                       base::Bind(&SimpleIndex::MergeInitializingSet,
+                                  base::Unretained(this)));
+}
+
+void SimpleIndex::MergeInitializingSet() {
+  // First, remove the entries that are in the |removals_set_| from both sets.
+  for (base::hash_set<std::string>::const_iterator it = removals_set_.begin();
+       it != removals_set_.end(); ++it) {
+    entries_set_.erase(*it);
+    initializing_set_.erase(*it);
+  }
+
+  // Recalculate the cache size while merging the two sets.
+  cache_size_ = 0;
+  for (EntrySet::const_iterator it = initializing_set_.begin();
+       it != initializing_set_.end(); ++it) {
+    // If there is already an entry in the current entries_set_, we need to
+    // merge the new data there with the data loaded in the initialization.
+    EntrySet::iterator current_entry = entries_set_.find(it->first);
+    if (current_entry != entries_set_.end()) {
+      // When Merging, existing valid data in the |current_entry| will prevail.
+      SimpleIndexFile::EntryMetadata::Merge(
+          it->second, &(current_entry->second));
+      cache_size_ += current_entry->second.entry_size;
+    } else {
+      InsertInternal(&entries_set_, it->second);
+      cache_size_ += it->second.entry_size;
+    }
+  }
+  initializing_set_.clear();
+
+  initialized_ = true;
 }
 
 void SimpleIndex::Serialize(std::string* out_buffer) {
@@ -232,7 +291,7 @@ void SimpleIndex::Serialize(std::string* out_buffer) {
   out_buffer->append(reinterpret_cast<const char*>(&footer), sizeof(footer));
 }
 
-void SimpleIndex::Cleanup() {
+void SimpleIndex::WriteToDisk() {
   scoped_ptr<std::string> buffer(new std::string());
   Serialize(buffer.get());
   cache_thread_->PostTask(FROM_HERE,