Avoid fragmenting the heap too much while reconstructing the index.

net/disk_cache/simple/simple_index_file.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/disk_cache/simple/simple_index_file.h"
 
 #include <vector>
 
 #include "base/file_util.h"
-#include "base/files/file_enumerator.h"
 #include "base/files/memory_mapped_file.h"
 #include "base/hash.h"
 #include "base/logging.h"
 #include "base/metrics/histogram.h"
 #include "base/pickle.h"
 #include "base/single_thread_task_runner.h"
 #include "base/task_runner_util.h"
 #include "base/threading/thread_restrictions.h"
 #include "net/disk_cache/simple/simple_entry_format.h"
 #include "net/disk_cache/simple/simple_index.h"
 #include "net/disk_cache/simple/simple_synchronous_entry.h"
 #include "net/disk_cache/simple/simple_util.h"
 #include "third_party/zlib/zlib.h"
 
-
+namespace disk_cache {
 namespace {
 
+const int kEntryFilesHashLength = 16;
+const int kEntryFilesSuffixLength = 2;
+
 const uint64 kMaxEntiresInIndex = 100000000;
 
+const char kFakeIndexFileName[] = "index";
+const char kIndexFileName[] = "the-real-index";
+const char kTempIndexFileName[] = "temp-index";
+
 uint32 CalculatePickleCRC(const Pickle& pickle) {
   return crc32(crc32(0, Z_NULL, 0),
                reinterpret_cast<const Bytef*>(pickle.payload()),
                pickle.payload_size());
 }
 
 void DoomEntrySetReply(const net::CompletionCallback& reply_callback,
                        int result) {
   reply_callback.Run(result);
 }
@@ skipping 20 matching lines @@
   }
   if (app_on_background) {
     UMA_HISTOGRAM_TIMES("SimpleCache.IndexWriteToDiskTime.Background",
                         (base::TimeTicks::Now() - start_time));
   } else {
     UMA_HISTOGRAM_TIMES("SimpleCache.IndexWriteToDiskTime.Foreground",
                         (base::TimeTicks::Now() - start_time));
   }
 }
 
+// Called for each cache directory traversal iteration.
+void ProcessEntryFile(SimpleIndex::EntrySet* entries,
+                      const base::FilePath& file_path) {
+  static const size_t kEntryFilesLength =
+      kEntryFilesHashLength + kEntryFilesSuffixLength;
+  const base::FilePath::StringType file_name_string =
+      file_path.BaseName().value();
+  if (file_name_string.size() != kEntryFilesLength) {
+    if (file_name_string != kFakeIndexFileName &&
+            file_name_string != kIndexFileName) {
+      LOG(ERROR) << "unexpected file " << file_name_string;

[pasko, 2013/08/21 16:03:17]

it would be easy to forget to update this check when updating the structure of
the cache directory with something new, say new directory. So expect firing this
"useful" error message a bit later.

I'd just remove it.

[Philippe, 2013/08/21 16:20:58]

Done.

+    }
+    return;
+  }
+
+  const base::StringPiece hash_string(
+      file_name_string.begin(),
+      file_name_string.begin() + kEntryFilesHashLength);
+  uint64 hash_key = 0;
+  if (!simple_util::GetEntryHashKeyFromHexString(hash_string, &hash_key)) {
+    LOG(WARNING) << "Invalid entry hash key filename while restoring index from"
+                 << " disk: " << file_name_string;
+    // TODO(felipeg): Should we delete the invalid file here ?

[pasko, 2013/08/21 16:03:17]

I am absolutely sure we must *not* delete the invalid file. Please remove the
TODO.

[Philippe, 2013/08/21 16:20:58]

Done.

+    return;
+  }
+
+  base::PlatformFileInfo file_info;
+  if (!file_util::GetFileInfo(file_path, &file_info)) {
+    LOG(ERROR) << "Could not get file info for " << file_path.value();
+    return;
+  }
+  base::Time last_used_time;
+#if defined(OS_POSIX)
+  // For POSIX systems, a last access time is available. However, it's not
+  // guaranteed to be more accurate than mtime. It is no worse though.
+  last_used_time = file_info.last_accessed;
+#endif
+  if (last_used_time.is_null())
+    last_used_time = file_info.last_modified;
+
+  int64 file_size = file_info.size;
+  SimpleIndex::EntrySet::iterator it = entries->find(hash_key);
+  if (it == entries->end()) {
+    SimpleIndex::InsertInEntrySet(
+        hash_key,
+        EntryMetadata(last_used_time, file_size),
+        entries);
+  } else {
+    // Summing up the total size of the entry through all the *_[0-2] files
+    it->second.SetEntrySize(it->second.GetEntrySize() + file_size);
+  }
+}
+
 }  // namespace
 
-namespace disk_cache {
-
 SimpleIndexLoadResult::SimpleIndexLoadResult() : did_load(false),
                                                  flush_required(false) {
 }
 
 SimpleIndexLoadResult::~SimpleIndexLoadResult() {
 }
 
 void SimpleIndexLoadResult::Reset() {
   did_load = false;
   flush_required = false;
   entries.clear();
 }
 
-// static
-const char SimpleIndexFile::kIndexFileName[] = "the-real-index";
-// static
-const char SimpleIndexFile::kTempIndexFileName[] = "temp-index";
-
 SimpleIndexFile::IndexMetadata::IndexMetadata() :
     magic_number_(kSimpleIndexMagicNumber),
     version_(kSimpleVersion),
     number_of_entries_(0),
     cache_size_(0) {}
 
 SimpleIndexFile::IndexMetadata::IndexMetadata(
     uint64 number_of_entries, uint64 cache_size) :
     magic_number_(kSimpleIndexMagicNumber),
     version_(kSimpleVersion),
@@ skipping 241 matching lines @@
 
   out_result->did_load = true;
 }
 
 // static
 void SimpleIndexFile::SyncRestoreFromDisk(
     const base::FilePath& cache_directory,
     const base::FilePath& index_file_path,
     SimpleIndexLoadResult* out_result) {
   LOG(INFO) << "Simple Cache Index is being restored from disk.";
-
   base::DeleteFile(index_file_path, /* recursive = */ false);
   out_result->Reset();
   SimpleIndex::EntrySet* entries = &out_result->entries;
 
   // TODO(felipeg,gavinp): Fix this once we have a one-file per entry format.
   COMPILE_ASSERT(kSimpleEntryFileCount == 3,
                  file_pattern_must_match_file_count);
 
-  const int kFileSuffixLength = sizeof("_0") - 1;
-  const base::FilePath::StringType file_pattern = FILE_PATH_LITERAL("*_[0-2]");
-  base::FileEnumerator enumerator(cache_directory,
-                                  false /* recursive */,
-                                  base::FileEnumerator::FILES,
-                                  file_pattern);
-  for (base::FilePath file_path = enumerator.Next(); !file_path.empty();
-       file_path = enumerator.Next()) {
-    const base::FilePath::StringType base_name = file_path.BaseName().value();
-    // Converting to std::string is OK since we never use UTF8 wide chars in our
-    // file names.
-    const std::string hash_key_string(base_name.begin(),
-                                      base_name.end() - kFileSuffixLength);
-    uint64 hash_key = 0;
-    if (!simple_util::GetEntryHashKeyFromHexString(
-            hash_key_string, &hash_key)) {
-      LOG(WARNING) << "Invalid Entry Hash Key filename while restoring "
-                   << "Simple Index from disk: " << base_name;
-      // TODO(felipeg): Should we delete the invalid file here ?
-      continue;
-    }
-
-    base::FileEnumerator::FileInfo info = enumerator.GetInfo();
-    base::Time last_used_time;
-#if defined(OS_POSIX)
-    // For POSIX systems, a last access time is available. However, it's not
-    // guaranteed to be more accurate than mtime. It is no worse though.
-    last_used_time = base::Time::FromTimeT(info.stat().st_atime);
-#endif
-    if (last_used_time.is_null())
-      last_used_time = info.GetLastModifiedTime();
-
-    int64 file_size = info.GetSize();
-    SimpleIndex::EntrySet::iterator it = entries->find(hash_key);
-    if (it == entries->end()) {
-      SimpleIndex::InsertInEntrySet(
-          hash_key,
-          EntryMetadata(last_used_time, file_size),
-          entries);
-    } else {
-      // Summing up the total size of the entry through all the *_[0-2] files
-      it->second.SetEntrySize(it->second.GetEntrySize() + file_size);
-    }
+  const bool did_succeed = TraverseCacheDirectory(
+      cache_directory, base::Bind(&ProcessEntryFile, entries));
+  if (!did_succeed) {
+    LOG(ERROR) << "Could not reconstruct index from disk";
+    return;
   }
-
   out_result->did_load = true;
-
   // When we restore from disk we write the merged index file to disk right
   // away, this might save us from having to restore again next time.
   out_result->flush_required = true;
 }
 
 // static
 bool SimpleIndexFile::IsIndexFileStale(base::Time cache_last_modified,
                                        const base::FilePath& index_file_path) {
   base::Time index_mtime;
   if (!simple_util::GetMTime(index_file_path, &index_mtime))
     return true;
   return index_mtime < cache_last_modified;
 }
 
 }  // namespace disk_cache