Avoid fragmenting the heap too much while reconstructing the index.

net/disk_cache/simple/simple_index_file.h

Index: net/disk_cache/simple/simple_index_file.h
diff --git a/net/disk_cache/simple/simple_index_file.h b/net/disk_cache/simple/simple_index_file.h
index b536df9a1e72cbaaa8a32f01ef33e82fa6f915d3..9885c8f970d3629e06ea53a13786a7f1629eda4a 100644
--- a/net/disk_cache/simple/simple_index_file.h
+++ b/net/disk_cache/simple/simple_index_file.h
@@ -97,6 +97,9 @@ class NET_EXPORT_PRIVATE SimpleIndexFile {
  private:
   friend class WrappedSimpleIndexFile;
 
+  // Used for cache directory traversal.
+  typedef base::Callback<void (const std::string& file_name)> EntryFileCallback;
+
   // When loading the entries from disk, add this many extra hash buckets to
   // prevent reallocation on the IO thread when merging in new live entries.
   static const int kExtraSizeForMerge = 512;
@@ -123,6 +126,24 @@ class NET_EXPORT_PRIVATE SimpleIndexFile {
   static void Deserialize(const char* data, int data_len,
                           SimpleIndexLoadResult* out_result);
 
+  // Implemented either in simple_index_file_posix.cc or
+  // simple_index_file_win.cc. base::FileEnumerator turned out to be very
+  // expensive in terms of memory usage therefore it's used only on non-POSIX
+  // environments for convenience (for now). Note that this function is
+  // synchronous although its signature could suggest otherwise.  Returns
+  // whether the traversal succeeded.
+  static bool TraverseCacheDirectory(
+      const std::string& cache_path,
+      const EntryFileCallback& entry_file_callback);
+
+  // Called for each cache directory traversal iteration. The provided
+  // non-cleared |buffer| can be used accross iterations to avoid doing too many
+  // heap allocations given that cache directories can be very large.
+  static void ProcessEntryFile(const std::string& cache_path,

[gavinp, 2013/08/20 20:13:45]

I suggest you want:

static void ProcessEntryFile(SimpleIndex::EntrySet* entries,
                                          uint64 entry_hash,
                                          const base::PlatformFileInfo&
entry_info);

Which can be for-sure-for-sure not allocating.

[Philippe, 2013/08/21 09:37:33]

I want to keep the hash extraction in this method to avoid duplicating the logic
on Windows and POSIX (unless I'm missing your point). Let me know if you are ok
with the new signature.

+                               std::string* buffer,

[gavinp, 2013/08/20 18:53:23]

I'm a bit confused at this buffer being a std::string, it's possible that the
std::string will reallocate on every call to std::string::append or
std::string::assign, isn't it?

[Philippe, 2013/08/21 09:37:33]

I replaced it with a vector that does now a single resize (and uses std::copy
with a non-back_inserter iterator).

+                               SimpleIndex::EntrySet* entries,
+                               const std::string& file_name);
+
   // Scan the index directory for entries, returning an EntrySet of all entries
   // found.
   static void SyncRestoreFromDisk(const base::FilePath& cache_directory,