Avoid fragmenting the heap too much while reconstructing the index.

net/disk_cache/simple/simple_index_file.h

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_
 #define NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_
 
 #include <string>
 #include <vector>
 
 #include "base/basictypes.h"
 #include "base/containers/hash_tables.h"
 #include "base/files/file_path.h"
 #include "base/gtest_prod_util.h"
 #include "base/logging.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/pickle.h"
+#include "base/platform_file.h"

[gavinp, 2013/08/21 11:49:47]

Pourquoi? Just forward declare base::PlatformFileInfo I say!

[Philippe, 2013/08/21 12:41:38]

Yeah my bad. I used to have GetPlatformFileInfo() return a value-typed
PlatformFileInfo.

 #include "base/port.h"
 #include "net/base/net_export.h"
 #include "net/disk_cache/simple/simple_index.h"
 
 namespace base {
 class SingleThreadTaskRunner;
 class TaskRunner;
 }
 
 namespace disk_cache {
@@ skipping 62 matching lines @@
                            bool app_on_background);
 
   // Doom the entries specified in |entry_hashes|, calling |reply_callback|
   // with the result on the current thread when done.
   virtual void DoomEntrySet(scoped_ptr<std::vector<uint64> > entry_hashes,
                             const base::Callback<void(int)>& reply_callback);
 
  private:
   friend class WrappedSimpleIndexFile;
 
+  // Used for cache directory traversal.
+  typedef base::Callback<void (const char* file_name)> EntryFileCallback;
+
   // When loading the entries from disk, add this many extra hash buckets to
   // prevent reallocation on the IO thread when merging in new live entries.
   static const int kExtraSizeForMerge = 512;
 
   // Synchronous (IO performing) implementation of LoadIndexEntries.
   static void SyncLoadIndexEntries(base::Time cache_last_modified,
                                    const base::FilePath& cache_directory,
                                    const base::FilePath& index_file_path,
                                    SimpleIndexLoadResult* out_result);
 
   // Load the index file from disk returning an EntrySet. Upon failure, returns
   // NULL.
   static void SyncLoadFromDisk(const base::FilePath& index_filename,
                                SimpleIndexLoadResult* out_result);
 
   // Returns a scoped_ptr for a newly allocated Pickle containing the serialized
   // data to be written to a file.
   static scoped_ptr<Pickle> Serialize(
       const SimpleIndexFile::IndexMetadata& index_metadata,
       const SimpleIndex::EntrySet& entries);
 
   // Given the contents of an index file |data| of length |data_len|, returns
   // the corresponding EntrySet. Returns NULL on error.
-  static void Deserialize(const char* data, int data_len,
+  static void Deserialize(const char* data,
+                          int data_len,
                           SimpleIndexLoadResult* out_result);
 
+  // Implemented either in simple_index_file_posix.cc or
+  // simple_index_file_win.cc. base::FileEnumerator turned out to be very
+  // expensive in terms of memory usage therefore it's used only on non-POSIX
+  // environments for convenience (for now). Note that this function is
+  // synchronous although its signature could suggest otherwise.  Returns
+  // whether the traversal succeeded.
+  static bool TraverseCacheDirectory(
+      const std::string& cache_path,
+      const EntryFileCallback& entry_file_callback);
+
+  // Returns whether file info was successfully retrieved.
+  static bool GetPlatformFileInfo(const char* path,
+                                  base::PlatformFileInfo* info);
+
+  // Called for each cache directory traversal iteration. The provided
+  // non-cleared |buffer| can be used accross iterations to avoid doing too many
+  // heap allocations given that cache directories can be very large.
+  static void ProcessEntryFile(const std::string& cache_path,

[gavinp, 2013/08/21 11:49:47]

Can this method be moved into an anonymous namespace in simple_index_file.cc
easily?

[Philippe, 2013/08/21 12:41:38]

Yeah good point.

+                               std::vector<char>* buffer,

[gavinp, 2013/08/21 11:49:47]

char*! Let your inner C programmer out.

[Philippe, 2013/08/21 12:41:38]

Ahaha :) I got rid of the buffer.

+                               SimpleIndex::EntrySet* entries,
+                               const char* file_name);
+
   // Scan the index directory for entries, returning an EntrySet of all entries
   // found.
   static void SyncRestoreFromDisk(const base::FilePath& cache_directory,
                                   const base::FilePath& index_file_path,
                                   SimpleIndexLoadResult* out_result);
 
   // Determines if an index file is stale relative to the time of last
   // modification of the cache directory.
   static bool IsIndexFileStale(base::Time cache_last_modified,
                                const base::FilePath& index_file_path);
@@ skipping 11 matching lines @@
   static const char kIndexFileName[];
   static const char kTempIndexFileName[];
 
   DISALLOW_COPY_AND_ASSIGN(SimpleIndexFile);
 };
 
 
 }  // namespace disk_cache
 
 #endif  // NET_DISK_CACHE_SIMPLE_SIMPLE_INDEX_FILE_H_