Refactor our SimpleIndex file format and serialization to use Pickle instead of the previously bugg…

net/disk_cache/simple/simple_index_file.cc

+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/disk_cache/simple/simple_index_file.h"
+
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/file_util.h"
+#include "base/hash.h"
+#include "base/logging.h"
+#include "base/message_loop.h"
+#include "base/pickle.h"
+#include "base/task_runner.h"
+#include "net/disk_cache/simple/simple_disk_format.h"
+#include "net/disk_cache/simple/simple_index_util.h"
+#include "third_party/zlib/zlib.h"
+
+namespace {
+
+const uint64 kMaxEntiresInIndex = 100000000;
+
+}  // namespace
+
+namespace disk_cache {
+
+SimpleIndexFile::IndexMetadata::IndexMetadata() :
+    initial_magic_number_(kSimpleIndexInitialMagicNumber),
+    version_(kSimpleVersion),
+    number_of_entries_(0),
+    cache_size_(0) {}
+
+SimpleIndexFile::IndexMetadata::IndexMetadata(
+    uint64 number_of_entries, uint64 cache_size) :
+    initial_magic_number_(kSimpleIndexInitialMagicNumber),
+    version_(kSimpleVersion),
+    number_of_entries_(number_of_entries),
+    cache_size_(cache_size) {}
+
+void SimpleIndexFile::IndexMetadata::Serialize(Pickle* pickle) const {
+  DCHECK(pickle);
+  pickle->WriteUInt64(initial_magic_number_);
+  pickle->WriteUInt32(version_);
+  pickle->WriteUInt64(number_of_entries_);
+  pickle->WriteUInt64(cache_size_);
+}
+
+bool SimpleIndexFile::IndexMetadata::DeSerialize(PickleIterator* it,
+                                                 IndexMetadata* out) {
+  DCHECK(it);
+  DCHECK(out);
+  return it->ReadUInt64(&(out->initial_magic_number_)) &&
+      it->ReadUInt32(&(out->version_)) &&
+      it->ReadUInt64(&(out->number_of_entries_))&&
+      it->ReadUInt64(&(out->cache_size_));
+}
+
+bool SimpleIndexFile::IndexMetadata::CheckIndexMetadata() {
+  return number_of_entries_ <= kMaxEntiresInIndex &&
+      initial_magic_number_ == disk_cache::kSimpleIndexInitialMagicNumber &&
+      version_ == disk_cache::kSimpleVersion;
+}
+
+// static
+void SimpleIndexFile::LoadFromDisk(
+    const base::FilePath& index_filename,
+    const scoped_refptr<base::TaskRunner>& callback_runner,
+    const IndexCompletionCallback& completion_callback) {
+  std::string contents;
+  if(!file_util::ReadFileToString(index_filename,
+                                  &contents)) {
+    LOG(WARNING) << "Could not read Simple Index file.";
+    return RestoreFromDisk(
+        index_filename, callback_runner, completion_callback);
+  }
+
+  Pickle pickle (contents.data(), contents.size());
+  PickleIterator pickle_it(pickle);
+
+  SimpleIndexFile::PickleHeader* header_p =
+      pickle.headerT<SimpleIndexFile::PickleHeader>();
+  const uint32 crc_read = header_p->crc;
+  const uint32 crc_calculated = CalculatePickleCRC(pickle);
+
+  if (crc_read != crc_calculated) {
+    LOG(WARNING) << "Invalid CRC in Simple Index file.";
+    return RestoreFromDisk(
+        index_filename, callback_runner, completion_callback);
+  }
+
+  SimpleIndexFile::IndexMetadata index_metadata;
+  if (!SimpleIndexFile::IndexMetadata::DeSerialize(
+          &pickle_it, &index_metadata)) {
+    LOG(ERROR) << "Invalid index_metadata on Simple Cache Index.";
+    return RestoreFromDisk(
+        index_filename, callback_runner, completion_callback);
+  }
+
+  if (!index_metadata.CheckIndexMetadata()) {
+    LOG(ERROR) << "Invalid index_metadata on Simple Cache Index.";
+    return RestoreFromDisk(
+        index_filename, callback_runner, completion_callback);
+  }
+
+  scoped_ptr<EntrySet> index_file_entries(new EntrySet());
+  while (index_file_entries->size() < index_metadata.GetNumberOfEntries()) {
+    EntryMetadata entry_metadata;
+    if (!EntryMetadata::DeSerialize(&pickle_it, &entry_metadata)) {
+      LOG(WARNING) << "Invalid EntryMetadata in Simple Index file.";
+      return RestoreFromDisk(
+          index_filename, callback_runner, completion_callback);
+    }
+    InsertInEntrySet(index_file_entries.get(), entry_metadata);
+  }
+  callback_runner->PostTask(FROM_HERE,
+                            base::Bind(completion_callback,

[pasko-google - do not use, 2013/04/15 14:23:55]

can we please make pickling un-pickling independent from callbacks? It would be
easier to test then, also easier to read when serialization/deserialization is
contained within one plain data object.

All the job of passing data across threads would be more readable to find in the
backend, not even in the SimpleIndex

[felipeg, 2013/04/15 15:37:04]

Done.

+                                       base::Passed(&index_file_entries)));
+}
+
+// static
+void SimpleIndexFile::RestoreFromDisk(
+    const base::FilePath& index_filename,
+    const scoped_refptr<base::TaskRunner>& callback_runner,
+    const IndexCompletionCallback& completion_callback) {
+  using file_util::FileEnumerator;
+  LOG(INFO) << "Simple Cache Index is being restored from disk.";
+
+  file_util::Delete(index_filename, /* recursive = */ false);
+  scoped_ptr<EntrySet> index_file_entries(new EntrySet());
+
+  // TODO(felipeg,gavinp): Fix this once we have a one-file per entry format.
+  COMPILE_ASSERT(kSimpleEntryFileCount == 3,
+                 file_pattern_must_match_file_count);
+  const base::FilePath::StringType file_pattern = FILE_PATH_LITERAL("*_[0-2]");
+  FileEnumerator enumerator(index_filename.DirName(),
+                            false /* recursive */,
+                            FileEnumerator::FILES,
+                            file_pattern);
+  for (base::FilePath file_path = enumerator.Next(); !file_path.empty();
+       file_path = enumerator.Next()) {
+    const base::FilePath::StringType base_name = file_path.BaseName().value();
+    // Converting to std::string is OK since we never use UTF8 wide chars in our
+    // file names.
+    const std::string hash_name(base_name.begin(), base_name.end());
+    const std::string hash_key_string =
+        hash_name.substr(0, kEntryHashKeyAsHexStringSize);
+    uint64 hash_key = 0;
+    if (!GetEntryHashKeyFromHexString(hash_key_string, &hash_key)) {
+      LOG(WARNING) << "Invalid Entry Hash Key filename while restoring "
+                   << "Simple Index from disk: " << hash_name;
+      // TODO(felipeg): Should we delete the invalid file here ?
+      continue;
+    }
+
+    FileEnumerator::FindInfo find_info = {};
+    enumerator.GetFindInfo(&find_info);
+    base::Time last_used_time;
+#if defined(OS_POSIX)
+    // For POSIX systems, a last access time is available. However, it's not
+    // guaranteed to be more accurate than mtime. It is no worse though.
+    last_used_time = base::Time::FromTimeT(find_info.stat.st_atime);
+#endif
+    if (last_used_time.is_null())
+      last_used_time = FileEnumerator::GetLastModifiedTime(find_info);
+
+    int64 file_size = FileEnumerator::GetFilesize(find_info);
+    EntrySet::iterator it = index_file_entries->find(hash_key);
+    if (it == index_file_entries->end()) {
+      InsertInEntrySet(index_file_entries.get(), EntryMetadata(
+          hash_key, last_used_time, file_size));
+    } else {
+      // Summing up the total size of the entry through all the *_[0-2] files
+      it->second.SetEntrySize(it->second.GetEntrySize() + file_size);
+    }
+  }
+
+  callback_runner->PostTask(FROM_HERE,
+                            base::Bind(completion_callback,
+                                       base::Passed(&index_file_entries)));
+}
+
+// static
+scoped_ptr<Pickle> SimpleIndexFile::Serialize(
+    const SimpleIndexFile::IndexMetadata& index_metadata,
+    const EntrySet& entries) {
+  scoped_ptr<Pickle> pickle(new Pickle(sizeof(SimpleIndexFile::PickleHeader)));
+
+  // Serialize the |index_metadata|.
+  index_metadata.Serialize(pickle.get());
+  // Then all the entries from |entries| set.
+  for (EntrySet::const_iterator it = entries.begin();
+       it != entries.end(); ++it) {
+    it->second.Serialize(pickle.get());
+  }
+  SimpleIndexFile::PickleHeader* header_p =
+      pickle->headerT<SimpleIndexFile::PickleHeader>();
+  header_p->crc = CalculatePickleCRC(*pickle);
+  return pickle.Pass();
+}
+
+// static
+void SimpleIndexFile::WriteToDisk(const base::FilePath& index_filename,
+                                  scoped_ptr<Pickle> pickle) {
+  const base::FilePath temp_filename =
+      index_filename.DirName().AppendASCII("index_temp");
+  int bytes_written = file_util::WriteFile(
+      temp_filename,
+      reinterpret_cast<const char*>(pickle->data()),
+      pickle->size());
+  DCHECK_EQ(bytes_written, implicit_cast<int>(pickle->size()));
+  if (bytes_written != static_cast<int>(pickle->size())) {
+    // TODO(felipeg): Add better error handling.
+    LOG(ERROR) << "Could not write Simple Cache index to temporary file: "
+               << temp_filename.value();
+    file_util::Delete(temp_filename, /* recursive = */ false);
+    return;
+  }
+  // Swap temp and index_file.
+  bool result = file_util::ReplaceFile(temp_filename, index_filename);
+  DCHECK(result);
+}
+
+// static
+uint32 SimpleIndexFile::CalculatePickleCRC(const Pickle& pickle) {
+  return crc32(crc32(0, Z_NULL, 0),
+               reinterpret_cast<const Bytef*>(pickle.payload()),
+               implicit_cast<uInt>(pickle.payload_size()));
+}
+
+}  // namespace disk_cache