Disk cache: Reference CL for the implementation of file format version 3.

net/disk_cache/backend_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/disk_cache/backend_impl.h"
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/file_util.h"
 #include "base/files/file_path.h"
 #include "base/hash.h"
 #include "base/message_loop.h"
 #include "base/metrics/field_trial.h"
 #include "base/metrics/histogram.h"
 #include "base/metrics/stats_counters.h"
 #include "base/rand_util.h"
 #include "base/string_util.h"
 #include "base/stringprintf.h"
 #include "base/sys_info.h"
 #include "base/threading/thread_restrictions.h"
 #include "base/time.h"
 #include "base/timer.h"
 #include "net/base/net_errors.h"
 #include "net/disk_cache/cache_util.h"
+#include "net/disk_cache/disk_format.h"
 #include "net/disk_cache/entry_impl.h"
 #include "net/disk_cache/errors.h"
 #include "net/disk_cache/experiments.h"
 #include "net/disk_cache/file.h"
 
 // This has to be defined before including histogram_macros.h from this file.
 #define NET_DISK_CACHE_BACKEND_IMPL_CC_
 #include "net/disk_cache/histogram_macros.h"
 
 using base::Time;
@@ skipping 242 matching lines @@
   // the id cannot be zero, because that value is used as "not dirty".
   // Increasing the value once per second gives us many years before we start
   // having collisions.
   data_->header.this_id++;
   if (!data_->header.this_id)
     data_->header.this_id++;
 
   bool previous_crash = (data_->header.crash != 0);
   data_->header.crash = 1;
 
-  if (!block_files_.Init(create_files))
+  if (!block_files_.Init(create_files, kFirstAdditionalBlockFile))
     return net::ERR_FAILED;
 
   // We want to minimize the changes to cache for an AppCache.
   if (cache_type() == net::APP_CACHE) {
     DCHECK(!new_eviction_);
     read_only_ = true;
   } else if (cache_type() == net::SHADER_CACHE) {
     DCHECK(!new_eviction_);
   }
 
   eviction_.Init(this);
 
   // stats_ and rankings_ may end up calling back to us so we better be enabled.
   disabled_ = false;
-  if (!stats_.Init(this, &data_->header.stats))
+  if (!InitStats())
     return net::ERR_FAILED;
 
   disabled_ = !rankings_.Init(this, new_eviction_);
 
 #if defined(STRESS_CACHE_EXTENDED_VALIDATION)
   trace_object_->EnableTracing(false);
   int sc = SelfCheck();
   if (sc < 0 && sc != ERR_NUM_ENTRIES_MISMATCH)
     NOTREACHED();
   trace_object_->EnableTracing(true);
 #endif
 
   if (previous_crash) {
     ReportError(ERR_PREVIOUS_CRASH);
   } else if (!restarted_) {
     ReportError(ERR_NO_ERROR);
   }
 
   FlushIndex();
 
   return disabled_ ? net::ERR_FAILED : net::OK;
 }
 
 void BackendImpl::CleanupCache() {
   Trace("Backend Cleanup");
   eviction_.Stop();
   timer_.reset();
 
   if (init_) {
-    stats_.Store();
+    StoreStats();
     if (data_)
       data_->header.crash = 0;
 
     if (user_flags_ & kNoRandom) {
       // This is a net_unittest, verify that we are not 'leaking' entries.
       File::WaitForPendingIO(&num_pending_io_);
       DCHECK(!num_refs_);
     } else {
       File::DropPendingIO();
     }
@@ skipping 257 matching lines @@
     block_files_.DeleteBlock(entry_address, false);
     block_files_.DeleteBlock(node_address, false);
     LOG(ERROR) << "Create entry failed " << key.c_str();
     stats_.OnEvent(Stats::CREATE_ERROR);
     return NULL;
   }
 
   cache_entry->BeginLogging(net_log_, true);
 
   // We are not failing the operation; let's add this to the map.
-  open_entries_[entry_address.value()] = cache_entry;
+  open_entries_[entry_address.value()] = cache_entry.get();
 
   // Save the entry.
   cache_entry->entry()->Store();
   cache_entry->rankings()->Store();
   IncreaseNumEntries();
   entry_count_++;
 
   // Link this entry through the index.
   if (parent.get()) {
     parent->SetNextAddress(entry_address);
   } else {
     data_->table[hash & mask_] = entry_address.value();
   }
 
   // Link this entry through the lists.
-  eviction_.OnCreateEntry(cache_entry);
+  eviction_.OnCreateEntry(cache_entry.get());
 
   CACHE_UMA(AGE_MS, "CreateTime", 0, start);
   stats_.OnEvent(Stats::CREATE_HIT);
   SIMPLE_STATS_COUNTER("disk_cache.miss");
   Trace("create entry hit ");
   FlushIndex();
   cache_entry->AddRef();
   return cache_entry.get();
 }
 
@@ skipping 356 matching lines @@
   stats_.OnEvent(Stats::FATAL_ERROR);
   LogStats();
   ReportError(error);
 
   // Setting the index table length to an invalid value will force re-creation
   // of the cache files.
   data_->header.table_len = 1;
   disabled_ = true;
 
   if (!num_refs_)
-    MessageLoop::current()->PostTask(FROM_HERE,
-        base::Bind(&BackendImpl::RestartCache, GetWeakPtr(), true));
+    base::MessageLoop::current()->PostTask(
+        FROM_HERE, base::Bind(&BackendImpl::RestartCache, GetWeakPtr(), true));
 }
 
 void BackendImpl::ReportError(int error) {
   STRESS_DCHECK(!error || error == ERR_PREVIOUS_CRASH ||
                 error == ERR_CACHE_CREATED);
 
   // We transmit positive numbers, instead of direct error codes.
   DCHECK_LE(error, 0);
   CACHE_UMA(CACHE_ERROR, "Error", 0, error * -1);
 }
@@ skipping 44 matching lines @@
   if (!data_)
     first_timer_ = false;
   if (first_timer_) {
     first_timer_ = false;
     if (ShouldReportAgain())
       ReportStats();
   }
 
   // Save stats to disk at 5 min intervals.
   if (time % 10 == 0)
-    stats_.Store();
+    StoreStats();
 }
 
 void BackendImpl::IncrementIoCount() {
   num_pending_io_++;
 }
 
 void BackendImpl::DecrementIoCount() {
   num_pending_io_--;
 }
 
@@ skipping 160 matching lines @@
   stats->push_back(item);
 
   item.first = "Max size";
   item.second = base::StringPrintf("%d", max_size_);
   stats->push_back(item);
 
   item.first = "Current size";
   item.second = base::StringPrintf("%d", data_->header.num_bytes);
   stats->push_back(item);
 
+  item.first = "Cache type";
+  item.second = "Blockfile Cache";
+  stats->push_back(item);
+
   stats_.GetItems(stats);
 }
 
 void BackendImpl::OnExternalCacheHit(const std::string& key) {
   background_queue_.OnExternalCacheHit(key);
 }
 
 // ------------------------------------------------------------------------
 
 // We just created a new file so we're going to write the header and set the
@@ skipping 27 matching lines @@
               base::PLATFORM_FILE_OPEN_ALWAYS |
               base::PLATFORM_FILE_EXCLUSIVE_WRITE;
   scoped_refptr<disk_cache::File> file(new disk_cache::File(
       base::CreatePlatformFile(index_name, flags, file_created, NULL)));
 
   if (!file->IsValid())
     return false;
 
   bool ret = true;
   if (*file_created)
-    ret = CreateBackingStore(file);
+    ret = CreateBackingStore(file.get());
 
   file = NULL;
   if (!ret)
     return false;
 
   index_ = new MappedFile();
   data_ = reinterpret_cast<Index*>(index_->Init(index_name, 0));
   if (!data_) {
     LOG(ERROR) << "Unable to map Index file";
     return false;
@@ skipping 37 matching lines @@
 
   if (!table_len)
     return;
 
   // If we already have a table, adjust the size to it.
   int current_max_size = MaxStorageSizeForTable(table_len);
   if (max_size_ > current_max_size)
     max_size_= current_max_size;
 }
 
+bool BackendImpl::InitStats() {
+  Addr address(data_->header.stats);
+  int size = stats_.StorageSize();
+
+  if (!address.is_initialized()) {
+    FileType file_type = Addr::RequiredFileType(size);
+    DCHECK_NE(file_type, EXTERNAL);
+    int num_blocks = Addr::RequiredBlocks(size, file_type);
+
+    if (!CreateBlock(file_type, num_blocks, &address))
+      return false;
+    return stats_.Init(NULL, 0, address);
+  }
+
+  if (!address.is_block_file()) {
+    NOTREACHED();
+    return false;
+  }
+
+  // Load the required data.
+  size = address.num_blocks() * address.BlockSize();
+  MappedFile* file = File(address);
+  if (!file)
+    return false;
+
+  scoped_ptr<char[]> data(new char[size]);
+  size_t offset = address.start_block() * address.BlockSize() +
+                  kBlockHeaderSize;
+  if (!file->Read(data.get(), size, offset))
+    return false;
+
+  if (!stats_.Init(data.get(), size, address))
+    return false;
+  if (cache_type_ == net::DISK_CACHE && ShouldReportAgain())
+    stats_.InitSizeHistogram();
+  return true;
+}
+
+void BackendImpl::StoreStats() {
+  int size = stats_.StorageSize();
+  scoped_ptr<char[]> data(new char[size]);
+  Addr address;
+  size = stats_.SerializeStats(data.get(), size, &address);
+  DCHECK(size);
+  if (!address.is_initialized())
+    return;
+
+  MappedFile* file = File(address);
+  if (!file)
+    return;
+
+  size_t offset = address.start_block() * address.BlockSize() +
+                  kBlockHeaderSize;
+  file->Write(data.get(), size, offset);  // ignore result.
+}
+
 void BackendImpl::RestartCache(bool failure) {
   int64 errors = stats_.GetCounter(Stats::FATAL_ERROR);
   int64 full_dooms = stats_.GetCounter(Stats::DOOM_CACHE);
   int64 partial_dooms = stats_.GetCounter(Stats::DOOM_RECENT);
-  int64 ga_evictions = stats_.GetCounter(Stats::GAJS_EVICTED);
   int64 last_report = stats_.GetCounter(Stats::LAST_REPORT);
 
   PrepareForRestart();
   if (failure) {
     DCHECK(!num_refs_);
     DCHECK(!open_entries_.size());
     DelayedCacheCleanup(path_);
   } else {
     DeleteCache(path_, false);
   }
 
   // Don't call Init() if directed by the unit test: we are simulating a failure
   // trying to re-enable the cache.
   if (unit_test_)
     init_ = true;  // Let the destructor do proper cleanup.
   else if (SyncInit() == net::OK) {
     stats_.SetCounter(Stats::FATAL_ERROR, errors);
     stats_.SetCounter(Stats::DOOM_CACHE, full_dooms);
     stats_.SetCounter(Stats::DOOM_RECENT, partial_dooms);
-    stats_.SetCounter(Stats::GAJS_EVICTED, ga_evictions);
     stats_.SetCounter(Stats::LAST_REPORT, last_report);
   }
 }
 
 void BackendImpl::PrepareForRestart() {
   // Reset the mask_ if it was not given by the user.
   if (!(user_flags_ & kMask))
     mask_ = 0;
 
   if (!(user_flags_ & kNewEviction))
@@ skipping 15 matching lines @@
   if (it != open_entries_.end()) {
     // Easy job. This entry is already in memory.
     EntryImpl* this_entry = it->second;
     this_entry->AddRef();
     *entry = this_entry;
     return 0;
   }
 
   STRESS_DCHECK(block_files_.IsValid(address));
 
-  if (!address.SanityCheckForEntry()) {
+  if (!address.SanityCheckForEntryV2()) {
     LOG(WARNING) << "Wrong entry address.";
     STRESS_NOTREACHED();
     return ERR_INVALID_ADDRESS;
   }
 
   scoped_refptr<EntryImpl> cache_entry(
       new EntryImpl(this, address, read_only_));
   IncreaseNumRefs();
   *entry = NULL;
 
@@ skipping 37 matching lines @@
   }
 
   // Prevent overwriting the dirty flag on the destructor.
   cache_entry->SetDirtyFlag(GetCurrentEntryId());
 
   if (cache_entry->dirty()) {
     Trace("Dirty entry 0x%p 0x%x", reinterpret_cast<void*>(cache_entry.get()),
           address.value());
   }
 
-  open_entries_[address.value()] = cache_entry;
+  open_entries_[address.value()] = cache_entry.get();
 
   cache_entry->BeginLogging(net_log_, false);
   cache_entry.swap(entry);
   return 0;
 }
 
 EntryImpl* BackendImpl::MatchEntry(const std::string& key, uint32 hash,
                                    bool find_parent, Addr entry_addr,
                                    bool* match_error) {
   Addr address(data_->table[hash & mask_]);
@@ skipping 38 matching lines @@
       } else {
         data_->table[hash & mask_] = child.value();
       }
 
       Trace("MatchEntry dirty %d 0x%x 0x%x", find_parent, entry_addr.value(),
             address.value());
 
       if (!error) {
         // It is important to call DestroyInvalidEntry after removing this
         // entry from the table.
-        DestroyInvalidEntry(cache_entry);
+        DestroyInvalidEntry(cache_entry.get());
         cache_entry = NULL;
       } else {
         Trace("NewEntry failed on MatchEntry 0x%x", address.value());
       }
 
       // Restart the search.
       address.set_value(data_->table[hash & mask_]);
       visited.clear();
       continue;
     }
@@ skipping 229 matching lines @@
   num_refs_++;
   if (max_refs_ < num_refs_)
     max_refs_ = num_refs_;
 }
 
 void BackendImpl::DecreaseNumRefs() {
   DCHECK(num_refs_);
   num_refs_--;
 
   if (!num_refs_ && disabled_)
-    MessageLoop::current()->PostTask(FROM_HERE,
-        base::Bind(&BackendImpl::RestartCache, GetWeakPtr(), true));
+    base::MessageLoop::current()->PostTask(
+        FROM_HERE, base::Bind(&BackendImpl::RestartCache, GetWeakPtr(), true));
 }
 
 void BackendImpl::IncreaseNumEntries() {
   data_->header.num_entries++;
   DCHECK_GT(data_->header.num_entries, 0);
 }
 
 void BackendImpl::DecreaseNumEntries() {
   data_->header.num_entries--;
   if (data_->header.num_entries < 0) {
@@ skipping 33 matching lines @@
   CACHE_UMA(COUNTS_10000, "MaxOpenEntries2", 0,
             static_cast<int>(stats_.GetCounter(Stats::MAX_ENTRIES)));
   stats_.SetCounter(Stats::MAX_ENTRIES, 0);
 
   CACHE_UMA(COUNTS_10000, "TotalFatalErrors", 0,
             static_cast<int>(stats_.GetCounter(Stats::FATAL_ERROR)));
   CACHE_UMA(COUNTS_10000, "TotalDoomCache", 0,
             static_cast<int>(stats_.GetCounter(Stats::DOOM_CACHE)));
   CACHE_UMA(COUNTS_10000, "TotalDoomRecentEntries", 0,
             static_cast<int>(stats_.GetCounter(Stats::DOOM_RECENT)));
-  CACHE_UMA(COUNTS_10000, "TotalEvictionsGaJs", 0,
-            static_cast<int>(stats_.GetCounter(Stats::GAJS_EVICTED)));
   stats_.SetCounter(Stats::FATAL_ERROR, 0);
   stats_.SetCounter(Stats::DOOM_CACHE, 0);
   stats_.SetCounter(Stats::DOOM_RECENT, 0);
-  stats_.SetCounter(Stats::GAJS_EVICTED, 0);
 
   int64 total_hours = stats_.GetCounter(Stats::TIMER) / 120;
   if (!data_->header.create_time || !data_->header.lru.filled) {
     int cause = data_->header.create_time ? 0 : 1;
     if (!data_->header.lru.filled)
       cause |= 2;
     CACHE_UMA(CACHE_ERROR, "ShortReport", 0, cause);
     CACHE_UMA(HOURS, "TotalTimeNotFull", 0, static_cast<int>(total_hours));
     return;
   }
@@ skipping 201 matching lines @@
     if (total_memory > kMaxBuffersSize || total_memory <= 0)
       total_memory = kMaxBuffersSize;
 
     done = true;
   }
 
   return static_cast<int>(total_memory);
 }
 
 }  // namespace disk_cache