| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include "net/disk_cache/blockfile/backend_impl_v3.h" | |
| 6 | |
| 7 #include <limits> | |
| 8 | |
| 9 #include "base/bind.h" | |
| 10 #include "base/bind_helpers.h" | |
| 11 #include "base/files/file_path.h" | |
| 12 #include "base/files/file_util.h" | |
| 13 #include "base/hash.h" | |
| 14 #include "base/message_loop/message_loop.h" | |
| 15 #include "base/metrics/field_trial.h" | |
| 16 #include "base/rand_util.h" | |
| 17 #include "base/strings/string_number_conversions.h" | |
| 18 #include "base/strings/string_util.h" | |
| 19 #include "base/strings/stringprintf.h" | |
| 20 #include "base/sys_info.h" | |
| 21 #include "base/threading/thread_restrictions.h" | |
| 22 #include "base/time/time.h" | |
| 23 #include "base/timer/timer.h" | |
| 24 #include "net/base/net_errors.h" | |
| 25 #include "net/disk_cache/blockfile/disk_format_v3.h" | |
| 26 #include "net/disk_cache/blockfile/entry_impl_v3.h" | |
| 27 #include "net/disk_cache/blockfile/errors.h" | |
| 28 #include "net/disk_cache/blockfile/experiments.h" | |
| 29 #include "net/disk_cache/blockfile/file.h" | |
| 30 #include "net/disk_cache/blockfile/histogram_macros_v3.h" | |
| 31 #include "net/disk_cache/blockfile/index_table_v3.h" | |
| 32 #include "net/disk_cache/blockfile/storage_block-inl.h" | |
| 33 #include "net/disk_cache/cache_util.h" | |
| 34 | |
| 35 // Provide a BackendImpl object to macros from histogram_macros.h. | |
| 36 #define CACHE_UMA_BACKEND_IMPL_OBJ this | |
| 37 | |
| 38 using base::Time; | |
| 39 using base::TimeDelta; | |
| 40 using base::TimeTicks; | |
| 41 | |
| 42 namespace { | |
| 43 | |
| 44 #if defined(V3_NOT_JUST_YET_READY) | |
| 45 const int kDefaultCacheSize = 80 * 1024 * 1024; | |
| 46 | |
| 47 // Avoid trimming the cache for the first 5 minutes (10 timer ticks). | |
| 48 const int kTrimDelay = 10; | |
| 49 #endif // defined(V3_NOT_JUST_YET_READY). | |
| 50 | |
| 51 } // namespace | |
| 52 | |
| 53 // ------------------------------------------------------------------------ | |
| 54 | |
| 55 namespace disk_cache { | |
| 56 | |
| 57 BackendImplV3::BackendImplV3( | |
| 58 const base::FilePath& path, | |
| 59 const scoped_refptr<base::SingleThreadTaskRunner>& cache_thread, | |
| 60 net::NetLog* net_log) | |
| 61 : index_(NULL), | |
| 62 path_(path), | |
| 63 block_files_(), | |
| 64 max_size_(0), | |
| 65 up_ticks_(0), | |
| 66 cache_type_(net::DISK_CACHE), | |
| 67 uma_report_(0), | |
| 68 user_flags_(0), | |
| 69 init_(false), | |
| 70 restarted_(false), | |
| 71 read_only_(false), | |
| 72 disabled_(false), | |
| 73 lru_eviction_(true), | |
| 74 first_timer_(true), | |
| 75 user_load_(false), | |
| 76 net_log_(net_log), | |
| 77 ptr_factory_(this) { | |
| 78 } | |
| 79 | |
| 80 BackendImplV3::~BackendImplV3() { | |
| 81 CleanupCache(); | |
| 82 } | |
| 83 | |
| 84 int BackendImplV3::Init(const CompletionCallback& callback) { | |
| 85 DCHECK(!init_); | |
| 86 if (init_) | |
| 87 return net::ERR_FAILED; | |
| 88 | |
| 89 return net::ERR_IO_PENDING; | |
| 90 } | |
| 91 | |
| 92 // ------------------------------------------------------------------------ | |
| 93 | |
| 94 bool BackendImplV3::SetMaxSize(int max_bytes) { | |
| 95 static_assert(sizeof(max_bytes) == sizeof(max_size_), | |
| 96 "unsupported int model"); | |
| 97 if (max_bytes < 0) | |
| 98 return false; | |
| 99 | |
| 100 // Zero size means use the default. | |
| 101 if (!max_bytes) | |
| 102 return true; | |
| 103 | |
| 104 // Avoid a DCHECK later on. | |
| 105 if (max_bytes >= std::numeric_limits<int32_t>::max() - | |
| 106 std::numeric_limits<int32_t>::max() / 10) { | |
| 107 max_bytes = std::numeric_limits<int32_t>::max() - | |
| 108 std::numeric_limits<int32_t>::max() / 10 - 1; | |
| 109 } | |
| 110 | |
| 111 user_flags_ |= MAX_SIZE; | |
| 112 max_size_ = max_bytes; | |
| 113 return true; | |
| 114 } | |
| 115 | |
| 116 void BackendImplV3::SetType(net::CacheType type) { | |
| 117 DCHECK_NE(net::MEMORY_CACHE, type); | |
| 118 cache_type_ = type; | |
| 119 } | |
| 120 | |
| 121 bool BackendImplV3::CreateBlock(FileType block_type, int block_count, | |
| 122 Addr* block_address) { | |
| 123 return block_files_.CreateBlock(block_type, block_count, block_address); | |
| 124 } | |
| 125 | |
| 126 #if defined(V3_NOT_JUST_YET_READY) | |
| 127 void BackendImplV3::UpdateRank(EntryImplV3* entry, bool modified) { | |
| 128 if (read_only_ || (!modified && cache_type() == net::SHADER_CACHE)) | |
| 129 return; | |
| 130 eviction_.UpdateRank(entry, modified); | |
| 131 } | |
| 132 | |
| 133 void BackendImplV3::InternalDoomEntry(EntryImplV3* entry) { | |
| 134 uint32_t hash = entry->GetHash(); | |
| 135 std::string key = entry->GetKey(); | |
| 136 Addr entry_addr = entry->entry()->address(); | |
| 137 bool error; | |
| 138 EntryImpl* parent_entry = MatchEntry(key, hash, true, entry_addr, &error); | |
| 139 CacheAddr child(entry->GetNextAddress()); | |
| 140 | |
| 141 Trace("Doom entry 0x%p", entry); | |
| 142 | |
| 143 if (!entry->doomed()) { | |
| 144 // We may have doomed this entry from within MatchEntry. | |
| 145 eviction_.OnDoomEntry(entry); | |
| 146 entry->InternalDoom(); | |
| 147 if (!new_eviction_) { | |
| 148 DecreaseNumEntries(); | |
| 149 } | |
| 150 stats_.OnEvent(Stats::DOOM_ENTRY); | |
| 151 } | |
| 152 | |
| 153 if (parent_entry) { | |
| 154 parent_entry->SetNextAddress(Addr(child)); | |
| 155 parent_entry->Release(); | |
| 156 } else if (!error) { | |
| 157 data_->table[hash & mask_] = child; | |
| 158 } | |
| 159 | |
| 160 FlushIndex(); | |
| 161 } | |
| 162 | |
| 163 void BackendImplV3::OnEntryDestroyBegin(Addr address) { | |
| 164 EntriesMap::iterator it = open_entries_.find(address.value()); | |
| 165 if (it != open_entries_.end()) | |
| 166 open_entries_.erase(it); | |
| 167 } | |
| 168 | |
| 169 void BackendImplV3::OnEntryDestroyEnd() { | |
| 170 DecreaseNumRefs(); | |
| 171 if (data_->header.num_bytes > max_size_ && !read_only_ && | |
| 172 (up_ticks_ > kTrimDelay || user_flags_ & kNoRandom)) | |
| 173 eviction_.TrimCache(false); | |
| 174 } | |
| 175 | |
| 176 EntryImplV3* BackendImplV3::GetOpenEntry(Addr address) const { | |
| 177 DCHECK(rankings->HasData()); | |
| 178 EntriesMap::const_iterator it = | |
| 179 open_entries_.find(rankings->Data()->contents); | |
| 180 if (it != open_entries_.end()) { | |
| 181 // We have this entry in memory. | |
| 182 return it->second; | |
| 183 } | |
| 184 | |
| 185 return NULL; | |
| 186 } | |
| 187 | |
| 188 int BackendImplV3::MaxFileSize() const { | |
| 189 return max_size_ / 8; | |
| 190 } | |
| 191 | |
| 192 void BackendImplV3::ModifyStorageSize(int32_t old_size, int32_t new_size) { | |
| 193 if (disabled_ || old_size == new_size) | |
| 194 return; | |
| 195 if (old_size > new_size) | |
| 196 SubstractStorageSize(old_size - new_size); | |
| 197 else | |
| 198 AddStorageSize(new_size - old_size); | |
| 199 | |
| 200 // Update the usage statistics. | |
| 201 stats_.ModifyStorageStats(old_size, new_size); | |
| 202 } | |
| 203 | |
| 204 void BackendImplV3::TooMuchStorageRequested(int32_t size) { | |
| 205 stats_.ModifyStorageStats(0, size); | |
| 206 } | |
| 207 | |
| 208 bool BackendImplV3::IsAllocAllowed(int current_size, int new_size) { | |
| 209 DCHECK_GT(new_size, current_size); | |
| 210 if (user_flags_ & NO_BUFFERING) | |
| 211 return false; | |
| 212 | |
| 213 int to_add = new_size - current_size; | |
| 214 if (buffer_bytes_ + to_add > MaxBuffersSize()) | |
| 215 return false; | |
| 216 | |
| 217 buffer_bytes_ += to_add; | |
| 218 CACHE_UMA(COUNTS_50000, "BufferBytes", buffer_bytes_ / 1024); | |
| 219 return true; | |
| 220 } | |
| 221 #endif // defined(V3_NOT_JUST_YET_READY). | |
| 222 | |
| 223 void BackendImplV3::BufferDeleted(int size) { | |
| 224 DCHECK_GE(size, 0); | |
| 225 buffer_bytes_ -= size; | |
| 226 DCHECK_GE(buffer_bytes_, 0); | |
| 227 } | |
| 228 | |
| 229 bool BackendImplV3::IsLoaded() const { | |
| 230 if (user_flags_ & NO_LOAD_PROTECTION) | |
| 231 return false; | |
| 232 | |
| 233 return user_load_; | |
| 234 } | |
| 235 | |
| 236 std::string BackendImplV3::HistogramName(const char* name) const { | |
| 237 static const char* const names[] = { | |
| 238 "Http", "", "Media", "AppCache", "Shader" }; | |
| 239 DCHECK_NE(cache_type_, net::MEMORY_CACHE); | |
| 240 return base::StringPrintf("DiskCache3.%s_%s", name, names[cache_type_]); | |
| 241 } | |
| 242 | |
| 243 base::WeakPtr<BackendImplV3> BackendImplV3::GetWeakPtr() { | |
| 244 return ptr_factory_.GetWeakPtr(); | |
| 245 } | |
| 246 | |
| 247 #if defined(V3_NOT_JUST_YET_READY) | |
| 248 // We want to remove biases from some histograms so we only send data once per | |
| 249 // week. | |
| 250 bool BackendImplV3::ShouldReportAgain() { | |
| 251 if (uma_report_) | |
| 252 return uma_report_ == 2; | |
| 253 | |
| 254 uma_report_++; | |
| 255 int64_t last_report = stats_.GetCounter(Stats::LAST_REPORT); | |
| 256 Time last_time = Time::FromInternalValue(last_report); | |
| 257 if (!last_report || (Time::Now() - last_time).InDays() >= 7) { | |
| 258 stats_.SetCounter(Stats::LAST_REPORT, Time::Now().ToInternalValue()); | |
| 259 uma_report_++; | |
| 260 return true; | |
| 261 } | |
| 262 return false; | |
| 263 } | |
| 264 | |
| 265 void BackendImplV3::FirstEviction() { | |
| 266 IndexHeaderV3* header = index_.header(); | |
| 267 header->flags |= CACHE_EVICTED; | |
| 268 DCHECK(header->create_time); | |
| 269 if (!GetEntryCount()) | |
| 270 return; // This is just for unit tests. | |
| 271 | |
| 272 Time create_time = Time::FromInternalValue(header->create_time); | |
| 273 CACHE_UMA(AGE, "FillupAge", create_time); | |
| 274 | |
| 275 int64_t use_time = stats_.GetCounter(Stats::TIMER); | |
| 276 CACHE_UMA(HOURS, "FillupTime", static_cast<int>(use_time / 120)); | |
| 277 CACHE_UMA(PERCENTAGE, "FirstHitRatio", stats_.GetHitRatio()); | |
| 278 | |
| 279 if (!use_time) | |
| 280 use_time = 1; | |
| 281 CACHE_UMA(COUNTS_10000, "FirstEntryAccessRate", | |
| 282 static_cast<int>(header->num_entries / use_time)); | |
| 283 CACHE_UMA(COUNTS, "FirstByteIORate", | |
| 284 static_cast<int>((header->num_bytes / 1024) / use_time)); | |
| 285 | |
| 286 int avg_size = header->num_bytes / GetEntryCount(); | |
| 287 CACHE_UMA(COUNTS, "FirstEntrySize", avg_size); | |
| 288 | |
| 289 int large_entries_bytes = stats_.GetLargeEntriesSize(); | |
| 290 int large_ratio = large_entries_bytes * 100 / header->num_bytes; | |
| 291 CACHE_UMA(PERCENTAGE, "FirstLargeEntriesRatio", large_ratio); | |
| 292 | |
| 293 if (!lru_eviction_) { | |
| 294 CACHE_UMA(PERCENTAGE, "FirstResurrectRatio", stats_.GetResurrectRatio()); | |
| 295 CACHE_UMA(PERCENTAGE, "FirstNoUseRatio", | |
| 296 header->num_no_use_entries * 100 / header->num_entries); | |
| 297 CACHE_UMA(PERCENTAGE, "FirstLowUseRatio", | |
| 298 header->num_low_use_entries * 100 / header->num_entries); | |
| 299 CACHE_UMA(PERCENTAGE, "FirstHighUseRatio", | |
| 300 header->num_high_use_entries * 100 / header->num_entries); | |
| 301 } | |
| 302 | |
| 303 stats_.ResetRatios(); | |
| 304 } | |
| 305 | |
| 306 void BackendImplV3::OnEvent(Stats::Counters an_event) { | |
| 307 stats_.OnEvent(an_event); | |
| 308 } | |
| 309 | |
| 310 void BackendImplV3::OnRead(int32_t bytes) { | |
| 311 DCHECK_GE(bytes, 0); | |
| 312 byte_count_ += bytes; | |
| 313 if (byte_count_ < 0) | |
| 314 byte_count_ = std::numeric_limits<int32_t>::max(); | |
| 315 } | |
| 316 | |
| 317 void BackendImplV3::OnWrite(int32_t bytes) { | |
| 318 // We use the same implementation as OnRead... just log the number of bytes. | |
| 319 OnRead(bytes); | |
| 320 } | |
| 321 | |
| 322 void BackendImplV3::OnTimerTick() { | |
| 323 stats_.OnEvent(Stats::TIMER); | |
| 324 int64_t time = stats_.GetCounter(Stats::TIMER); | |
| 325 int64_t current = stats_.GetCounter(Stats::OPEN_ENTRIES); | |
| 326 | |
| 327 // OPEN_ENTRIES is a sampled average of the number of open entries, avoiding | |
| 328 // the bias towards 0. | |
| 329 if (num_refs_ && (current != num_refs_)) { | |
| 330 int64_t diff = (num_refs_ - current) / 50; | |
| 331 if (!diff) | |
| 332 diff = num_refs_ > current ? 1 : -1; | |
| 333 current = current + diff; | |
| 334 stats_.SetCounter(Stats::OPEN_ENTRIES, current); | |
| 335 stats_.SetCounter(Stats::MAX_ENTRIES, max_refs_); | |
| 336 } | |
| 337 | |
| 338 CACHE_UMA(COUNTS, "NumberOfReferences", num_refs_); | |
| 339 | |
| 340 CACHE_UMA(COUNTS_10000, "EntryAccessRate", entry_count_); | |
| 341 CACHE_UMA(COUNTS, "ByteIORate", byte_count_ / 1024); | |
| 342 | |
| 343 // These values cover about 99.5% of the population (Oct 2011). | |
| 344 user_load_ = (entry_count_ > 300 || byte_count_ > 7 * 1024 * 1024); | |
| 345 entry_count_ = 0; | |
| 346 byte_count_ = 0; | |
| 347 up_ticks_++; | |
| 348 | |
| 349 if (!data_) | |
| 350 first_timer_ = false; | |
| 351 if (first_timer_) { | |
| 352 first_timer_ = false; | |
| 353 if (ShouldReportAgain()) | |
| 354 ReportStats(); | |
| 355 } | |
| 356 | |
| 357 // Save stats to disk at 5 min intervals. | |
| 358 if (time % 10 == 0) | |
| 359 StoreStats(); | |
| 360 } | |
| 361 | |
| 362 void BackendImplV3::SetUnitTestMode() { | |
| 363 user_flags_ |= UNIT_TEST_MODE; | |
| 364 } | |
| 365 | |
| 366 void BackendImplV3::SetUpgradeMode() { | |
| 367 user_flags_ |= UPGRADE_MODE; | |
| 368 read_only_ = true; | |
| 369 } | |
| 370 | |
| 371 void BackendImplV3::SetNewEviction() { | |
| 372 user_flags_ |= EVICTION_V2; | |
| 373 lru_eviction_ = false; | |
| 374 } | |
| 375 | |
| 376 void BackendImplV3::SetFlags(uint32_t flags) { | |
| 377 user_flags_ |= flags; | |
| 378 } | |
| 379 | |
| 380 int BackendImplV3::FlushQueueForTest(const CompletionCallback& callback) { | |
| 381 background_queue_.FlushQueue(callback); | |
| 382 return net::ERR_IO_PENDING; | |
| 383 } | |
| 384 | |
| 385 void BackendImplV3::TrimForTest(bool empty) { | |
| 386 eviction_.SetTestMode(); | |
| 387 eviction_.TrimCache(empty); | |
| 388 } | |
| 389 | |
| 390 void BackendImplV3::TrimDeletedListForTest(bool empty) { | |
| 391 eviction_.SetTestMode(); | |
| 392 eviction_.TrimDeletedList(empty); | |
| 393 } | |
| 394 | |
| 395 int BackendImplV3::SelfCheck() { | |
| 396 if (!init_) { | |
| 397 LOG(ERROR) << "Init failed"; | |
| 398 return ERR_INIT_FAILED; | |
| 399 } | |
| 400 | |
| 401 int num_entries = rankings_.SelfCheck(); | |
| 402 if (num_entries < 0) { | |
| 403 LOG(ERROR) << "Invalid rankings list, error " << num_entries; | |
| 404 #if !defined(NET_BUILD_STRESS_CACHE) | |
| 405 return num_entries; | |
| 406 #endif | |
| 407 } | |
| 408 | |
| 409 if (num_entries != data_->header.num_entries) { | |
| 410 LOG(ERROR) << "Number of entries mismatch"; | |
| 411 #if !defined(NET_BUILD_STRESS_CACHE) | |
| 412 return ERR_NUM_ENTRIES_MISMATCH; | |
| 413 #endif | |
| 414 } | |
| 415 | |
| 416 return CheckAllEntries(); | |
| 417 } | |
| 418 | |
| 419 // ------------------------------------------------------------------------ | |
| 420 | |
| 421 net::CacheType BackendImplV3::GetCacheType() const { | |
| 422 return cache_type_; | |
| 423 } | |
| 424 | |
| 425 int32_t BackendImplV3::GetEntryCount() const { | |
| 426 if (disabled_) | |
| 427 return 0; | |
| 428 DCHECK(init_); | |
| 429 return index_.header()->num_entries; | |
| 430 } | |
| 431 | |
| 432 int BackendImplV3::OpenEntry(const std::string& key, Entry** entry, | |
| 433 const CompletionCallback& callback) { | |
| 434 if (disabled_) | |
| 435 return NULL; | |
| 436 | |
| 437 TimeTicks start = TimeTicks::Now(); | |
| 438 uint32_t hash = base::Hash(key); | |
| 439 Trace("Open hash 0x%x", hash); | |
| 440 | |
| 441 bool error; | |
| 442 EntryImpl* cache_entry = MatchEntry(key, hash, false, Addr(), &error); | |
| 443 if (cache_entry && ENTRY_NORMAL != cache_entry->entry()->Data()->state) { | |
| 444 // The entry was already evicted. | |
| 445 cache_entry->Release(); | |
| 446 cache_entry = NULL; | |
| 447 } | |
| 448 | |
| 449 int current_size = data_->header.num_bytes / (1024 * 1024); | |
| 450 int64_t total_hours = stats_.GetCounter(Stats::TIMER) / 120; | |
| 451 int64_t no_use_hours = stats_.GetCounter(Stats::LAST_REPORT_TIMER) / 120; | |
| 452 int64_t use_hours = total_hours - no_use_hours; | |
| 453 | |
| 454 if (!cache_entry) { | |
| 455 CACHE_UMA(AGE_MS, "OpenTime.Miss", 0, start); | |
| 456 CACHE_UMA(COUNTS_10000, "AllOpenBySize.Miss", 0, current_size); | |
| 457 CACHE_UMA(HOURS, "AllOpenByTotalHours.Miss", 0, total_hours); | |
| 458 CACHE_UMA(HOURS, "AllOpenByUseHours.Miss", 0, use_hours); | |
| 459 stats_.OnEvent(Stats::OPEN_MISS); | |
| 460 return NULL; | |
| 461 } | |
| 462 | |
| 463 eviction_.OnOpenEntry(cache_entry); | |
| 464 entry_count_++; | |
| 465 | |
| 466 Trace("Open hash 0x%x end: 0x%x", hash, | |
| 467 cache_entry->entry()->address().value()); | |
| 468 CACHE_UMA(AGE_MS, "OpenTime", 0, start); | |
| 469 CACHE_UMA(COUNTS_10000, "AllOpenBySize.Hit", 0, current_size); | |
| 470 CACHE_UMA(HOURS, "AllOpenByTotalHours.Hit", 0, total_hours); | |
| 471 CACHE_UMA(HOURS, "AllOpenByUseHours.Hit", 0, use_hours); | |
| 472 stats_.OnEvent(Stats::OPEN_HIT); | |
| 473 SIMPLE_STATS_COUNTER("disk_cache.hit"); | |
| 474 return cache_entry; | |
| 475 } | |
| 476 | |
| 477 int BackendImplV3::CreateEntry(const std::string& key, Entry** entry, | |
| 478 const CompletionCallback& callback) { | |
| 479 if (disabled_ || key.empty()) | |
| 480 return NULL; | |
| 481 | |
| 482 TimeTicks start = TimeTicks::Now(); | |
| 483 Trace("Create hash 0x%x", hash); | |
| 484 | |
| 485 scoped_refptr<EntryImpl> parent; | |
| 486 Addr entry_address(data_->table[hash & mask_]); | |
| 487 if (entry_address.is_initialized()) { | |
| 488 // We have an entry already. It could be the one we are looking for, or just | |
| 489 // a hash conflict. | |
| 490 bool error; | |
| 491 EntryImpl* old_entry = MatchEntry(key, hash, false, Addr(), &error); | |
| 492 if (old_entry) | |
| 493 return ResurrectEntry(old_entry); | |
| 494 | |
| 495 EntryImpl* parent_entry = MatchEntry(key, hash, true, Addr(), &error); | |
| 496 DCHECK(!error); | |
| 497 if (parent_entry) { | |
| 498 parent.swap(&parent_entry); | |
| 499 } else if (data_->table[hash & mask_]) { | |
| 500 // We should have corrected the problem. | |
| 501 NOTREACHED(); | |
| 502 return NULL; | |
| 503 } | |
| 504 } | |
| 505 | |
| 506 // The general flow is to allocate disk space and initialize the entry data, | |
| 507 // followed by saving that to disk, then linking the entry though the index | |
| 508 // and finally through the lists. If there is a crash in this process, we may | |
| 509 // end up with: | |
| 510 // a. Used, unreferenced empty blocks on disk (basically just garbage). | |
| 511 // b. Used, unreferenced but meaningful data on disk (more garbage). | |
| 512 // c. A fully formed entry, reachable only through the index. | |
| 513 // d. A fully formed entry, also reachable through the lists, but still dirty. | |
| 514 // | |
| 515 // Anything after (b) can be automatically cleaned up. We may consider saving | |
| 516 // the current operation (as we do while manipulating the lists) so that we | |
| 517 // can detect and cleanup (a) and (b). | |
| 518 | |
| 519 int num_blocks = EntryImpl::NumBlocksForEntry(key.size()); | |
| 520 if (!block_files_.CreateBlock(BLOCK_256, num_blocks, &entry_address)) { | |
| 521 LOG(ERROR) << "Create entry failed " << key.c_str(); | |
| 522 stats_.OnEvent(Stats::CREATE_ERROR); | |
| 523 return NULL; | |
| 524 } | |
| 525 | |
| 526 Addr node_address(0); | |
| 527 if (!block_files_.CreateBlock(RANKINGS, 1, &node_address)) { | |
| 528 block_files_.DeleteBlock(entry_address, false); | |
| 529 LOG(ERROR) << "Create entry failed " << key.c_str(); | |
| 530 stats_.OnEvent(Stats::CREATE_ERROR); | |
| 531 return NULL; | |
| 532 } | |
| 533 | |
| 534 scoped_refptr<EntryImpl> cache_entry( | |
| 535 new EntryImpl(this, entry_address, false)); | |
| 536 IncreaseNumRefs(); | |
| 537 | |
| 538 if (!cache_entry->CreateEntry(node_address, key, hash)) { | |
| 539 block_files_.DeleteBlock(entry_address, false); | |
| 540 block_files_.DeleteBlock(node_address, false); | |
| 541 LOG(ERROR) << "Create entry failed " << key.c_str(); | |
| 542 stats_.OnEvent(Stats::CREATE_ERROR); | |
| 543 return NULL; | |
| 544 } | |
| 545 | |
| 546 cache_entry->BeginLogging(net_log_, true); | |
| 547 | |
| 548 // We are not failing the operation; let's add this to the map. | |
| 549 open_entries_[entry_address.value()] = cache_entry.get(); | |
| 550 | |
| 551 // Save the entry. | |
| 552 cache_entry->entry()->Store(); | |
| 553 cache_entry->rankings()->Store(); | |
| 554 IncreaseNumEntries(); | |
| 555 entry_count_++; | |
| 556 | |
| 557 // Link this entry through the index. | |
| 558 if (parent.get()) { | |
| 559 parent->SetNextAddress(entry_address); | |
| 560 } else { | |
| 561 data_->table[hash & mask_] = entry_address.value(); | |
| 562 } | |
| 563 | |
| 564 // Link this entry through the lists. | |
| 565 eviction_.OnCreateEntry(cache_entry.get()); | |
| 566 | |
| 567 CACHE_UMA(AGE_MS, "CreateTime", 0, start); | |
| 568 stats_.OnEvent(Stats::CREATE_HIT); | |
| 569 SIMPLE_STATS_COUNTER("disk_cache.miss"); | |
| 570 Trace("create entry hit "); | |
| 571 FlushIndex(); | |
| 572 cache_entry->AddRef(); | |
| 573 return cache_entry.get(); | |
| 574 } | |
| 575 | |
| 576 int BackendImplV3::DoomEntry(const std::string& key, | |
| 577 const CompletionCallback& callback) { | |
| 578 if (disabled_) | |
| 579 return net::ERR_FAILED; | |
| 580 | |
| 581 EntryImpl* entry = OpenEntryImpl(key); | |
| 582 if (!entry) | |
| 583 return net::ERR_FAILED; | |
| 584 | |
| 585 entry->DoomImpl(); | |
| 586 entry->Release(); | |
| 587 return net::OK; | |
| 588 } | |
| 589 | |
| 590 int BackendImplV3::DoomAllEntries(const CompletionCallback& callback) { | |
| 591 // This is not really an error, but it is an interesting condition. | |
| 592 ReportError(ERR_CACHE_DOOMED); | |
| 593 stats_.OnEvent(Stats::DOOM_CACHE); | |
| 594 if (!num_refs_) { | |
| 595 RestartCache(false); | |
| 596 return disabled_ ? net::ERR_FAILED : net::OK; | |
| 597 } else { | |
| 598 if (disabled_) | |
| 599 return net::ERR_FAILED; | |
| 600 | |
| 601 eviction_.TrimCache(true); | |
| 602 return net::OK; | |
| 603 } | |
| 604 } | |
| 605 | |
| 606 int BackendImplV3::DoomEntriesBetween(base::Time initial_time, | |
| 607 base::Time end_time, | |
| 608 const CompletionCallback& callback) { | |
| 609 DCHECK_NE(net::APP_CACHE, cache_type_); | |
| 610 if (end_time.is_null()) | |
| 611 return SyncDoomEntriesSince(initial_time); | |
| 612 | |
| 613 DCHECK(end_time >= initial_time); | |
| 614 | |
| 615 if (disabled_) | |
| 616 return net::ERR_FAILED; | |
| 617 | |
| 618 EntryImpl* node; | |
| 619 void* iter = NULL; | |
| 620 EntryImpl* next = OpenNextEntryImpl(&iter); | |
| 621 if (!next) | |
| 622 return net::OK; | |
| 623 | |
| 624 while (next) { | |
| 625 node = next; | |
| 626 next = OpenNextEntryImpl(&iter); | |
| 627 | |
| 628 if (node->GetLastUsed() >= initial_time && | |
| 629 node->GetLastUsed() < end_time) { | |
| 630 node->DoomImpl(); | |
| 631 } else if (node->GetLastUsed() < initial_time) { | |
| 632 if (next) | |
| 633 next->Release(); | |
| 634 next = NULL; | |
| 635 SyncEndEnumeration(iter); | |
| 636 } | |
| 637 | |
| 638 node->Release(); | |
| 639 } | |
| 640 | |
| 641 return net::OK; | |
| 642 } | |
| 643 | |
| 644 int BackendImplV3::DoomEntriesSince(base::Time initial_time, | |
| 645 const CompletionCallback& callback) { | |
| 646 DCHECK_NE(net::APP_CACHE, cache_type_); | |
| 647 if (disabled_) | |
| 648 return net::ERR_FAILED; | |
| 649 | |
| 650 stats_.OnEvent(Stats::DOOM_RECENT); | |
| 651 for (;;) { | |
| 652 void* iter = NULL; | |
| 653 EntryImpl* entry = OpenNextEntryImpl(&iter); | |
| 654 if (!entry) | |
| 655 return net::OK; | |
| 656 | |
| 657 if (initial_time > entry->GetLastUsed()) { | |
| 658 entry->Release(); | |
| 659 SyncEndEnumeration(iter); | |
| 660 return net::OK; | |
| 661 } | |
| 662 | |
| 663 entry->DoomImpl(); | |
| 664 entry->Release(); | |
| 665 SyncEndEnumeration(iter); // Dooming the entry invalidates the iterator. | |
| 666 } | |
| 667 } | |
| 668 | |
| 669 class BackendImplV3::IteratorImpl : public Backend::Iterator { | |
| 670 public: | |
| 671 explicit IteratorImpl(base::WeakPtr<InFlightBackendIO> background_queue) | |
| 672 : background_queue_(background_queue), data_(NULL) { | |
| 673 } | |
| 674 | |
| 675 int OpenNextEntry(Entry** next_entry, | |
| 676 const net::CompletionCallback& callback) override { | |
| 677 if (!background_queue_) | |
| 678 return net::ERR_FAILED; | |
| 679 background_queue_->OpenNextEntry(&data_, next_entry, callback); | |
| 680 return net::ERR_IO_PENDING; | |
| 681 } | |
| 682 | |
| 683 private: | |
| 684 const base::WeakPtr<InFlightBackendIO> background_queue_; | |
| 685 void* data_; | |
| 686 }; | |
| 687 | |
| 688 std::unique_ptr<Backend::Iterator> BackendImplV3::CreateIterator() { | |
| 689 return std::unique_ptr<Backend::Iterator>( | |
| 690 new IteratorImpl(GetBackgroundQueue())); | |
| 691 } | |
| 692 | |
| 693 void BackendImplV3::GetStats(StatsItems* stats) { | |
| 694 if (disabled_) | |
| 695 return; | |
| 696 | |
| 697 std::pair<std::string, std::string> item; | |
| 698 | |
| 699 item.first = "Entries"; | |
| 700 item.second = base::IntToString(data_->header.num_entries); | |
| 701 stats->push_back(item); | |
| 702 | |
| 703 item.first = "Pending IO"; | |
| 704 item.second = base::IntToString(num_pending_io_); | |
| 705 stats->push_back(item); | |
| 706 | |
| 707 item.first = "Max size"; | |
| 708 item.second = base::IntToString(max_size_); | |
| 709 stats->push_back(item); | |
| 710 | |
| 711 item.first = "Current size"; | |
| 712 item.second = base::IntToString(data_->header.num_bytes); | |
| 713 stats->push_back(item); | |
| 714 | |
| 715 item.first = "Cache type"; | |
| 716 item.second = "Blockfile Cache"; | |
| 717 stats->push_back(item); | |
| 718 | |
| 719 stats_.GetItems(stats); | |
| 720 } | |
| 721 | |
| 722 void BackendImplV3::OnExternalCacheHit(const std::string& key) { | |
| 723 if (disabled_) | |
| 724 return; | |
| 725 | |
| 726 uint32_t hash = base::Hash(key); | |
| 727 bool error; | |
| 728 EntryImpl* cache_entry = MatchEntry(key, hash, false, Addr(), &error); | |
| 729 if (cache_entry) { | |
| 730 if (ENTRY_NORMAL == cache_entry->entry()->Data()->state) { | |
| 731 UpdateRank(cache_entry, cache_type() == net::SHADER_CACHE); | |
| 732 } | |
| 733 cache_entry->Release(); | |
| 734 } | |
| 735 } | |
| 736 | |
| 737 // ------------------------------------------------------------------------ | |
| 738 | |
| 739 // The maximum cache size will be either set explicitly by the caller, or | |
| 740 // calculated by this code. | |
| 741 void BackendImplV3::AdjustMaxCacheSize(int table_len) { | |
| 742 if (max_size_) | |
| 743 return; | |
| 744 | |
| 745 // If table_len is provided, the index file exists. | |
| 746 DCHECK(!table_len || data_->header.magic); | |
| 747 | |
| 748 // The user is not setting the size, let's figure it out. | |
| 749 int64_t available = base::SysInfo::AmountOfFreeDiskSpace(path_); | |
| 750 if (available < 0) { | |
| 751 max_size_ = kDefaultCacheSize; | |
| 752 return; | |
| 753 } | |
| 754 | |
| 755 if (table_len) | |
| 756 available += data_->header.num_bytes; | |
| 757 | |
| 758 max_size_ = PreferedCacheSize(available); | |
| 759 | |
| 760 // Let's not use more than the default size while we tune-up the performance | |
| 761 // of bigger caches. TODO(rvargas): remove this limit. | |
| 762 if (max_size_ > kDefaultCacheSize * 4) | |
| 763 max_size_ = kDefaultCacheSize * 4; | |
| 764 | |
| 765 if (!table_len) | |
| 766 return; | |
| 767 | |
| 768 // If we already have a table, adjust the size to it. | |
| 769 int current_max_size = MaxStorageSizeForTable(table_len); | |
| 770 if (max_size_ > current_max_size) | |
| 771 max_size_= current_max_size; | |
| 772 } | |
| 773 | |
| 774 bool BackendImplV3::InitStats() { | |
| 775 Addr address(data_->header.stats); | |
| 776 int size = stats_.StorageSize(); | |
| 777 | |
| 778 if (!address.is_initialized()) { | |
| 779 FileType file_type = Addr::RequiredFileType(size); | |
| 780 DCHECK_NE(file_type, EXTERNAL); | |
| 781 int num_blocks = Addr::RequiredBlocks(size, file_type); | |
| 782 | |
| 783 if (!CreateBlock(file_type, num_blocks, &address)) | |
| 784 return false; | |
| 785 return stats_.Init(NULL, 0, address); | |
| 786 } | |
| 787 | |
| 788 if (!address.is_block_file()) { | |
| 789 NOTREACHED(); | |
| 790 return false; | |
| 791 } | |
| 792 | |
| 793 // Load the required data. | |
| 794 size = address.num_blocks() * address.BlockSize(); | |
| 795 MappedFile* file = File(address); | |
| 796 if (!file) | |
| 797 return false; | |
| 798 | |
| 799 std::unique_ptr<char[]> data(new char[size]); | |
| 800 size_t offset = address.start_block() * address.BlockSize() + | |
| 801 kBlockHeaderSize; | |
| 802 if (!file->Read(data.get(), size, offset)) | |
| 803 return false; | |
| 804 | |
| 805 if (!stats_.Init(data.get(), size, address)) | |
| 806 return false; | |
| 807 if (cache_type_ == net::DISK_CACHE && ShouldReportAgain()) | |
| 808 stats_.InitSizeHistogram(); | |
| 809 return true; | |
| 810 } | |
| 811 | |
| 812 void BackendImplV3::StoreStats() { | |
| 813 int size = stats_.StorageSize(); | |
| 814 std::unique_ptr<char[]> data(new char[size]); | |
| 815 Addr address; | |
| 816 size = stats_.SerializeStats(data.get(), size, &address); | |
| 817 DCHECK(size); | |
| 818 if (!address.is_initialized()) | |
| 819 return; | |
| 820 | |
| 821 MappedFile* file = File(address); | |
| 822 if (!file) | |
| 823 return; | |
| 824 | |
| 825 size_t offset = address.start_block() * address.BlockSize() + | |
| 826 kBlockHeaderSize; | |
| 827 file->Write(data.get(), size, offset); // ignore result. | |
| 828 } | |
| 829 | |
| 830 void BackendImplV3::RestartCache(bool failure) { | |
| 831 int64_t errors = stats_.GetCounter(Stats::FATAL_ERROR); | |
| 832 int64_t full_dooms = stats_.GetCounter(Stats::DOOM_CACHE); | |
| 833 int64_t partial_dooms = stats_.GetCounter(Stats::DOOM_RECENT); | |
| 834 int64_t last_report = stats_.GetCounter(Stats::LAST_REPORT); | |
| 835 | |
| 836 PrepareForRestart(); | |
| 837 if (failure) { | |
| 838 DCHECK(!num_refs_); | |
| 839 DCHECK(!open_entries_.size()); | |
| 840 DelayedCacheCleanup(path_); | |
| 841 } else { | |
| 842 DeleteCache(path_, false); | |
| 843 } | |
| 844 | |
| 845 // Don't call Init() if directed by the unit test: we are simulating a failure | |
| 846 // trying to re-enable the cache. | |
| 847 if (unit_test_) | |
| 848 init_ = true; // Let the destructor do proper cleanup. | |
| 849 else if (SyncInit() == net::OK) { | |
| 850 stats_.SetCounter(Stats::FATAL_ERROR, errors); | |
| 851 stats_.SetCounter(Stats::DOOM_CACHE, full_dooms); | |
| 852 stats_.SetCounter(Stats::DOOM_RECENT, partial_dooms); | |
| 853 stats_.SetCounter(Stats::LAST_REPORT, last_report); | |
| 854 } | |
| 855 } | |
| 856 | |
| 857 void BackendImplV3::PrepareForRestart() { | |
| 858 if (!(user_flags_ & EVICTION_V2)) | |
| 859 lru_eviction_ = true; | |
| 860 | |
| 861 disabled_ = true; | |
| 862 data_->header.crash = 0; | |
| 863 index_->Flush(); | |
| 864 index_ = NULL; | |
| 865 data_ = NULL; | |
| 866 block_files_.CloseFiles(); | |
| 867 rankings_.Reset(); | |
| 868 init_ = false; | |
| 869 restarted_ = true; | |
| 870 } | |
| 871 | |
| 872 void BackendImplV3::CleanupCache() { | |
| 873 Trace("Backend Cleanup"); | |
| 874 eviction_.Stop(); | |
| 875 timer_.reset(); | |
| 876 | |
| 877 if (init_) { | |
| 878 StoreStats(); | |
| 879 if (data_) | |
| 880 data_->header.crash = 0; | |
| 881 | |
| 882 if (user_flags_ & kNoRandom) { | |
| 883 // This is a net_unittest, verify that we are not 'leaking' entries. | |
| 884 File::WaitForPendingIO(&num_pending_io_); | |
| 885 DCHECK(!num_refs_); | |
| 886 } else { | |
| 887 File::DropPendingIO(); | |
| 888 } | |
| 889 } | |
| 890 block_files_.CloseFiles(); | |
| 891 FlushIndex(); | |
| 892 index_ = NULL; | |
| 893 ptr_factory_.InvalidateWeakPtrs(); | |
| 894 done_.Signal(); | |
| 895 } | |
| 896 | |
| 897 int BackendImplV3::NewEntry(Addr address, EntryImplV3** entry) { | |
| 898 EntriesMap::iterator it = open_entries_.find(address.value()); | |
| 899 if (it != open_entries_.end()) { | |
| 900 // Easy job. This entry is already in memory. | |
| 901 EntryImpl* this_entry = it->second; | |
| 902 this_entry->AddRef(); | |
| 903 *entry = this_entry; | |
| 904 return 0; | |
| 905 } | |
| 906 | |
| 907 STRESS_DCHECK(block_files_.IsValid(address)); | |
| 908 | |
| 909 if (!address.SanityCheckForEntry()) { | |
| 910 LOG(WARNING) << "Wrong entry address."; | |
| 911 STRESS_NOTREACHED(); | |
| 912 return ERR_INVALID_ADDRESS; | |
| 913 } | |
| 914 | |
| 915 scoped_refptr<EntryImpl> cache_entry( | |
| 916 new EntryImpl(this, address, read_only_)); | |
| 917 IncreaseNumRefs(); | |
| 918 *entry = NULL; | |
| 919 | |
| 920 TimeTicks start = TimeTicks::Now(); | |
| 921 if (!cache_entry->entry()->Load()) | |
| 922 return ERR_READ_FAILURE; | |
| 923 | |
| 924 if (IsLoaded()) { | |
| 925 CACHE_UMA(AGE_MS, "LoadTime", 0, start); | |
| 926 } | |
| 927 | |
| 928 if (!cache_entry->SanityCheck()) { | |
| 929 LOG(WARNING) << "Messed up entry found."; | |
| 930 STRESS_NOTREACHED(); | |
| 931 return ERR_INVALID_ENTRY; | |
| 932 } | |
| 933 | |
| 934 STRESS_DCHECK(block_files_.IsValid( | |
| 935 Addr(cache_entry->entry()->Data()->rankings_node))); | |
| 936 | |
| 937 if (!cache_entry->LoadNodeAddress()) | |
| 938 return ERR_READ_FAILURE; | |
| 939 | |
| 940 if (!rankings_.SanityCheck(cache_entry->rankings(), false)) { | |
| 941 STRESS_NOTREACHED(); | |
| 942 cache_entry->SetDirtyFlag(0); | |
| 943 // Don't remove this from the list (it is not linked properly). Instead, | |
| 944 // break the link back to the entry because it is going away, and leave the | |
| 945 // rankings node to be deleted if we find it through a list. | |
| 946 rankings_.SetContents(cache_entry->rankings(), 0); | |
| 947 } else if (!rankings_.DataSanityCheck(cache_entry->rankings(), false)) { | |
| 948 STRESS_NOTREACHED(); | |
| 949 cache_entry->SetDirtyFlag(0); | |
| 950 rankings_.SetContents(cache_entry->rankings(), address.value()); | |
| 951 } | |
| 952 | |
| 953 if (!cache_entry->DataSanityCheck()) { | |
| 954 LOG(WARNING) << "Messed up entry found."; | |
| 955 cache_entry->SetDirtyFlag(0); | |
| 956 cache_entry->FixForDelete(); | |
| 957 } | |
| 958 | |
| 959 // Prevent overwriting the dirty flag on the destructor. | |
| 960 cache_entry->SetDirtyFlag(GetCurrentEntryId()); | |
| 961 | |
| 962 if (cache_entry->dirty()) { | |
| 963 Trace("Dirty entry 0x%p 0x%x", reinterpret_cast<void*>(cache_entry.get()), | |
| 964 address.value()); | |
| 965 } | |
| 966 | |
| 967 open_entries_[address.value()] = cache_entry.get(); | |
| 968 | |
| 969 cache_entry->BeginLogging(net_log_, false); | |
| 970 cache_entry.swap(entry); | |
| 971 return 0; | |
| 972 } | |
| 973 | |
| 974 void BackendImplV3::AddStorageSize(int32_t bytes) { | |
| 975 data_->header.num_bytes += bytes; | |
| 976 DCHECK_GE(data_->header.num_bytes, 0); | |
| 977 } | |
| 978 | |
| 979 void BackendImplV3::SubstractStorageSize(int32_t bytes) { | |
| 980 data_->header.num_bytes -= bytes; | |
| 981 DCHECK_GE(data_->header.num_bytes, 0); | |
| 982 } | |
| 983 | |
| 984 void BackendImplV3::IncreaseNumRefs() { | |
| 985 num_refs_++; | |
| 986 if (max_refs_ < num_refs_) | |
| 987 max_refs_ = num_refs_; | |
| 988 } | |
| 989 | |
| 990 void BackendImplV3::DecreaseNumRefs() { | |
| 991 DCHECK(num_refs_); | |
| 992 num_refs_--; | |
| 993 | |
| 994 if (!num_refs_ && disabled_) | |
| 995 base::MessageLoop::current()->PostTask( | |
| 996 FROM_HERE, base::Bind(&BackendImpl::RestartCache, GetWeakPtr(), true)); | |
| 997 } | |
| 998 | |
| 999 void BackendImplV3::IncreaseNumEntries() { | |
| 1000 index_.header()->num_entries++; | |
| 1001 DCHECK_GT(index_.header()->num_entries, 0); | |
| 1002 } | |
| 1003 | |
| 1004 void BackendImplV3::DecreaseNumEntries() { | |
| 1005 index_.header()->num_entries--; | |
| 1006 if (index_.header()->num_entries < 0) { | |
| 1007 NOTREACHED(); | |
| 1008 index_.header()->num_entries = 0; | |
| 1009 } | |
| 1010 } | |
| 1011 | |
| 1012 int BackendImplV3::SyncInit() { | |
| 1013 #if defined(NET_BUILD_STRESS_CACHE) | |
| 1014 // Start evictions right away. | |
| 1015 up_ticks_ = kTrimDelay * 2; | |
| 1016 #endif | |
| 1017 DCHECK(!init_); | |
| 1018 if (init_) | |
| 1019 return net::ERR_FAILED; | |
| 1020 | |
| 1021 bool create_files = false; | |
| 1022 if (!InitBackingStore(&create_files)) { | |
| 1023 ReportError(ERR_STORAGE_ERROR); | |
| 1024 return net::ERR_FAILED; | |
| 1025 } | |
| 1026 | |
| 1027 num_refs_ = num_pending_io_ = max_refs_ = 0; | |
| 1028 entry_count_ = byte_count_ = 0; | |
| 1029 | |
| 1030 if (!restarted_) { | |
| 1031 buffer_bytes_ = 0; | |
| 1032 trace_object_ = TraceObject::GetTraceObject(); | |
| 1033 // Create a recurrent timer of 30 secs. | |
| 1034 int timer_delay = unit_test_ ? 1000 : 30000; | |
| 1035 timer_.reset(new base::RepeatingTimer()); | |
| 1036 timer_->Start(FROM_HERE, TimeDelta::FromMilliseconds(timer_delay), this, | |
| 1037 &BackendImplV3::OnStatsTimer); | |
| 1038 } | |
| 1039 | |
| 1040 init_ = true; | |
| 1041 Trace("Init"); | |
| 1042 | |
| 1043 if (data_->header.experiment != NO_EXPERIMENT && | |
| 1044 cache_type_ != net::DISK_CACHE) { | |
| 1045 // No experiment for other caches. | |
| 1046 return net::ERR_FAILED; | |
| 1047 } | |
| 1048 | |
| 1049 if (!(user_flags_ & kNoRandom)) { | |
| 1050 // The unit test controls directly what to test. | |
| 1051 new_eviction_ = (cache_type_ == net::DISK_CACHE); | |
| 1052 } | |
| 1053 | |
| 1054 if (!CheckIndex()) { | |
| 1055 ReportError(ERR_INIT_FAILED); | |
| 1056 return net::ERR_FAILED; | |
| 1057 } | |
| 1058 | |
| 1059 if (!restarted_ && (create_files || !data_->header.num_entries)) | |
| 1060 ReportError(ERR_CACHE_CREATED); | |
| 1061 | |
| 1062 if (!(user_flags_ & kNoRandom) && cache_type_ == net::DISK_CACHE && | |
| 1063 !InitExperiment(&data_->header, create_files)) { | |
| 1064 return net::ERR_FAILED; | |
| 1065 } | |
| 1066 | |
| 1067 // We don't care if the value overflows. The only thing we care about is that | |
| 1068 // the id cannot be zero, because that value is used as "not dirty". | |
| 1069 // Increasing the value once per second gives us many years before we start | |
| 1070 // having collisions. | |
| 1071 data_->header.this_id++; | |
| 1072 if (!data_->header.this_id) | |
| 1073 data_->header.this_id++; | |
| 1074 | |
| 1075 bool previous_crash = (data_->header.crash != 0); | |
| 1076 data_->header.crash = 1; | |
| 1077 | |
| 1078 if (!block_files_.Init(create_files)) | |
| 1079 return net::ERR_FAILED; | |
| 1080 | |
| 1081 // We want to minimize the changes to cache for an AppCache. | |
| 1082 if (cache_type() == net::APP_CACHE) { | |
| 1083 DCHECK(!new_eviction_); | |
| 1084 read_only_ = true; | |
| 1085 } else if (cache_type() == net::SHADER_CACHE) { | |
| 1086 DCHECK(!new_eviction_); | |
| 1087 } | |
| 1088 | |
| 1089 eviction_.Init(this); | |
| 1090 | |
| 1091 // stats_ and rankings_ may end up calling back to us so we better be enabled. | |
| 1092 disabled_ = false; | |
| 1093 if (!InitStats()) | |
| 1094 return net::ERR_FAILED; | |
| 1095 | |
| 1096 disabled_ = !rankings_.Init(this, new_eviction_); | |
| 1097 | |
| 1098 #if defined(STRESS_CACHE_EXTENDED_VALIDATION) | |
| 1099 trace_object_->EnableTracing(false); | |
| 1100 int sc = SelfCheck(); | |
| 1101 if (sc < 0 && sc != ERR_NUM_ENTRIES_MISMATCH) | |
| 1102 NOTREACHED(); | |
| 1103 trace_object_->EnableTracing(true); | |
| 1104 #endif | |
| 1105 | |
| 1106 if (previous_crash) { | |
| 1107 ReportError(ERR_PREVIOUS_CRASH); | |
| 1108 } else if (!restarted_) { | |
| 1109 ReportError(ERR_NO_ERROR); | |
| 1110 } | |
| 1111 | |
| 1112 FlushIndex(); | |
| 1113 | |
| 1114 return disabled_ ? net::ERR_FAILED : net::OK; | |
| 1115 } | |
| 1116 | |
| 1117 EntryImpl* BackendImplV3::ResurrectEntry(EntryImpl* deleted_entry) { | |
| 1118 if (ENTRY_NORMAL == deleted_entry->entry()->Data()->state) { | |
| 1119 deleted_entry->Release(); | |
| 1120 stats_.OnEvent(Stats::CREATE_MISS); | |
| 1121 Trace("create entry miss "); | |
| 1122 return NULL; | |
| 1123 } | |
| 1124 | |
| 1125 // We are attempting to create an entry and found out that the entry was | |
| 1126 // previously deleted. | |
| 1127 | |
| 1128 eviction_.OnCreateEntry(deleted_entry); | |
| 1129 entry_count_++; | |
| 1130 | |
| 1131 stats_.OnEvent(Stats::RESURRECT_HIT); | |
| 1132 Trace("Resurrect entry hit "); | |
| 1133 return deleted_entry; | |
| 1134 } | |
| 1135 | |
| 1136 EntryImpl* BackendImplV3::CreateEntryImpl(const std::string& key) { | |
| 1137 if (disabled_ || key.empty()) | |
| 1138 return NULL; | |
| 1139 | |
| 1140 TimeTicks start = TimeTicks::Now(); | |
| 1141 Trace("Create hash 0x%x", hash); | |
| 1142 | |
| 1143 scoped_refptr<EntryImpl> parent; | |
| 1144 Addr entry_address(data_->table[hash & mask_]); | |
| 1145 if (entry_address.is_initialized()) { | |
| 1146 // We have an entry already. It could be the one we are looking for, or just | |
| 1147 // a hash conflict. | |
| 1148 bool error; | |
| 1149 EntryImpl* old_entry = MatchEntry(key, hash, false, Addr(), &error); | |
| 1150 if (old_entry) | |
| 1151 return ResurrectEntry(old_entry); | |
| 1152 | |
| 1153 EntryImpl* parent_entry = MatchEntry(key, hash, true, Addr(), &error); | |
| 1154 DCHECK(!error); | |
| 1155 if (parent_entry) { | |
| 1156 parent.swap(&parent_entry); | |
| 1157 } else if (data_->table[hash & mask_]) { | |
| 1158 // We should have corrected the problem. | |
| 1159 NOTREACHED(); | |
| 1160 return NULL; | |
| 1161 } | |
| 1162 } | |
| 1163 | |
| 1164 // The general flow is to allocate disk space and initialize the entry data, | |
| 1165 // followed by saving that to disk, then linking the entry though the index | |
| 1166 // and finally through the lists. If there is a crash in this process, we may | |
| 1167 // end up with: | |
| 1168 // a. Used, unreferenced empty blocks on disk (basically just garbage). | |
| 1169 // b. Used, unreferenced but meaningful data on disk (more garbage). | |
| 1170 // c. A fully formed entry, reachable only through the index. | |
| 1171 // d. A fully formed entry, also reachable through the lists, but still dirty. | |
| 1172 // | |
| 1173 // Anything after (b) can be automatically cleaned up. We may consider saving | |
| 1174 // the current operation (as we do while manipulating the lists) so that we | |
| 1175 // can detect and cleanup (a) and (b). | |
| 1176 | |
| 1177 int num_blocks = EntryImpl::NumBlocksForEntry(key.size()); | |
| 1178 if (!block_files_.CreateBlock(BLOCK_256, num_blocks, &entry_address)) { | |
| 1179 LOG(ERROR) << "Create entry failed " << key.c_str(); | |
| 1180 stats_.OnEvent(Stats::CREATE_ERROR); | |
| 1181 return NULL; | |
| 1182 } | |
| 1183 | |
| 1184 Addr node_address(0); | |
| 1185 if (!block_files_.CreateBlock(RANKINGS, 1, &node_address)) { | |
| 1186 block_files_.DeleteBlock(entry_address, false); | |
| 1187 LOG(ERROR) << "Create entry failed " << key.c_str(); | |
| 1188 stats_.OnEvent(Stats::CREATE_ERROR); | |
| 1189 return NULL; | |
| 1190 } | |
| 1191 | |
| 1192 scoped_refptr<EntryImpl> cache_entry( | |
| 1193 new EntryImpl(this, entry_address, false)); | |
| 1194 IncreaseNumRefs(); | |
| 1195 | |
| 1196 if (!cache_entry->CreateEntry(node_address, key, hash)) { | |
| 1197 block_files_.DeleteBlock(entry_address, false); | |
| 1198 block_files_.DeleteBlock(node_address, false); | |
| 1199 LOG(ERROR) << "Create entry failed " << key.c_str(); | |
| 1200 stats_.OnEvent(Stats::CREATE_ERROR); | |
| 1201 return NULL; | |
| 1202 } | |
| 1203 | |
| 1204 cache_entry->BeginLogging(net_log_, true); | |
| 1205 | |
| 1206 // We are not failing the operation; let's add this to the map. | |
| 1207 open_entries_[entry_address.value()] = cache_entry; | |
| 1208 | |
| 1209 // Save the entry. | |
| 1210 cache_entry->entry()->Store(); | |
| 1211 cache_entry->rankings()->Store(); | |
| 1212 IncreaseNumEntries(); | |
| 1213 entry_count_++; | |
| 1214 | |
| 1215 // Link this entry through the index. | |
| 1216 if (parent.get()) { | |
| 1217 parent->SetNextAddress(entry_address); | |
| 1218 } else { | |
| 1219 data_->table[hash & mask_] = entry_address.value(); | |
| 1220 } | |
| 1221 | |
| 1222 // Link this entry through the lists. | |
| 1223 eviction_.OnCreateEntry(cache_entry); | |
| 1224 | |
| 1225 CACHE_UMA(AGE_MS, "CreateTime", 0, start); | |
| 1226 stats_.OnEvent(Stats::CREATE_HIT); | |
| 1227 SIMPLE_STATS_COUNTER("disk_cache.miss"); | |
| 1228 Trace("create entry hit "); | |
| 1229 FlushIndex(); | |
| 1230 cache_entry->AddRef(); | |
| 1231 return cache_entry.get(); | |
| 1232 } | |
| 1233 | |
| 1234 void BackendImplV3::LogStats() { | |
| 1235 StatsItems stats; | |
| 1236 GetStats(&stats); | |
| 1237 | |
| 1238 for (size_t index = 0; index < stats.size(); index++) | |
| 1239 VLOG(1) << stats[index].first << ": " << stats[index].second; | |
| 1240 } | |
| 1241 | |
| 1242 void BackendImplV3::ReportStats() { | |
| 1243 IndexHeaderV3* header = index_.header(); | |
| 1244 CACHE_UMA(COUNTS, "Entries", header->num_entries); | |
| 1245 | |
| 1246 int current_size = header->num_bytes / (1024 * 1024); | |
| 1247 int max_size = max_size_ / (1024 * 1024); | |
| 1248 | |
| 1249 CACHE_UMA(COUNTS_10000, "Size", current_size); | |
| 1250 CACHE_UMA(COUNTS_10000, "MaxSize", max_size); | |
| 1251 if (!max_size) | |
| 1252 max_size++; | |
| 1253 CACHE_UMA(PERCENTAGE, "UsedSpace", current_size * 100 / max_size); | |
| 1254 | |
| 1255 CACHE_UMA(COUNTS_10000, "AverageOpenEntries", | |
| 1256 static_cast<int>(stats_.GetCounter(Stats::OPEN_ENTRIES))); | |
| 1257 CACHE_UMA(COUNTS_10000, "MaxOpenEntries", | |
| 1258 static_cast<int>(stats_.GetCounter(Stats::MAX_ENTRIES))); | |
| 1259 stats_.SetCounter(Stats::MAX_ENTRIES, 0); | |
| 1260 | |
| 1261 CACHE_UMA(COUNTS_10000, "TotalFatalErrors", | |
| 1262 static_cast<int>(stats_.GetCounter(Stats::FATAL_ERROR))); | |
| 1263 CACHE_UMA(COUNTS_10000, "TotalDoomCache", | |
| 1264 static_cast<int>(stats_.GetCounter(Stats::DOOM_CACHE))); | |
| 1265 CACHE_UMA(COUNTS_10000, "TotalDoomRecentEntries", | |
| 1266 static_cast<int>(stats_.GetCounter(Stats::DOOM_RECENT))); | |
| 1267 stats_.SetCounter(Stats::FATAL_ERROR, 0); | |
| 1268 stats_.SetCounter(Stats::DOOM_CACHE, 0); | |
| 1269 stats_.SetCounter(Stats::DOOM_RECENT, 0); | |
| 1270 | |
| 1271 int64_t total_hours = stats_.GetCounter(Stats::TIMER) / 120; | |
| 1272 if (!(header->flags & CACHE_EVICTED)) { | |
| 1273 CACHE_UMA(HOURS, "TotalTimeNotFull", static_cast<int>(total_hours)); | |
| 1274 return; | |
| 1275 } | |
| 1276 | |
| 1277 // This is an up to date client that will report FirstEviction() data. After | |
| 1278 // that event, start reporting this: | |
| 1279 | |
| 1280 CACHE_UMA(HOURS, "TotalTime", static_cast<int>(total_hours)); | |
| 1281 | |
| 1282 int64_t use_hours = stats_.GetCounter(Stats::LAST_REPORT_TIMER) / 120; | |
| 1283 stats_.SetCounter(Stats::LAST_REPORT_TIMER, stats_.GetCounter(Stats::TIMER)); | |
| 1284 | |
| 1285 // We may see users with no use_hours at this point if this is the first time | |
| 1286 // we are running this code. | |
| 1287 if (use_hours) | |
| 1288 use_hours = total_hours - use_hours; | |
| 1289 | |
| 1290 if (!use_hours || !GetEntryCount() || !header->num_bytes) | |
| 1291 return; | |
| 1292 | |
| 1293 CACHE_UMA(HOURS, "UseTime", static_cast<int>(use_hours)); | |
| 1294 | |
| 1295 int64_t trim_rate = stats_.GetCounter(Stats::TRIM_ENTRY) / use_hours; | |
| 1296 CACHE_UMA(COUNTS, "TrimRate", static_cast<int>(trim_rate)); | |
| 1297 | |
| 1298 int avg_size = header->num_bytes / GetEntryCount(); | |
| 1299 CACHE_UMA(COUNTS, "EntrySize", avg_size); | |
| 1300 CACHE_UMA(COUNTS, "EntriesFull", header->num_entries); | |
| 1301 | |
| 1302 int large_entries_bytes = stats_.GetLargeEntriesSize(); | |
| 1303 int large_ratio = large_entries_bytes * 100 / header->num_bytes; | |
| 1304 CACHE_UMA(PERCENTAGE, "LargeEntriesRatio", large_ratio); | |
| 1305 | |
| 1306 if (!lru_eviction_) { | |
| 1307 CACHE_UMA(PERCENTAGE, "ResurrectRatio", stats_.GetResurrectRatio()); | |
| 1308 CACHE_UMA(PERCENTAGE, "NoUseRatio", | |
| 1309 header->num_no_use_entries * 100 / header->num_entries); | |
| 1310 CACHE_UMA(PERCENTAGE, "LowUseRatio", | |
| 1311 header->num_low_use_entries * 100 / header->num_entries); | |
| 1312 CACHE_UMA(PERCENTAGE, "HighUseRatio", | |
| 1313 header->num_high_use_entries * 100 / header->num_entries); | |
| 1314 CACHE_UMA(PERCENTAGE, "DeletedRatio", | |
| 1315 header->num_evicted_entries * 100 / header->num_entries); | |
| 1316 } | |
| 1317 | |
| 1318 stats_.ResetRatios(); | |
| 1319 stats_.SetCounter(Stats::TRIM_ENTRY, 0); | |
| 1320 | |
| 1321 if (cache_type_ == net::DISK_CACHE) | |
| 1322 block_files_.ReportStats(); | |
| 1323 } | |
| 1324 | |
| 1325 void BackendImplV3::ReportError(int error) { | |
| 1326 STRESS_DCHECK(!error || error == ERR_PREVIOUS_CRASH || | |
| 1327 error == ERR_CACHE_CREATED); | |
| 1328 | |
| 1329 // We transmit positive numbers, instead of direct error codes. | |
| 1330 DCHECK_LE(error, 0); | |
| 1331 CACHE_UMA(CACHE_ERROR, "Error", error * -1); | |
| 1332 } | |
| 1333 | |
| 1334 bool BackendImplV3::CheckIndex() { | |
| 1335 DCHECK(data_); | |
| 1336 | |
| 1337 size_t current_size = index_->GetLength(); | |
| 1338 if (current_size < sizeof(Index)) { | |
| 1339 LOG(ERROR) << "Corrupt Index file"; | |
| 1340 return false; | |
| 1341 } | |
| 1342 | |
| 1343 if (new_eviction_) { | |
| 1344 // We support versions 2.0 and 2.1, upgrading 2.0 to 2.1. | |
| 1345 if (kIndexMagic != data_->header.magic || | |
| 1346 kCurrentVersion >> 16 != data_->header.version >> 16) { | |
| 1347 LOG(ERROR) << "Invalid file version or magic"; | |
| 1348 return false; | |
| 1349 } | |
| 1350 if (kCurrentVersion == data_->header.version) { | |
| 1351 // We need file version 2.1 for the new eviction algorithm. | |
| 1352 UpgradeTo2_1(); | |
| 1353 } | |
| 1354 } else { | |
| 1355 if (kIndexMagic != data_->header.magic || | |
| 1356 kCurrentVersion != data_->header.version) { | |
| 1357 LOG(ERROR) << "Invalid file version or magic"; | |
| 1358 return false; | |
| 1359 } | |
| 1360 } | |
| 1361 | |
| 1362 if (!data_->header.table_len) { | |
| 1363 LOG(ERROR) << "Invalid table size"; | |
| 1364 return false; | |
| 1365 } | |
| 1366 | |
| 1367 if (current_size < GetIndexSize(data_->header.table_len) || | |
| 1368 data_->header.table_len & (kBaseTableLen - 1)) { | |
| 1369 LOG(ERROR) << "Corrupt Index file"; | |
| 1370 return false; | |
| 1371 } | |
| 1372 | |
| 1373 AdjustMaxCacheSize(data_->header.table_len); | |
| 1374 | |
| 1375 #if !defined(NET_BUILD_STRESS_CACHE) | |
| 1376 if (data_->header.num_bytes < 0 || | |
| 1377 (max_size_ < std::numeric_limits<int32_t>::max() - kDefaultCacheSize && | |
| 1378 data_->header.num_bytes > max_size_ + kDefaultCacheSize)) { | |
| 1379 LOG(ERROR) << "Invalid cache (current) size"; | |
| 1380 return false; | |
| 1381 } | |
| 1382 #endif | |
| 1383 | |
| 1384 if (data_->header.num_entries < 0) { | |
| 1385 LOG(ERROR) << "Invalid number of entries"; | |
| 1386 return false; | |
| 1387 } | |
| 1388 | |
| 1389 if (!mask_) | |
| 1390 mask_ = data_->header.table_len - 1; | |
| 1391 | |
| 1392 // Load the table into memory with a single read. | |
| 1393 std::unique_ptr<char[]> buf(new char[current_size]); | |
| 1394 return index_->Read(buf.get(), current_size, 0); | |
| 1395 } | |
| 1396 | |
| 1397 int BackendImplV3::CheckAllEntries() { | |
| 1398 int num_dirty = 0; | |
| 1399 int num_entries = 0; | |
| 1400 DCHECK(mask_ < std::numeric_limits<uint32_t>::max()); | |
| 1401 for (unsigned int i = 0; i <= mask_; i++) { | |
| 1402 Addr address(data_->table[i]); | |
| 1403 if (!address.is_initialized()) | |
| 1404 continue; | |
| 1405 for (;;) { | |
| 1406 EntryImpl* tmp; | |
| 1407 int ret = NewEntry(address, &tmp); | |
| 1408 if (ret) { | |
| 1409 STRESS_NOTREACHED(); | |
| 1410 return ret; | |
| 1411 } | |
| 1412 scoped_refptr<EntryImpl> cache_entry; | |
| 1413 cache_entry.swap(&tmp); | |
| 1414 | |
| 1415 if (cache_entry->dirty()) | |
| 1416 num_dirty++; | |
| 1417 else if (CheckEntry(cache_entry.get())) | |
| 1418 num_entries++; | |
| 1419 else | |
| 1420 return ERR_INVALID_ENTRY; | |
| 1421 | |
| 1422 DCHECK_EQ(i, cache_entry->entry()->Data()->hash & mask_); | |
| 1423 address.set_value(cache_entry->GetNextAddress()); | |
| 1424 if (!address.is_initialized()) | |
| 1425 break; | |
| 1426 } | |
| 1427 } | |
| 1428 | |
| 1429 Trace("CheckAllEntries End"); | |
| 1430 if (num_entries + num_dirty != data_->header.num_entries) { | |
| 1431 LOG(ERROR) << "Number of entries " << num_entries << " " << num_dirty << | |
| 1432 " " << data_->header.num_entries; | |
| 1433 DCHECK_LT(num_entries, data_->header.num_entries); | |
| 1434 return ERR_NUM_ENTRIES_MISMATCH; | |
| 1435 } | |
| 1436 | |
| 1437 return num_dirty; | |
| 1438 } | |
| 1439 | |
| 1440 bool BackendImplV3::CheckEntry(EntryImpl* cache_entry) { | |
| 1441 bool ok = block_files_.IsValid(cache_entry->entry()->address()); | |
| 1442 ok = ok && block_files_.IsValid(cache_entry->rankings()->address()); | |
| 1443 EntryStore* data = cache_entry->entry()->Data(); | |
| 1444 for (size_t i = 0; i < arraysize(data->data_addr); i++) { | |
| 1445 if (data->data_addr[i]) { | |
| 1446 Addr address(data->data_addr[i]); | |
| 1447 if (address.is_block_file()) | |
| 1448 ok = ok && block_files_.IsValid(address); | |
| 1449 } | |
| 1450 } | |
| 1451 | |
| 1452 return ok && cache_entry->rankings()->VerifyHash(); | |
| 1453 } | |
| 1454 | |
| 1455 int BackendImplV3::MaxBuffersSize() { | |
| 1456 static int64_t total_memory = base::SysInfo::AmountOfPhysicalMemory(); | |
| 1457 static bool done = false; | |
| 1458 | |
| 1459 if (!done) { | |
| 1460 const int kMaxBuffersSize = 30 * 1024 * 1024; | |
| 1461 | |
| 1462 // We want to use up to 2% of the computer's memory. | |
| 1463 total_memory = total_memory * 2 / 100; | |
| 1464 if (total_memory > kMaxBuffersSize || total_memory <= 0) | |
| 1465 total_memory = kMaxBuffersSize; | |
| 1466 | |
| 1467 done = true; | |
| 1468 } | |
| 1469 | |
| 1470 return static_cast<int>(total_memory); | |
| 1471 } | |
| 1472 | |
| 1473 #endif // defined(V3_NOT_JUST_YET_READY). | |
| 1474 | |
| 1475 bool BackendImplV3::IsAllocAllowed(int current_size, int new_size) { | |
| 1476 return false; | |
| 1477 } | |
| 1478 | |
| 1479 net::CacheType BackendImplV3::GetCacheType() const { | |
| 1480 return cache_type_; | |
| 1481 } | |
| 1482 | |
| 1483 int32_t BackendImplV3::GetEntryCount() const { | |
| 1484 return 0; | |
| 1485 } | |
| 1486 | |
| 1487 int BackendImplV3::OpenEntry(const std::string& key, Entry** entry, | |
| 1488 const CompletionCallback& callback) { | |
| 1489 return net::ERR_FAILED; | |
| 1490 } | |
| 1491 | |
| 1492 int BackendImplV3::CreateEntry(const std::string& key, Entry** entry, | |
| 1493 const CompletionCallback& callback) { | |
| 1494 return net::ERR_FAILED; | |
| 1495 } | |
| 1496 | |
| 1497 int BackendImplV3::DoomEntry(const std::string& key, | |
| 1498 const CompletionCallback& callback) { | |
| 1499 return net::ERR_FAILED; | |
| 1500 } | |
| 1501 | |
| 1502 int BackendImplV3::DoomAllEntries(const CompletionCallback& callback) { | |
| 1503 return net::ERR_FAILED; | |
| 1504 } | |
| 1505 | |
| 1506 int BackendImplV3::DoomEntriesBetween(base::Time initial_time, | |
| 1507 base::Time end_time, | |
| 1508 const CompletionCallback& callback) { | |
| 1509 return net::ERR_FAILED; | |
| 1510 } | |
| 1511 | |
| 1512 int BackendImplV3::DoomEntriesSince(base::Time initial_time, | |
| 1513 const CompletionCallback& callback) { | |
| 1514 return net::ERR_FAILED; | |
| 1515 } | |
| 1516 | |
| 1517 int BackendImplV3::CalculateSizeOfAllEntries( | |
| 1518 const CompletionCallback& callback) { | |
| 1519 return net::ERR_FAILED; | |
| 1520 } | |
| 1521 | |
| 1522 class BackendImplV3::NotImplementedIterator : public Backend::Iterator { | |
| 1523 public: | |
| 1524 int OpenNextEntry(disk_cache::Entry** next_entry, | |
| 1525 const net::CompletionCallback& callback) override { | |
| 1526 return net::ERR_NOT_IMPLEMENTED; | |
| 1527 } | |
| 1528 }; | |
| 1529 | |
| 1530 std::unique_ptr<Backend::Iterator> BackendImplV3::CreateIterator() { | |
| 1531 return std::unique_ptr<Iterator>(new NotImplementedIterator()); | |
| 1532 } | |
| 1533 | |
| 1534 void BackendImplV3::GetStats(StatsItems* stats) { | |
| 1535 NOTIMPLEMENTED(); | |
| 1536 } | |
| 1537 | |
| 1538 void BackendImplV3::OnExternalCacheHit(const std::string& key) { | |
| 1539 NOTIMPLEMENTED(); | |
| 1540 } | |
| 1541 | |
| 1542 void BackendImplV3::CleanupCache() { | |
| 1543 } | |
| 1544 | |
| 1545 } // namespace disk_cache | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1910023002:
Remove partial blockfile v3 disk_cache implementation. (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master