| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #include "net/http/infinite_cache.h" | |
| 6 | |
| 7 #include <algorithm> | |
| 8 | |
| 9 #include "base/compiler_specific.h" | |
| 10 #include "base/bind.h" | |
| 11 #include "base/bind_helpers.h" | |
| 12 #include "base/file_path.h" | |
| 13 #include "base/file_util.h" | |
| 14 #include "base/hash.h" | |
| 15 #include "base/hash_tables.h" | |
| 16 #include "base/location.h" | |
| 17 #include "base/memory/ref_counted.h" | |
| 18 #include "base/metrics/histogram.h" | |
| 19 #include "base/pickle.h" | |
| 20 #include "base/platform_file.h" | |
| 21 #include "base/rand_util.h" | |
| 22 #include "base/sha1.h" | |
| 23 #include "base/time.h" | |
| 24 #include "base/threading/sequenced_worker_pool.h" | |
| 25 #include "net/base/net_errors.h" | |
| 26 #include "net/http/http_cache_transaction.h" | |
| 27 #include "net/http/http_request_info.h" | |
| 28 #include "net/http/http_response_headers.h" | |
| 29 #include "net/http/http_response_info.h" | |
| 30 #include "net/http/http_util.h" | |
| 31 #include "third_party/zlib/zlib.h" | |
| 32 | |
| 33 using base::PlatformFile; | |
| 34 using base::Time; | |
| 35 using base::TimeDelta; | |
| 36 | |
| 37 namespace { | |
| 38 | |
| 39 // Flags to use with a particular resource. | |
| 40 enum Flags { | |
| 41 NO_CACHE = 1 << 0, | |
| 42 NO_STORE = 1 << 1, | |
| 43 EXPIRED = 1 << 2, | |
| 44 TRUNCATED = 1 << 3, | |
| 45 RESUMABLE = 1 << 4, | |
| 46 REVALIDATEABLE = 1 << 5, | |
| 47 DOOM_METHOD = 1 << 6, | |
| 48 CACHED = 1 << 7 | |
| 49 }; | |
| 50 | |
| 51 const int kKeySizeBytes = 20; | |
| 52 COMPILE_ASSERT(base::kSHA1Length == kKeySizeBytes, invalid_key_length); | |
| 53 struct Key { | |
| 54 char value[kKeySizeBytes]; | |
| 55 }; | |
| 56 | |
| 57 // The actual data that we store for every resource. | |
| 58 struct Details { | |
| 59 int32 expiration; | |
| 60 int32 last_access; | |
| 61 uint16 flags; | |
| 62 uint8 use_count; | |
| 63 uint8 update_count; | |
| 64 uint32 vary_hash; | |
| 65 int32 headers_size; | |
| 66 int32 response_size; | |
| 67 uint32 headers_hash; | |
| 68 uint32 response_hash; | |
| 69 }; | |
| 70 const size_t kRecordSize = sizeof(Key) + sizeof(Details); | |
| 71 | |
| 72 // Some constants related to the database file. | |
| 73 uint32 kMagicSignature = 0x1f00cace; | |
| 74 uint32 kCurrentVersion = 0x10001; | |
| 75 | |
| 76 // Basic limits for the experiment. | |
| 77 int kMaxNumEntries = 200 * 1000; | |
| 78 int kMaxTrackingSize = 40 * 1024 * 1024; | |
| 79 | |
| 80 // Settings that control how we generate histograms. | |
| 81 int kTimerMinutes = 5; | |
| 82 int kReportSizeStep = 100 * 1024 * 1024; | |
| 83 | |
| 84 // Buffer to read and write the file. | |
| 85 const size_t kBufferSize = 1024 * 1024; | |
| 86 const size_t kMaxRecordsToRead = kBufferSize / kRecordSize; | |
| 87 COMPILE_ASSERT(kRecordSize * kMaxRecordsToRead < kBufferSize, wrong_buffer); | |
| 88 | |
| 89 // Functor for operator <. | |
| 90 struct Key_less { | |
| 91 bool operator()(const Key& left, const Key& right) const { | |
| 92 // left < right. | |
| 93 return (memcmp(left.value, right.value, kKeySizeBytes) < 0); | |
| 94 } | |
| 95 }; | |
| 96 | |
| 97 // Functor for operator ==. | |
| 98 struct Key_eq { | |
| 99 bool operator()(const Key& left, const Key& right) const { | |
| 100 return (memcmp(left.value, right.value, kKeySizeBytes) == 0); | |
| 101 } | |
| 102 }; | |
| 103 | |
| 104 // Simple adaptor for the sha1 interface. | |
| 105 void CryptoHash(std::string source, Key* destination) { | |
| 106 base::SHA1HashBytes(reinterpret_cast<const unsigned char*>(source.data()), | |
| 107 source.size(), | |
| 108 reinterpret_cast<unsigned char*>(destination->value)); | |
| 109 } | |
| 110 | |
| 111 // Simple adaptor for base::ReadPlatformFile. | |
| 112 bool ReadPlatformFile(PlatformFile file, size_t offset, | |
| 113 void* buffer, size_t buffer_len) { | |
| 114 DCHECK_LE(offset, static_cast<size_t>(kuint32max)); | |
| 115 int bytes = base::ReadPlatformFile(file, static_cast<int64>(offset), | |
| 116 reinterpret_cast<char*>(buffer), | |
| 117 static_cast<int>(buffer_len)); | |
| 118 return (bytes == static_cast<int>(buffer_len)); | |
| 119 } | |
| 120 | |
| 121 // Simple adaptor for base::WritePlatformFile. | |
| 122 bool WritePlatformFile(PlatformFile file, size_t offset, | |
| 123 const void* buffer, size_t buffer_len) { | |
| 124 DCHECK_LE(offset, static_cast<size_t>(kuint32max)); | |
| 125 int bytes = base::WritePlatformFile(file, static_cast<int64>(offset), | |
| 126 reinterpret_cast<const char*>(buffer), | |
| 127 static_cast<int>(buffer_len)); | |
| 128 return (bytes == static_cast<int>(buffer_len)); | |
| 129 } | |
| 130 | |
| 131 // 1 second resolution, +- 68 years from the baseline. | |
| 132 int32 TimeToInt(Time time) { | |
| 133 int64 seconds = (time - Time::UnixEpoch()).InSeconds(); | |
| 134 if (seconds > kint32max) | |
| 135 seconds = kint32max; | |
| 136 if (seconds < kint32min) | |
| 137 seconds = kint32min; | |
| 138 return static_cast<int32>(seconds); | |
| 139 } | |
| 140 | |
| 141 Time IntToTime(int32 time) { | |
| 142 return Time::UnixEpoch() + TimeDelta::FromSeconds(time); | |
| 143 } | |
| 144 | |
| 145 int32 GetExpiration(const net::HttpResponseInfo* response) { | |
| 146 TimeDelta freshness = | |
| 147 response->headers->GetFreshnessLifetime(response->response_time); | |
| 148 | |
| 149 // Avoid overflow when adding to current time. | |
| 150 if (freshness.InDays() > 365 * 10) | |
| 151 freshness = TimeDelta::FromDays(365 * 10); | |
| 152 return TimeToInt(response->response_time + freshness); | |
| 153 } | |
| 154 | |
| 155 uint32 GetCacheability(const net::HttpResponseInfo* response) { | |
| 156 uint32 cacheability = 0; | |
| 157 const net::HttpResponseHeaders* headers = response->headers; | |
| 158 if (headers->HasHeaderValue("cache-control", "no-cache") || | |
| 159 headers->HasHeaderValue("pragma", "no-cache") || | |
| 160 headers->HasHeaderValue("vary", "*")) { | |
| 161 cacheability |= NO_CACHE; | |
| 162 } | |
| 163 | |
| 164 if (headers->HasHeaderValue("cache-control", "no-store")) | |
| 165 cacheability |= NO_STORE; | |
| 166 | |
| 167 TimeDelta max_age; | |
| 168 if (headers->GetMaxAgeValue(&max_age) && max_age.InSeconds() <= 0) | |
| 169 cacheability |= NO_CACHE; | |
| 170 | |
| 171 return cacheability; | |
| 172 } | |
| 173 | |
| 174 uint32 GetRevalidationFlags(const net::HttpResponseInfo* response) { | |
| 175 uint32 revalidation = 0; | |
| 176 std::string etag; | |
| 177 response->headers->EnumerateHeader(NULL, "etag", &etag); | |
| 178 | |
| 179 std::string last_modified; | |
| 180 response->headers->EnumerateHeader(NULL, "last-modified", &last_modified); | |
| 181 | |
| 182 if (!etag.empty() || !last_modified.empty()) | |
| 183 revalidation = REVALIDATEABLE; | |
| 184 | |
| 185 if (response->headers->HasStrongValidators()) | |
| 186 revalidation = RESUMABLE; | |
| 187 | |
| 188 return revalidation; | |
| 189 } | |
| 190 | |
| 191 | |
| 192 uint32 GetVaryHash(const net::HttpResponseInfo* response) { | |
| 193 if (!response->vary_data.is_valid()) | |
| 194 return 0; | |
| 195 | |
| 196 uint32 hash = adler32(0, Z_NULL, 0); | |
| 197 Pickle pickle; | |
| 198 response->vary_data.Persist(&pickle); | |
| 199 return adler32(hash, reinterpret_cast<const Bytef*>(pickle.data()), | |
| 200 pickle.size()); | |
| 201 } | |
| 202 | |
| 203 // Adaptor for PostTaskAndReply. | |
| 204 void OnComplete(const net::CompletionCallback& callback, int* result) { | |
| 205 callback.Run(*result); | |
| 206 } | |
| 207 | |
| 208 } // namespace | |
| 209 | |
| 210 namespace BASE_HASH_NAMESPACE { | |
| 211 #if defined(COMPILER_MSVC) | |
| 212 inline size_t hash_value(const Key& key) { | |
| 213 return base::Hash(key.value, kKeySizeBytes); | |
| 214 } | |
| 215 #elif defined(COMPILER_GCC) | |
| 216 template <> | |
| 217 struct hash<Key> { | |
| 218 size_t operator()(const Key& key) const { | |
| 219 return base::Hash(key.value, kKeySizeBytes); | |
| 220 } | |
| 221 }; | |
| 222 #endif | |
| 223 | |
| 224 } // BASE_HASH_NAMESPACE | |
| 225 | |
| 226 namespace net { | |
| 227 | |
| 228 struct InfiniteCacheTransaction::ResourceData { | |
| 229 ResourceData() { | |
| 230 memset(this, 0, sizeof(*this)); | |
| 231 } | |
| 232 | |
| 233 Key key; | |
| 234 Details details; | |
| 235 }; | |
| 236 | |
| 237 InfiniteCacheTransaction::InfiniteCacheTransaction(InfiniteCache* cache) | |
| 238 : cache_(cache->AsWeakPtr()), must_doom_entry_(false) { | |
| 239 } | |
| 240 | |
| 241 InfiniteCacheTransaction::~InfiniteCacheTransaction() { | |
| 242 Finish(); | |
| 243 } | |
| 244 | |
| 245 void InfiniteCacheTransaction::OnRequestStart(const HttpRequestInfo* request) { | |
| 246 if (!cache_) | |
| 247 return; | |
| 248 | |
| 249 std::string method = request->method; | |
| 250 if (method == "POST" || method == "DELETE" || method == "PUT") { | |
| 251 must_doom_entry_ = true; | |
| 252 } else if (method != "GET") { | |
| 253 cache_.reset(); | |
| 254 return; | |
| 255 } | |
| 256 | |
| 257 resource_data_.reset(new ResourceData); | |
| 258 CryptoHash(cache_->GenerateKey(request), &resource_data_->key); | |
| 259 } | |
| 260 | |
| 261 void InfiniteCacheTransaction::OnResponseReceived( | |
| 262 const HttpResponseInfo* response) { | |
| 263 if (!cache_) | |
| 264 return; | |
| 265 | |
| 266 Details& details = resource_data_->details; | |
| 267 | |
| 268 details.expiration = GetExpiration(response); | |
| 269 details.last_access = TimeToInt(response->request_time); | |
| 270 details.flags = GetCacheability(response); | |
| 271 details.vary_hash = GetVaryHash(response); | |
| 272 details.response_hash = adler32(0, Z_NULL, 0); // Init the hash. | |
| 273 | |
| 274 if (!details.flags && | |
| 275 TimeToInt(response->response_time) == details.expiration) { | |
| 276 details.flags = EXPIRED; | |
| 277 } | |
| 278 details.flags |= GetRevalidationFlags(response); | |
| 279 | |
| 280 if (must_doom_entry_) | |
| 281 details.flags |= DOOM_METHOD; | |
| 282 | |
| 283 Pickle pickle; | |
| 284 response->Persist(&pickle, true, false); // Skip transient headers. | |
| 285 details.headers_size = pickle.size(); | |
| 286 details.headers_hash = adler32(0, Z_NULL, 0); | |
| 287 details.headers_hash = adler32(details.headers_hash, | |
| 288 reinterpret_cast<const Bytef*>(pickle.data()), | |
| 289 pickle.size()); | |
| 290 } | |
| 291 | |
| 292 void InfiniteCacheTransaction::OnDataRead(const char* data, int data_len) { | |
| 293 if (!cache_) | |
| 294 return; | |
| 295 | |
| 296 if (!data_len) | |
| 297 return Finish(); | |
| 298 | |
| 299 resource_data_->details.response_size += data_len; | |
| 300 | |
| 301 resource_data_->details.response_hash = | |
| 302 adler32(resource_data_->details.response_hash, | |
| 303 reinterpret_cast<const Bytef*>(data), data_len); | |
| 304 } | |
| 305 | |
| 306 void InfiniteCacheTransaction::OnTruncatedResponse() { | |
| 307 if (!cache_) | |
| 308 return; | |
| 309 | |
| 310 resource_data_->details.flags |= TRUNCATED; | |
| 311 } | |
| 312 | |
| 313 void InfiniteCacheTransaction::OnServedFromCache() { | |
| 314 if (!cache_) | |
| 315 return; | |
| 316 | |
| 317 resource_data_->details.flags |= CACHED; | |
| 318 } | |
| 319 | |
| 320 void InfiniteCacheTransaction::Finish() { | |
| 321 if (!cache_ || !resource_data_.get()) | |
| 322 return; | |
| 323 | |
| 324 if (!resource_data_->details.headers_size) | |
| 325 return; | |
| 326 | |
| 327 cache_->ProcessResource(resource_data_.Pass()); | |
| 328 cache_.reset(); | |
| 329 } | |
| 330 | |
| 331 // ---------------------------------------------------------------------------- | |
| 332 | |
| 333 // This is the object that performs the bulk of the work. | |
| 334 // InfiniteCacheTransaction posts the transaction data to the InfiniteCache, and | |
| 335 // the InfiniteCache basically just forward requests to the Worker for actual | |
| 336 // processing. | |
| 337 // The Worker lives on a worker thread (basically a dedicated worker pool with | |
| 338 // only one thread), and flushes data to disk once every five minutes, when it | |
| 339 // is notified by the InfiniteCache. | |
| 340 // In general, there are no callbacks on completion of tasks, and the Worker can | |
| 341 // be as behind as it has to when processing requests. | |
| 342 class InfiniteCache::Worker : public base::RefCountedThreadSafe<Worker> { | |
| 343 public: | |
| 344 Worker() : init_(false), flushed_(false) {} | |
| 345 | |
| 346 // Construction and destruction helpers. | |
| 347 void Init(const FilePath& path); | |
| 348 void Cleanup(); | |
| 349 | |
| 350 // Deletes all tracked data. | |
| 351 void DeleteData(int* result); | |
| 352 | |
| 353 // Deletes requests between |initial_time| and |end_time|. | |
| 354 void DeleteDataBetween(base::Time initial_time, | |
| 355 base::Time end_time, | |
| 356 int* result); | |
| 357 | |
| 358 // Performs the actual processing of a new transaction. Takes ownership of | |
| 359 // the transaction |data|. | |
| 360 void Process(scoped_ptr<InfiniteCacheTransaction::ResourceData> data); | |
| 361 | |
| 362 // Test helpers. | |
| 363 void Query(int* result); | |
| 364 void Flush(int* result); | |
| 365 | |
| 366 // Timer notification. | |
| 367 void OnTimer(); | |
| 368 | |
| 369 private: | |
| 370 friend base::RefCountedThreadSafe<Worker>; | |
| 371 #if defined(COMPILER_MSVC) | |
| 372 typedef BASE_HASH_NAMESPACE::hash_map< | |
| 373 Key, Details, BASE_HASH_NAMESPACE::hash_compare<Key, Key_less> > KeyMap; | |
| 374 #elif defined(COMPILER_GCC) | |
| 375 typedef BASE_HASH_NAMESPACE::hash_map< | |
| 376 Key, Details, BASE_HASH_NAMESPACE::hash<Key>, Key_eq> KeyMap; | |
| 377 #endif | |
| 378 | |
| 379 // Header for the data file. The file starts with the header, followed by | |
| 380 // all the records, and a data hash at the end (just of the records, not the | |
| 381 // header). Note that the header has a dedicated hash. | |
| 382 struct Header { | |
| 383 uint32 magic; | |
| 384 uint32 version; | |
| 385 int32 num_entries; | |
| 386 int32 generation; | |
| 387 uint64 creation_time; | |
| 388 uint64 update_time; | |
| 389 int64 total_size; | |
| 390 int64 size_last_report; | |
| 391 int32 use_minutes; | |
| 392 int32 num_hits; | |
| 393 int32 num_bad_hits; | |
| 394 int32 num_requests; | |
| 395 int32 disabled; | |
| 396 uint32 header_hash; | |
| 397 }; | |
| 398 | |
| 399 ~Worker() {} | |
| 400 | |
| 401 // Methods to load and store data on disk. | |
| 402 void LoadData(); | |
| 403 void StoreData(); | |
| 404 void InitializeData(); | |
| 405 bool ReadData(PlatformFile file); | |
| 406 bool WriteData(PlatformFile file); | |
| 407 bool ReadAndVerifyHeader(PlatformFile file); | |
| 408 | |
| 409 // Book-keeping methods. | |
| 410 void Add(const Details& details); | |
| 411 void Remove(const Details& details); | |
| 412 void UpdateSize(int old_size, int new_size); | |
| 413 | |
| 414 // Bulk of report generation methods. | |
| 415 void RecordHit(const Details& old, Details* details); | |
| 416 void RecordUpdate(const Details& old, Details* details); | |
| 417 void GenerateHistograms(); | |
| 418 | |
| 419 // Cache logic methods. | |
| 420 bool CanReuse(const Details& old, const Details& current); | |
| 421 bool DataChanged(const Details& old, const Details& current); | |
| 422 bool HeadersChanged(const Details& old, const Details& current); | |
| 423 | |
| 424 KeyMap map_; | |
| 425 bool init_; | |
| 426 bool flushed_; | |
| 427 scoped_ptr<Header> header_; | |
| 428 FilePath path_; | |
| 429 | |
| 430 DISALLOW_COPY_AND_ASSIGN(Worker); | |
| 431 }; | |
| 432 | |
| 433 void InfiniteCache::Worker::Init(const FilePath& path) { | |
| 434 path_ = path; | |
| 435 LoadData(); | |
| 436 } | |
| 437 | |
| 438 void InfiniteCache::Worker::Cleanup() { | |
| 439 if (init_) | |
| 440 StoreData(); | |
| 441 | |
| 442 map_.clear(); | |
| 443 } | |
| 444 | |
| 445 void InfiniteCache::Worker::DeleteData(int* result) { | |
| 446 if (!init_) | |
| 447 return; | |
| 448 | |
| 449 map_.clear(); | |
| 450 InitializeData(); | |
| 451 file_util::Delete(path_, false); | |
| 452 *result = OK; | |
| 453 UMA_HISTOGRAM_BOOLEAN("InfiniteCache.DeleteAll", true); | |
| 454 } | |
| 455 | |
| 456 void InfiniteCache::Worker::DeleteDataBetween(base::Time initial_time, | |
| 457 base::Time end_time, | |
| 458 int* result) { | |
| 459 if (!init_) | |
| 460 return; | |
| 461 | |
| 462 for (KeyMap::iterator i = map_.begin(); i != map_.end();) { | |
| 463 Time last_access = IntToTime(i->second.last_access); | |
| 464 if (last_access >= initial_time && last_access <= end_time) { | |
| 465 KeyMap::iterator next = i; | |
| 466 ++next; | |
| 467 Remove(i->second); | |
| 468 map_.erase(i); | |
| 469 i = next; | |
| 470 continue; | |
| 471 } | |
| 472 ++i; | |
| 473 } | |
| 474 | |
| 475 file_util::Delete(path_, false); | |
| 476 StoreData(); | |
| 477 *result = OK; | |
| 478 UMA_HISTOGRAM_BOOLEAN("InfiniteCache.DeleteAll", true); | |
| 479 } | |
| 480 | |
| 481 void InfiniteCache::Worker::Process( | |
| 482 scoped_ptr<InfiniteCacheTransaction::ResourceData> data) { | |
| 483 if (!init_) | |
| 484 return; | |
| 485 | |
| 486 if (data->details.response_size > kMaxTrackingSize) | |
| 487 return; | |
| 488 | |
| 489 if (header_->num_entries == kMaxNumEntries) | |
| 490 return; | |
| 491 | |
| 492 header_->num_requests++; | |
| 493 KeyMap::iterator i = map_.find(data->key); | |
| 494 if (i != map_.end()) { | |
| 495 if (data->details.flags & DOOM_METHOD) { | |
| 496 Remove(i->second); | |
| 497 map_.erase(i); | |
| 498 return; | |
| 499 } | |
| 500 data->details.use_count = i->second.use_count; | |
| 501 data->details.update_count = i->second.update_count; | |
| 502 if (data->details.flags & CACHED) { | |
| 503 RecordHit(i->second, &data->details); | |
| 504 } else { | |
| 505 bool reused = CanReuse(i->second, data->details); | |
| 506 bool data_changed = DataChanged(i->second, data->details); | |
| 507 bool headers_changed = HeadersChanged(i->second, data->details); | |
| 508 | |
| 509 if (reused && data_changed) | |
| 510 header_->num_bad_hits++; | |
| 511 | |
| 512 if (reused) | |
| 513 RecordHit(i->second, &data->details); | |
| 514 | |
| 515 if (headers_changed) | |
| 516 UMA_HISTOGRAM_BOOLEAN("InfiniteCache.HeadersChange", true); | |
| 517 | |
| 518 if (data_changed) | |
| 519 RecordUpdate(i->second, &data->details); | |
| 520 } | |
| 521 | |
| 522 if (data->details.flags & NO_STORE) { | |
| 523 Remove(i->second); | |
| 524 map_.erase(i); | |
| 525 return; | |
| 526 } | |
| 527 | |
| 528 map_[data->key] = data->details; | |
| 529 return; | |
| 530 } | |
| 531 | |
| 532 if (data->details.flags & NO_STORE) | |
| 533 return; | |
| 534 | |
| 535 if (data->details.flags & DOOM_METHOD) | |
| 536 return; | |
| 537 | |
| 538 map_[data->key] = data->details; | |
| 539 Add(data->details); | |
| 540 } | |
| 541 | |
| 542 void InfiniteCache::Worker::LoadData() { | |
| 543 if (path_.empty()) | |
| 544 return InitializeData();; | |
| 545 | |
| 546 PlatformFile file = base::CreatePlatformFile( | |
| 547 path_, base::PLATFORM_FILE_OPEN | base::PLATFORM_FILE_READ, NULL, NULL); | |
| 548 if (file == base::kInvalidPlatformFileValue) | |
| 549 return InitializeData(); | |
| 550 if (!ReadData(file)) | |
| 551 InitializeData(); | |
| 552 base::ClosePlatformFile(file); | |
| 553 if (header_->disabled) | |
| 554 map_.clear(); | |
| 555 } | |
| 556 | |
| 557 void InfiniteCache::Worker::StoreData() { | |
| 558 if (!init_ || flushed_ || path_.empty()) | |
| 559 return; | |
| 560 | |
| 561 header_->update_time = Time::Now().ToInternalValue(); | |
| 562 header_->generation++; | |
| 563 header_->header_hash = base::Hash( | |
| 564 reinterpret_cast<char*>(header_.get()), offsetof(Header, header_hash)); | |
| 565 | |
| 566 FilePath temp_file = path_.ReplaceExtension(FILE_PATH_LITERAL("tmp")); | |
| 567 PlatformFile file = base::CreatePlatformFile( | |
| 568 temp_file, base::PLATFORM_FILE_CREATE_ALWAYS | base::PLATFORM_FILE_WRITE, | |
| 569 NULL, NULL); | |
| 570 if (file == base::kInvalidPlatformFileValue) | |
| 571 return; | |
| 572 bool success = WriteData(file); | |
| 573 base::ClosePlatformFile(file); | |
| 574 if (success) { | |
| 575 if (!file_util::ReplaceFile(temp_file, path_)) | |
| 576 file_util::Delete(temp_file, false); | |
| 577 } else { | |
| 578 LOG(ERROR) << "Failed to write experiment data"; | |
| 579 } | |
| 580 } | |
| 581 | |
| 582 void InfiniteCache::Worker::InitializeData() { | |
| 583 header_.reset(new Header); | |
| 584 memset(header_.get(), 0, sizeof(Header)); | |
| 585 header_->magic = kMagicSignature; | |
| 586 header_->version = kCurrentVersion; | |
| 587 header_->creation_time = Time::Now().ToInternalValue(); | |
| 588 | |
| 589 UMA_HISTOGRAM_BOOLEAN("InfiniteCache.Initialize", true); | |
| 590 init_ = true; | |
| 591 } | |
| 592 | |
| 593 bool InfiniteCache::Worker::ReadData(PlatformFile file) { | |
| 594 if (!ReadAndVerifyHeader(file)) | |
| 595 return false; | |
| 596 | |
| 597 scoped_array<char> buffer(new char[kBufferSize]); | |
| 598 size_t offset = sizeof(Header); | |
| 599 uint32 hash = adler32(0, Z_NULL, 0); | |
| 600 | |
| 601 for (int remaining_records = header_->num_entries; remaining_records;) { | |
| 602 int num_records = std::min(header_->num_entries, | |
| 603 static_cast<int>(kMaxRecordsToRead)); | |
| 604 size_t num_bytes = num_records * kRecordSize; | |
| 605 remaining_records -= num_records; | |
| 606 if (!remaining_records) | |
| 607 num_bytes += sizeof(uint32); // Trailing hash. | |
| 608 DCHECK_LE(num_bytes, kBufferSize); | |
| 609 | |
| 610 if (!ReadPlatformFile(file, offset, buffer.get(), num_bytes)) | |
| 611 return false; | |
| 612 | |
| 613 hash = adler32(hash, reinterpret_cast<const Bytef*>(buffer.get()), | |
| 614 num_records * kRecordSize); | |
| 615 if (!remaining_records && | |
| 616 hash != *reinterpret_cast<uint32*>(buffer.get() + | |
| 617 num_records * kRecordSize)) { | |
| 618 return false; | |
| 619 } | |
| 620 | |
| 621 for (int i = 0; i < num_records; i++) { | |
| 622 char* record = buffer.get() + i * kRecordSize; | |
| 623 Key key = *reinterpret_cast<Key*>(record); | |
| 624 Details details = *reinterpret_cast<Details*>(record + sizeof(key)); | |
| 625 map_[key] = details; | |
| 626 } | |
| 627 offset += num_bytes; | |
| 628 } | |
| 629 if (header_->num_entries != static_cast<int>(map_.size())) { | |
| 630 NOTREACHED(); | |
| 631 return false; | |
| 632 } | |
| 633 | |
| 634 init_ = true; | |
| 635 return true; | |
| 636 } | |
| 637 | |
| 638 bool InfiniteCache::Worker::WriteData(PlatformFile file) { | |
| 639 if (!base::TruncatePlatformFile(file, 0)) | |
| 640 return false; | |
| 641 | |
| 642 if (!WritePlatformFile(file, 0, header_.get(), sizeof(Header))) | |
| 643 return false; | |
| 644 | |
| 645 scoped_array<char> buffer(new char[kBufferSize]); | |
| 646 size_t offset = sizeof(Header); | |
| 647 uint32 hash = adler32(0, Z_NULL, 0); | |
| 648 | |
| 649 DCHECK_EQ(header_->num_entries, static_cast<int32>(map_.size())); | |
| 650 KeyMap::iterator iterator = map_.begin(); | |
| 651 for (int remaining_records = header_->num_entries; remaining_records;) { | |
| 652 int num_records = std::min(header_->num_entries, | |
| 653 static_cast<int>(kMaxRecordsToRead)); | |
| 654 size_t num_bytes = num_records * kRecordSize; | |
| 655 remaining_records -= num_records; | |
| 656 | |
| 657 for (int i = 0; i < num_records; i++) { | |
| 658 if (iterator == map_.end()) { | |
| 659 NOTREACHED(); | |
| 660 return false; | |
| 661 } | |
| 662 char* record = buffer.get() + i * kRecordSize; | |
| 663 *reinterpret_cast<Key*>(record) = iterator->first; | |
| 664 *reinterpret_cast<Details*>(record + sizeof(Key)) = iterator->second; | |
| 665 ++iterator; | |
| 666 } | |
| 667 | |
| 668 hash = adler32(hash, reinterpret_cast<const Bytef*>(buffer.get()), | |
| 669 num_bytes); | |
| 670 | |
| 671 if (!remaining_records) { | |
| 672 num_bytes += sizeof(uint32); // Trailing hash. | |
| 673 *reinterpret_cast<uint32*>(buffer.get() + | |
| 674 num_records * kRecordSize) = hash; | |
| 675 } | |
| 676 | |
| 677 DCHECK_LE(num_bytes, kBufferSize); | |
| 678 if (!WritePlatformFile(file, offset, buffer.get(), num_bytes)) | |
| 679 return false; | |
| 680 | |
| 681 offset += num_bytes; | |
| 682 } | |
| 683 base::FlushPlatformFile(file); // Ignore return value. | |
| 684 return true; | |
| 685 } | |
| 686 | |
| 687 bool InfiniteCache::Worker::ReadAndVerifyHeader(PlatformFile file) { | |
| 688 base::PlatformFileInfo info; | |
| 689 if (!base::GetPlatformFileInfo(file, &info)) | |
| 690 return false; | |
| 691 | |
| 692 if (info.size < static_cast<int>(sizeof(Header))) | |
| 693 return false; | |
| 694 | |
| 695 header_.reset(new Header); | |
| 696 if (!ReadPlatformFile(file, 0, header_.get(), sizeof(Header))) | |
| 697 return false; | |
| 698 | |
| 699 if (header_->magic != kMagicSignature) | |
| 700 return false; | |
| 701 | |
| 702 if (header_->version != kCurrentVersion) | |
| 703 return false; | |
| 704 | |
| 705 if (header_->num_entries > kMaxNumEntries) | |
| 706 return false; | |
| 707 | |
| 708 size_t expected_size = kRecordSize * header_->num_entries + | |
| 709 sizeof(Header) + sizeof(uint32); // Trailing hash. | |
| 710 | |
| 711 if (info.size < static_cast<int>(expected_size)) | |
| 712 return false; | |
| 713 | |
| 714 uint32 hash = base::Hash(reinterpret_cast<char*>(header_.get()), | |
| 715 offsetof(Header, header_hash)); | |
| 716 if (hash != header_->header_hash) | |
| 717 return false; | |
| 718 | |
| 719 return true; | |
| 720 } | |
| 721 | |
| 722 void InfiniteCache::Worker::Query(int* result) { | |
| 723 *result = static_cast<int>(map_.size()); | |
| 724 } | |
| 725 | |
| 726 void InfiniteCache::Worker::Flush(int* result) { | |
| 727 flushed_ = false; | |
| 728 StoreData(); | |
| 729 flushed_ = true; | |
| 730 *result = OK; | |
| 731 } | |
| 732 | |
| 733 void InfiniteCache::Worker::OnTimer() { | |
| 734 header_->use_minutes += kTimerMinutes; | |
| 735 GenerateHistograms(); | |
| 736 StoreData(); | |
| 737 } | |
| 738 | |
| 739 void InfiniteCache::Worker::Add(const Details& details) { | |
| 740 UpdateSize(0, details.headers_size); | |
| 741 UpdateSize(0, details.response_size); | |
| 742 header_->num_entries = static_cast<int>(map_.size()); | |
| 743 if (header_->num_entries == kMaxNumEntries) { | |
| 744 int use_hours = header_->use_minutes / 60; | |
| 745 int age_hours = (Time::Now() - | |
| 746 Time::FromInternalValue(header_->creation_time)).InHours(); | |
| 747 UMA_HISTOGRAM_COUNTS_10000("InfiniteCache.MaxUseTime", use_hours); | |
| 748 UMA_HISTOGRAM_COUNTS_10000("InfiniteCache.MaxAge", age_hours); | |
| 749 | |
| 750 int entry_size = static_cast<int>(header_->total_size / kMaxNumEntries); | |
| 751 UMA_HISTOGRAM_COUNTS("InfiniteCache.FinalAvgEntrySize", entry_size); | |
| 752 header_->disabled = 1; | |
| 753 map_.clear(); | |
| 754 } | |
| 755 } | |
| 756 | |
| 757 void InfiniteCache::Worker::Remove(const Details& details) { | |
| 758 UpdateSize(details.headers_size, 0); | |
| 759 UpdateSize(details.response_size, 0); | |
| 760 header_->num_entries--; | |
| 761 } | |
| 762 | |
| 763 void InfiniteCache::Worker::UpdateSize(int old_size, int new_size) { | |
| 764 header_->total_size += new_size - old_size; | |
| 765 DCHECK_GE(header_->total_size, 0); | |
| 766 } | |
| 767 | |
| 768 void InfiniteCache::Worker::RecordHit(const Details& old, Details* details) { | |
| 769 header_->num_hits++; | |
| 770 int access_delta = (IntToTime(details->last_access) - | |
| 771 IntToTime(old.last_access)).InMinutes(); | |
| 772 if (old.use_count) | |
| 773 UMA_HISTOGRAM_COUNTS("InfiniteCache.ReuseAge", access_delta); | |
| 774 else | |
| 775 UMA_HISTOGRAM_COUNTS("InfiniteCache.FirstReuseAge", access_delta); | |
| 776 | |
| 777 details->use_count = old.use_count; | |
| 778 if (details->use_count < kuint8max) | |
| 779 details->use_count++; | |
| 780 UMA_HISTOGRAM_CUSTOM_COUNTS("InfiniteCache.UseCount", details->use_count, 0, | |
| 781 kuint8max, 25); | |
| 782 } | |
| 783 | |
| 784 void InfiniteCache::Worker::RecordUpdate(const Details& old, Details* details) { | |
| 785 int access_delta = (IntToTime(details->last_access) - | |
| 786 IntToTime(old.last_access)).InMinutes(); | |
| 787 if (old.update_count) | |
| 788 UMA_HISTOGRAM_COUNTS("InfiniteCache.UpdateAge", access_delta); | |
| 789 else | |
| 790 UMA_HISTOGRAM_COUNTS("InfiniteCache.FirstUpdateAge", access_delta); | |
| 791 | |
| 792 details->update_count = old.update_count; | |
| 793 if (details->update_count < kuint8max) | |
| 794 details->update_count++; | |
| 795 | |
| 796 UMA_HISTOGRAM_CUSTOM_COUNTS("InfiniteCache.UpdateCount", | |
| 797 details->update_count, 0, kuint8max, 25); | |
| 798 details->use_count = 0; | |
| 799 } | |
| 800 | |
| 801 void InfiniteCache::Worker::GenerateHistograms() { | |
| 802 bool new_size_step = (header_->total_size / kReportSizeStep != | |
| 803 header_->size_last_report / kReportSizeStep); | |
| 804 header_->size_last_report = header_->total_size; | |
| 805 if (!new_size_step && (header_->use_minutes % 60 != 0)) | |
| 806 return; | |
| 807 | |
| 808 if (header_->disabled) | |
| 809 return; | |
| 810 | |
| 811 int hit_ratio = header_->num_hits * 100; | |
| 812 if (header_->num_requests) | |
| 813 hit_ratio /= header_->num_requests; | |
| 814 else | |
| 815 hit_ratio = 0; | |
| 816 | |
| 817 // We'll be generating pairs of histograms that can be used to get the hit | |
| 818 // ratio for any bucket of the paired histogram. | |
| 819 bool report_second_stat = base::RandInt(0, 99) < hit_ratio; | |
| 820 | |
| 821 if (header_->use_minutes % 60 == 0) { | |
| 822 int use_hours = header_->use_minutes / 60; | |
| 823 int age_hours = (Time::Now() - | |
| 824 Time::FromInternalValue(header_->creation_time)).InHours(); | |
| 825 UMA_HISTOGRAM_COUNTS_10000("InfiniteCache.UseTime", use_hours); | |
| 826 UMA_HISTOGRAM_COUNTS_10000("InfiniteCache.Age", age_hours); | |
| 827 if (report_second_stat) { | |
| 828 UMA_HISTOGRAM_COUNTS_10000("InfiniteCache.HitRatioByUseTime", use_hours); | |
| 829 UMA_HISTOGRAM_COUNTS_10000("InfiniteCache.HitRatioByAge", age_hours); | |
| 830 } | |
| 831 } | |
| 832 | |
| 833 if (new_size_step) { | |
| 834 int size_bucket = static_cast<int>(header_->total_size / kReportSizeStep); | |
| 835 UMA_HISTOGRAM_ENUMERATION("InfiniteCache.Size", std::min(size_bucket, 50), | |
| 836 51); | |
| 837 UMA_HISTOGRAM_ENUMERATION("InfiniteCache.SizeCoarse", size_bucket / 5, 51); | |
| 838 UMA_HISTOGRAM_COUNTS("InfiniteCache.Entries", header_->num_entries); | |
| 839 UMA_HISTOGRAM_COUNTS_10000("InfiniteCache.BadHits", header_->num_bad_hits); | |
| 840 if (report_second_stat) { | |
| 841 UMA_HISTOGRAM_ENUMERATION("InfiniteCache.HitRatioBySize", | |
| 842 std::min(size_bucket, 50), 51); | |
| 843 UMA_HISTOGRAM_ENUMERATION("InfiniteCache.HitRatioBySizeCoarse", | |
| 844 size_bucket / 5, 51); | |
| 845 UMA_HISTOGRAM_COUNTS("InfiniteCache.HitRatioByEntries", | |
| 846 header_->num_entries); | |
| 847 } | |
| 848 header_->num_hits = 0; | |
| 849 header_->num_bad_hits = 0; | |
| 850 header_->num_requests = 0; | |
| 851 } | |
| 852 } | |
| 853 | |
| 854 bool InfiniteCache::Worker::CanReuse(const Details& old, | |
| 855 const Details& current) { | |
| 856 enum ReuseStatus { | |
| 857 REUSE_OK = 0, | |
| 858 REUSE_NO_CACHE, | |
| 859 REUSE_ALWAYS_EXPIRED, | |
| 860 REUSE_EXPIRED, | |
| 861 REUSE_TRUNCATED, | |
| 862 REUSE_VARY, | |
| 863 REUSE_DUMMY_VALUE, | |
| 864 // Not an individual value; it's added to another reason. | |
| 865 REUSE_REVALIDATEABLE = 7 | |
| 866 }; | |
| 867 int reason = REUSE_OK; | |
| 868 | |
| 869 if (old.flags & NO_CACHE) | |
| 870 reason = REUSE_NO_CACHE; | |
| 871 | |
| 872 if (old.flags & EXPIRED) | |
| 873 reason = REUSE_ALWAYS_EXPIRED; | |
| 874 | |
| 875 if (old.flags & TRUNCATED) | |
| 876 reason = REUSE_TRUNCATED; | |
| 877 | |
| 878 Time expiration = IntToTime(old.expiration); | |
| 879 if (expiration < Time::Now()) | |
| 880 reason = REUSE_EXPIRED; | |
| 881 | |
| 882 if (old.vary_hash != current.vary_hash) | |
| 883 reason = REUSE_VARY; | |
| 884 | |
| 885 bool have_to_drop = (old.flags & TRUNCATED) && !(old.flags & RESUMABLE); | |
| 886 if (reason && (old.flags & REVALIDATEABLE) && !have_to_drop) | |
| 887 reason += REUSE_REVALIDATEABLE; | |
| 888 | |
| 889 UMA_HISTOGRAM_ENUMERATION("InfiniteCache.ReuseFailure", reason, 15); | |
| 890 return !reason; | |
| 891 } | |
| 892 | |
| 893 bool InfiniteCache::Worker::DataChanged(const Details& old, | |
| 894 const Details& current) { | |
| 895 bool changed = false; | |
| 896 if (old.response_size != current.response_size) { | |
| 897 changed = true; | |
| 898 UpdateSize(old.response_size, current.response_size); | |
| 899 } | |
| 900 | |
| 901 if (old.response_hash != current.response_hash) | |
| 902 changed = true; | |
| 903 | |
| 904 return changed; | |
| 905 } | |
| 906 | |
| 907 bool InfiniteCache::Worker::HeadersChanged(const Details& old, | |
| 908 const Details& current) { | |
| 909 bool changed = false; | |
| 910 if (old.headers_size != current.headers_size) { | |
| 911 changed = true; | |
| 912 UpdateSize(old.headers_size, current.headers_size); | |
| 913 } | |
| 914 | |
| 915 if (old.headers_hash != current.headers_hash) | |
| 916 changed = true; | |
| 917 | |
| 918 return changed; | |
| 919 } | |
| 920 | |
| 921 // ---------------------------------------------------------------------------- | |
| 922 | |
| 923 InfiniteCache::InfiniteCache() { | |
| 924 } | |
| 925 | |
| 926 InfiniteCache::~InfiniteCache() { | |
| 927 if (!worker_) | |
| 928 return; | |
| 929 | |
| 930 task_runner_->PostTask(FROM_HERE, | |
| 931 base::Bind(&InfiniteCache::Worker::Cleanup, worker_)); | |
| 932 worker_ = NULL; | |
| 933 } | |
| 934 | |
| 935 void InfiniteCache::Init(const FilePath& path) { | |
| 936 worker_pool_ = new base::SequencedWorkerPool(1, "Infinite cache thread"); | |
| 937 task_runner_ = worker_pool_->GetSequencedTaskRunnerWithShutdownBehavior( | |
| 938 worker_pool_->GetSequenceToken(), | |
| 939 base::SequencedWorkerPool::CONTINUE_ON_SHUTDOWN); | |
| 940 | |
| 941 worker_ = new Worker(); | |
| 942 task_runner_->PostTask(FROM_HERE, | |
| 943 base::Bind(&InfiniteCache::Worker::Init, worker_, | |
| 944 path)); | |
| 945 | |
| 946 timer_.Start(FROM_HERE, TimeDelta::FromMinutes(kTimerMinutes), this, | |
| 947 &InfiniteCache::OnTimer); | |
| 948 } | |
| 949 | |
| 950 InfiniteCacheTransaction* InfiniteCache::CreateInfiniteCacheTransaction() { | |
| 951 if (!worker_) | |
| 952 return NULL; | |
| 953 return new InfiniteCacheTransaction(this); | |
| 954 } | |
| 955 | |
| 956 int InfiniteCache::DeleteData(const CompletionCallback& callback) { | |
| 957 if (!worker_) | |
| 958 return OK; | |
| 959 int* result = new int; | |
| 960 task_runner_->PostTaskAndReply( | |
| 961 FROM_HERE, base::Bind(&InfiniteCache::Worker::DeleteData, worker_, | |
| 962 result), | |
| 963 base::Bind(&OnComplete, callback, base::Owned(result))); | |
| 964 return ERR_IO_PENDING; | |
| 965 } | |
| 966 | |
| 967 int InfiniteCache::DeleteDataBetween(base::Time initial_time, | |
| 968 base::Time end_time, | |
| 969 const CompletionCallback& callback) { | |
| 970 if (!worker_) | |
| 971 return OK; | |
| 972 int* result = new int; | |
| 973 task_runner_->PostTaskAndReply( | |
| 974 FROM_HERE, base::Bind(&InfiniteCache::Worker::DeleteDataBetween, worker_, | |
| 975 initial_time, end_time, result), | |
| 976 base::Bind(&OnComplete, callback, base::Owned(result))); | |
| 977 return ERR_IO_PENDING; | |
| 978 } | |
| 979 | |
| 980 std::string InfiniteCache::GenerateKey(const HttpRequestInfo* request) { | |
| 981 // Don't add any upload data identifier. | |
| 982 return HttpUtil::SpecForRequest(request->url); | |
| 983 } | |
| 984 | |
| 985 void InfiniteCache::ProcessResource( | |
| 986 scoped_ptr<InfiniteCacheTransaction::ResourceData> data) { | |
| 987 if (!worker_) | |
| 988 return; | |
| 989 task_runner_->PostTask(FROM_HERE, | |
| 990 base::Bind(&InfiniteCache::Worker::Process, worker_, | |
| 991 base::Passed(&data))); | |
| 992 } | |
| 993 | |
| 994 void InfiniteCache::OnTimer() { | |
| 995 task_runner_->PostTask(FROM_HERE, | |
| 996 base::Bind(&InfiniteCache::Worker::OnTimer, worker_)); | |
| 997 } | |
| 998 | |
| 999 int InfiniteCache::QueryItemsForTest(const CompletionCallback& callback) { | |
| 1000 DCHECK(worker_); | |
| 1001 int* result = new int; | |
| 1002 task_runner_->PostTaskAndReply( | |
| 1003 FROM_HERE, base::Bind(&InfiniteCache::Worker::Query, worker_, result), | |
| 1004 base::Bind(&OnComplete, callback, base::Owned(result))); | |
| 1005 return net::ERR_IO_PENDING; | |
| 1006 } | |
| 1007 | |
| 1008 int InfiniteCache::FlushDataForTest(const CompletionCallback& callback) { | |
| 1009 DCHECK(worker_); | |
| 1010 int* result = new int; | |
| 1011 task_runner_->PostTaskAndReply( | |
| 1012 FROM_HERE, base::Bind(&InfiniteCache::Worker::Flush, worker_, result), | |
| 1013 base::Bind(&OnComplete, callback, base::Owned(result))); | |
| 1014 return net::ERR_IO_PENDING; | |
| 1015 } | |
| 1016 | |
| 1017 } // namespace net | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 10907241:
Revert 156846 - Http Cache: Add code for simulating an infinite HTTP cache. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/src/