Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h" | 5 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h" |
| 6 | 6 |
| 7 #include "base/md5.h" | 7 #include "base/md5.h" |
| 8 #include "base/metrics/histogram.h" | 8 #include "base/metrics/histogram.h" |
| 9 | 9 |
| 10 namespace { | 10 namespace { |
| (...skipping 30 matching lines...) Expand all Loading... | |
| 41 // Although fseek takes negative values, for this case, we only want | 41 // Although fseek takes negative values, for this case, we only want |
| 42 // to skip forward. | 42 // to skip forward. |
| 43 DCHECK(static_cast<long>(bytes) >= 0); | 43 DCHECK(static_cast<long>(bytes) >= 0); |
| 44 if (static_cast<long>(bytes) < 0) | 44 if (static_cast<long>(bytes) < 0) |
| 45 return false; | 45 return false; |
| 46 int rv = fseek(fp, static_cast<long>(bytes), SEEK_CUR); | 46 int rv = fseek(fp, static_cast<long>(bytes), SEEK_CUR); |
| 47 DCHECK_EQ(rv, 0); | 47 DCHECK_EQ(rv, 0); |
| 48 return rv == 0; | 48 return rv == 0; |
| 49 } | 49 } |
| 50 | 50 |
| 51 // Read an array of |nmemb| items from |fp| into |ptr|, and fold the | 51 // Read from |fp| into |item|, and fold the input data into the |
| 52 // input data into the checksum in |context|, if non-NULL. Return | 52 // checksum in |context|, if non-NULL. Return true on success. |
| 53 // true on success. | |
| 54 template <class T> | 53 template <class T> |
| 55 bool ReadArray(T* ptr, size_t nmemb, FILE* fp, base::MD5Context* context) { | 54 bool ReadItem(T* item, FILE* fp, base::MD5Context* context) { |
| 56 const size_t ret = fread(ptr, sizeof(T), nmemb, fp); | 55 const size_t ret = fread(item, sizeof(T), 1, fp); |
| 57 if (ret != nmemb) | 56 if (ret != 1) |
| 58 return false; | 57 return false; |
| 59 | 58 |
| 60 if (context) { | 59 if (context) { |
| 61 base::MD5Update(context, | 60 base::MD5Update(context, |
| 62 base::StringPiece(reinterpret_cast<char*>(ptr), | 61 base::StringPiece(reinterpret_cast<char*>(item), |
| 63 sizeof(T) * nmemb)); | 62 sizeof(T))); |
| 64 } | 63 } |
| 65 return true; | 64 return true; |
| 66 } | 65 } |
| 67 | 66 |
| 68 // Write an array of |nmemb| items from |ptr| to |fp|, and fold the | 67 // Write |item| to |fp|, and fold the output data into the checksum in |
| 69 // output data into the checksum in |context|, if non-NULL. Return | 68 // |context|, if non-NULL. Return true on success. |
| 70 // true on success. | |
| 71 template <class T> | 69 template <class T> |
| 72 bool WriteArray(const T* ptr, size_t nmemb, FILE* fp, | 70 bool WriteItem(const T& item, FILE* fp, base::MD5Context* context) { |
| 73 base::MD5Context* context) { | 71 const size_t ret = fwrite(&item, sizeof(T), 1, fp); |
| 74 const size_t ret = fwrite(ptr, sizeof(T), nmemb, fp); | 72 if (ret != 1) |
| 75 if (ret != nmemb) | |
| 76 return false; | 73 return false; |
| 77 | 74 |
| 78 if (context) { | 75 if (context) { |
| 79 base::MD5Update(context, | 76 base::MD5Update(context, |
| 80 base::StringPiece(reinterpret_cast<const char*>(ptr), | 77 base::StringPiece(reinterpret_cast<const char*>(&item), |
| 81 sizeof(T) * nmemb)); | 78 sizeof(T))); |
| 82 } | 79 } |
| 83 | 80 |
| 84 return true; | 81 return true; |
| 85 } | 82 } |
| 86 | 83 |
| 87 // Expand |values| to fit |count| new items, read those items from | 84 // Read |count| items into |values| from |fp|, and fold them into the |
| 88 // |fp| and fold them into the checksum in |context|. Returns true on | 85 // checksum in |context|. Returns true on success. |
| 89 // success. | 86 template <typename CT> |
| 90 template <class T> | 87 bool ReadToContainer(CT* values, size_t count, FILE* fp, |
| 91 bool ReadToVector(std::vector<T>* values, size_t count, FILE* fp, | 88 base::MD5Context* context) { |
| 92 base::MD5Context* context) { | |
| 93 // Pointers into an empty vector may not be valid. | |
| 94 if (!count) | 89 if (!count) |
| 95 return true; | 90 return true; |
| 96 | 91 |
| 97 // Grab the size for purposes of finding where to read to. The | 92 for (size_t i = 0; i < count; ++i) { |
| 98 // resize could invalidate any iterator captured here. | 93 typename CT::value_type value; |
| 99 const size_t original_size = values->size(); | 94 if (!ReadItem(&value, fp, context)) |
| 100 values->resize(original_size + count); | 95 return false; |
| 101 | 96 values->insert(values->end(), value); |
|
jar (doing other things)
2011/10/21 19:10:45
It might be clearer to:
values->push_back(value);
Scott Hess - ex-Googler
2011/10/21 23:39:32
That is the magic which allows the function to wor
| |
| 102 // Sayeth Herb Sutter: Vectors are guaranteed to be contiguous. So | |
| 103 // get a pointer to where to read the data to. | |
| 104 T* ptr = &((*values)[original_size]); | |
| 105 if (!ReadArray(ptr, count, fp, context)) { | |
| 106 values->resize(original_size); | |
| 107 return false; | |
| 108 } | 97 } |
| 109 | 98 |
| 110 return true; | 99 return true; |
| 111 } | 100 } |
| 112 | 101 |
| 113 // Write all of |values| to |fp|, and fold the data into the checksum | 102 // Write all of |values| to |fp|, and fold the data into the checksum |
| 114 // in |context|, if non-NULL. Returns true on succsess. | 103 // in |context|, if non-NULL. Returns true on succsess. |
| 115 template <class T> | 104 template <typename CT> |
| 116 bool WriteVector(const std::vector<T>& values, FILE* fp, | 105 bool WriteContainer(const CT& values, FILE* fp, |
| 117 base::MD5Context* context) { | 106 base::MD5Context* context) { |
| 118 // Pointers into empty vectors may not be valid. | |
| 119 if (values.empty()) | 107 if (values.empty()) |
| 120 return true; | 108 return true; |
| 121 | 109 |
| 122 // Sayeth Herb Sutter: Vectors are guaranteed to be contiguous. So | 110 for (typename CT::const_iterator iter = values.begin(); |
| 123 // get a pointer to where to write from. | 111 iter != values.end(); ++iter) { |
| 124 const T* ptr = &(values[0]); | 112 if (!WriteItem(*iter, fp, context)) |
| 125 return WriteArray(ptr, values.size(), fp, context); | 113 return false; |
| 126 } | 114 } |
| 127 | |
| 128 // Read an array of |count| integers and add them to |values|. | |
| 129 // Returns true on success. | |
| 130 bool ReadToChunkSet(std::set<int32>* values, size_t count, FILE* fp, | |
| 131 base::MD5Context* context) { | |
| 132 if (!count) | |
| 133 return true; | |
| 134 | |
| 135 std::vector<int32> flat_values; | |
| 136 if (!ReadToVector(&flat_values, count, fp, context)) | |
| 137 return false; | |
| 138 | |
| 139 values->insert(flat_values.begin(), flat_values.end()); | |
| 140 return true; | 115 return true; |
| 141 } | 116 } |
| 142 | 117 |
| 143 // Write the contents of |values| as an array of integers. Returns | |
| 144 // true on success. | |
| 145 bool WriteChunkSet(const std::set<int32>& values, FILE* fp, | |
| 146 base::MD5Context* context) { | |
| 147 if (values.empty()) | |
| 148 return true; | |
| 149 | |
| 150 const std::vector<int32> flat_values(values.begin(), values.end()); | |
| 151 return WriteVector(flat_values, fp, context); | |
| 152 } | |
| 153 | |
| 154 // Delete the chunks in |deleted| from |chunks|. | 118 // Delete the chunks in |deleted| from |chunks|. |
| 155 void DeleteChunksFromSet(const base::hash_set<int32>& deleted, | 119 void DeleteChunksFromSet(const base::hash_set<int32>& deleted, |
| 156 std::set<int32>* chunks) { | 120 std::set<int32>* chunks) { |
| 157 for (std::set<int32>::iterator iter = chunks->begin(); | 121 for (std::set<int32>::iterator iter = chunks->begin(); |
| 158 iter != chunks->end();) { | 122 iter != chunks->end();) { |
| 159 std::set<int32>::iterator prev = iter++; | 123 std::set<int32>::iterator prev = iter++; |
| 160 if (deleted.count(*prev) > 0) | 124 if (deleted.count(*prev) > 0) |
| 161 chunks->erase(prev); | 125 chunks->erase(prev); |
| 162 } | 126 } |
| 163 } | 127 } |
| (...skipping 20 matching lines...) Expand all Loading... | |
| 184 | 148 |
| 185 return true; | 149 return true; |
| 186 } | 150 } |
| 187 | 151 |
| 188 // This a helper function that reads header to |header|. Returns true if the | 152 // This a helper function that reads header to |header|. Returns true if the |
| 189 // magic number is correct and santiy check passes. | 153 // magic number is correct and santiy check passes. |
| 190 bool ReadAndVerifyHeader(const FilePath& filename, | 154 bool ReadAndVerifyHeader(const FilePath& filename, |
| 191 FILE* fp, | 155 FILE* fp, |
| 192 FileHeader* header, | 156 FileHeader* header, |
| 193 base::MD5Context* context) { | 157 base::MD5Context* context) { |
| 194 if (!ReadArray(header, 1, fp, context)) | 158 if (!ReadItem(header, fp, context)) |
| 195 return false; | 159 return false; |
| 196 if (header->magic != kFileMagic || header->version != kFileVersion) | 160 if (header->magic != kFileMagic || header->version != kFileVersion) |
| 197 return false; | 161 return false; |
| 198 if (!FileHeaderSanityCheck(filename, *header)) | 162 if (!FileHeaderSanityCheck(filename, *header)) |
| 199 return false; | 163 return false; |
| 200 return true; | 164 return true; |
| 201 } | 165 } |
| 202 | 166 |
| 203 } // namespace | 167 } // namespace |
| 204 | 168 |
| (...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 286 | 250 |
| 287 bool SafeBrowsingStoreFile::BeginChunk() { | 251 bool SafeBrowsingStoreFile::BeginChunk() { |
| 288 return ClearChunkBuffers(); | 252 return ClearChunkBuffers(); |
| 289 } | 253 } |
| 290 | 254 |
| 291 bool SafeBrowsingStoreFile::WriteAddPrefix(int32 chunk_id, SBPrefix prefix) { | 255 bool SafeBrowsingStoreFile::WriteAddPrefix(int32 chunk_id, SBPrefix prefix) { |
| 292 add_prefixes_.push_back(SBAddPrefix(chunk_id, prefix)); | 256 add_prefixes_.push_back(SBAddPrefix(chunk_id, prefix)); |
| 293 return true; | 257 return true; |
| 294 } | 258 } |
| 295 | 259 |
| 296 bool SafeBrowsingStoreFile::GetAddPrefixes( | 260 bool SafeBrowsingStoreFile::GetAddPrefixes(SBAddPrefixContainer* add_prefixes) { |
| 297 std::vector<SBAddPrefix>* add_prefixes) { | |
| 298 add_prefixes->clear(); | 261 add_prefixes->clear(); |
| 299 | 262 |
| 300 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); | 263 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); |
| 301 if (file.get() == NULL) return false; | 264 if (file.get() == NULL) return false; |
| 302 | 265 |
| 303 FileHeader header; | 266 FileHeader header; |
| 304 if (!ReadAndVerifyHeader(filename_, file.get(), &header, NULL)) | 267 if (!ReadAndVerifyHeader(filename_, file.get(), &header, NULL)) |
| 305 return OnCorruptDatabase(); | 268 return OnCorruptDatabase(); |
| 306 | 269 |
| 307 size_t add_prefix_offset = header.add_chunk_count * sizeof(int32) + | 270 size_t add_prefix_offset = header.add_chunk_count * sizeof(int32) + |
| 308 header.sub_chunk_count * sizeof(int32); | 271 header.sub_chunk_count * sizeof(int32); |
| 309 if (!FileSkip(add_prefix_offset, file.get())) | 272 if (!FileSkip(add_prefix_offset, file.get())) |
| 310 return false; | 273 return false; |
| 311 | 274 |
| 312 if (!ReadToVector(add_prefixes, header.add_prefix_count, file.get(), NULL)) | 275 if (!ReadToContainer(add_prefixes, header.add_prefix_count, file.get(), NULL)) |
| 313 return false; | 276 return false; |
| 314 | 277 |
| 315 return true; | 278 return true; |
| 316 } | 279 } |
| 317 | 280 |
| 318 bool SafeBrowsingStoreFile::GetAddFullHashes( | 281 bool SafeBrowsingStoreFile::GetAddFullHashes( |
| 319 std::vector<SBAddFullHash>* add_full_hashes) { | 282 std::vector<SBAddFullHash>* add_full_hashes) { |
| 320 add_full_hashes->clear(); | 283 add_full_hashes->clear(); |
| 321 | 284 |
| 322 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); | 285 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); |
| 323 if (file.get() == NULL) return false; | 286 if (file.get() == NULL) return false; |
| 324 | 287 |
| 325 FileHeader header; | 288 FileHeader header; |
| 326 if (!ReadAndVerifyHeader(filename_, file.get(), &header, NULL)) | 289 if (!ReadAndVerifyHeader(filename_, file.get(), &header, NULL)) |
| 327 return OnCorruptDatabase(); | 290 return OnCorruptDatabase(); |
| 328 | 291 |
| 329 size_t offset = | 292 size_t offset = |
| 330 header.add_chunk_count * sizeof(int32) + | 293 header.add_chunk_count * sizeof(int32) + |
| 331 header.sub_chunk_count * sizeof(int32) + | 294 header.sub_chunk_count * sizeof(int32) + |
| 332 header.add_prefix_count * sizeof(SBAddPrefix) + | 295 header.add_prefix_count * sizeof(SBAddPrefix) + |
| 333 header.sub_prefix_count * sizeof(SBSubPrefix); | 296 header.sub_prefix_count * sizeof(SBSubPrefix); |
| 334 if (!FileSkip(offset, file.get())) | 297 if (!FileSkip(offset, file.get())) |
| 335 return false; | 298 return false; |
| 336 | 299 |
| 337 return ReadToVector(add_full_hashes, | 300 return ReadToContainer(add_full_hashes, |
| 338 header.add_hash_count, | 301 header.add_hash_count, |
| 339 file.get(), | 302 file.get(), |
| 340 NULL); | 303 NULL); |
| 341 } | 304 } |
| 342 | 305 |
| 343 bool SafeBrowsingStoreFile::WriteAddHash(int32 chunk_id, | 306 bool SafeBrowsingStoreFile::WriteAddHash(int32 chunk_id, |
| 344 base::Time receive_time, | 307 base::Time receive_time, |
| 345 const SBFullHash& full_hash) { | 308 const SBFullHash& full_hash) { |
| 346 add_hashes_.push_back(SBAddFullHash(chunk_id, receive_time, full_hash)); | 309 add_hashes_.push_back(SBAddFullHash(chunk_id, receive_time, full_hash)); |
| 347 return true; | 310 return true; |
| 348 } | 311 } |
| 349 | 312 |
| 350 bool SafeBrowsingStoreFile::WriteSubPrefix(int32 chunk_id, | 313 bool SafeBrowsingStoreFile::WriteSubPrefix(int32 chunk_id, |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 386 // Structures should all be clear unless something bad happened. | 349 // Structures should all be clear unless something bad happened. |
| 387 DCHECK(add_chunks_cache_.empty()); | 350 DCHECK(add_chunks_cache_.empty()); |
| 388 DCHECK(sub_chunks_cache_.empty()); | 351 DCHECK(sub_chunks_cache_.empty()); |
| 389 DCHECK(add_del_cache_.empty()); | 352 DCHECK(add_del_cache_.empty()); |
| 390 DCHECK(sub_del_cache_.empty()); | 353 DCHECK(sub_del_cache_.empty()); |
| 391 DCHECK(add_prefixes_.empty()); | 354 DCHECK(add_prefixes_.empty()); |
| 392 DCHECK(sub_prefixes_.empty()); | 355 DCHECK(sub_prefixes_.empty()); |
| 393 DCHECK(add_hashes_.empty()); | 356 DCHECK(add_hashes_.empty()); |
| 394 DCHECK(sub_hashes_.empty()); | 357 DCHECK(sub_hashes_.empty()); |
| 395 DCHECK_EQ(chunks_written_, 0); | 358 DCHECK_EQ(chunks_written_, 0); |
| 396 add_prefixes_added_ = 0; | |
| 397 | 359 |
| 398 // Since the following code will already hit the profile looking for | 360 // Since the following code will already hit the profile looking for |
| 399 // database files, this is a reasonable to time delete any old | 361 // database files, this is a reasonable to time delete any old |
| 400 // files. | 362 // files. |
| 401 CheckForOriginalAndDelete(filename_); | 363 CheckForOriginalAndDelete(filename_); |
| 402 | 364 |
| 403 corruption_seen_ = false; | 365 corruption_seen_ = false; |
| 404 | 366 |
| 405 const FilePath new_filename = TemporaryFileForFilename(filename_); | 367 const FilePath new_filename = TemporaryFileForFilename(filename_); |
| 406 file_util::ScopedFILE new_file(file_util::OpenFile(new_filename, "wb+")); | 368 file_util::ScopedFILE new_file(file_util::OpenFile(new_filename, "wb+")); |
| 407 if (new_file.get() == NULL) | 369 if (new_file.get() == NULL) |
| 408 return false; | 370 return false; |
| 409 | 371 |
| 410 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); | 372 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); |
| 411 empty_ = (file.get() == NULL); | 373 empty_ = (file.get() == NULL); |
| 412 if (empty_) { | 374 if (empty_) { |
| 413 // If the file exists but cannot be opened, try to delete it (not | 375 // If the file exists but cannot be opened, try to delete it (not |
| 414 // deleting directly, the bloom filter needs to be deleted, too). | 376 // deleting directly, the bloom filter needs to be deleted, too). |
| 415 if (file_util::PathExists(filename_)) | 377 if (file_util::PathExists(filename_)) |
| 416 return OnCorruptDatabase(); | 378 return OnCorruptDatabase(); |
| 417 | 379 |
| 418 new_file_.swap(new_file); | 380 new_file_.swap(new_file); |
| 419 return true; | 381 return true; |
| 420 } | 382 } |
| 421 | 383 |
| 422 FileHeader header; | 384 FileHeader header; |
| 423 if (!ReadArray(&header, 1, file.get(), NULL)) | 385 if (!ReadItem(&header, file.get(), NULL)) |
| 424 return OnCorruptDatabase(); | 386 return OnCorruptDatabase(); |
| 425 | 387 |
| 426 if (header.magic != kFileMagic || header.version != kFileVersion) { | 388 if (header.magic != kFileMagic || header.version != kFileVersion) { |
| 427 if (!strcmp(reinterpret_cast<char*>(&header.magic), "SQLite format 3")) { | 389 if (!strcmp(reinterpret_cast<char*>(&header.magic), "SQLite format 3")) { |
| 428 RecordFormatEvent(FORMAT_EVENT_FOUND_SQLITE); | 390 RecordFormatEvent(FORMAT_EVENT_FOUND_SQLITE); |
| 429 } else { | 391 } else { |
| 430 RecordFormatEvent(FORMAT_EVENT_FOUND_UNKNOWN); | 392 RecordFormatEvent(FORMAT_EVENT_FOUND_UNKNOWN); |
| 431 } | 393 } |
| 432 | 394 |
| 433 // Close the file so that it can be deleted. | 395 // Close the file so that it can be deleted. |
| 434 file.reset(); | 396 file.reset(); |
| 435 | 397 |
| 436 return OnCorruptDatabase(); | 398 return OnCorruptDatabase(); |
| 437 } | 399 } |
| 438 | 400 |
| 439 // TODO(shess): Under POSIX it is possible that this could size a | 401 // TODO(shess): Under POSIX it is possible that this could size a |
| 440 // file different from the file which was opened. | 402 // file different from the file which was opened. |
| 441 if (!FileHeaderSanityCheck(filename_, header)) | 403 if (!FileHeaderSanityCheck(filename_, header)) |
| 442 return OnCorruptDatabase(); | 404 return OnCorruptDatabase(); |
| 443 | 405 |
| 444 // Pull in the chunks-seen data for purposes of implementing | 406 // Pull in the chunks-seen data for purposes of implementing |
| 445 // |GetAddChunks()| and |GetSubChunks()|. This data is sent up to | 407 // |GetAddChunks()| and |GetSubChunks()|. This data is sent up to |
| 446 // the server at the beginning of an update. | 408 // the server at the beginning of an update. |
| 447 if (!ReadToChunkSet(&add_chunks_cache_, header.add_chunk_count, | 409 if (!ReadToContainer(&add_chunks_cache_, header.add_chunk_count, |
| 448 file.get(), NULL) || | 410 file.get(), NULL) || |
| 449 !ReadToChunkSet(&sub_chunks_cache_, header.sub_chunk_count, | 411 !ReadToContainer(&sub_chunks_cache_, header.sub_chunk_count, |
| 450 file.get(), NULL)) | 412 file.get(), NULL)) |
| 451 return OnCorruptDatabase(); | 413 return OnCorruptDatabase(); |
| 452 | 414 |
| 453 file_.swap(file); | 415 file_.swap(file); |
| 454 new_file_.swap(new_file); | 416 new_file_.swap(new_file); |
| 455 return true; | 417 return true; |
| 456 } | 418 } |
| 457 | 419 |
| 458 bool SafeBrowsingStoreFile::FinishChunk() { | 420 bool SafeBrowsingStoreFile::FinishChunk() { |
| 459 if (!add_prefixes_.size() && !sub_prefixes_.size() && | 421 if (!add_prefixes_.size() && !sub_prefixes_.size() && |
| 460 !add_hashes_.size() && !sub_hashes_.size()) | 422 !add_hashes_.size() && !sub_hashes_.size()) |
| 461 return true; | 423 return true; |
| 462 | 424 |
| 463 ChunkHeader header; | 425 ChunkHeader header; |
| 464 header.add_prefix_count = add_prefixes_.size(); | 426 header.add_prefix_count = add_prefixes_.size(); |
| 465 header.sub_prefix_count = sub_prefixes_.size(); | 427 header.sub_prefix_count = sub_prefixes_.size(); |
| 466 header.add_hash_count = add_hashes_.size(); | 428 header.add_hash_count = add_hashes_.size(); |
| 467 header.sub_hash_count = sub_hashes_.size(); | 429 header.sub_hash_count = sub_hashes_.size(); |
| 468 if (!WriteArray(&header, 1, new_file_.get(), NULL)) | 430 if (!WriteItem(header, new_file_.get(), NULL)) |
| 469 return false; | 431 return false; |
| 470 | 432 |
| 471 if (!WriteVector(add_prefixes_, new_file_.get(), NULL) || | 433 if (!WriteContainer(add_prefixes_, new_file_.get(), NULL) || |
| 472 !WriteVector(sub_prefixes_, new_file_.get(), NULL) || | 434 !WriteContainer(sub_prefixes_, new_file_.get(), NULL) || |
| 473 !WriteVector(add_hashes_, new_file_.get(), NULL) || | 435 !WriteContainer(add_hashes_, new_file_.get(), NULL) || |
| 474 !WriteVector(sub_hashes_, new_file_.get(), NULL)) | 436 !WriteContainer(sub_hashes_, new_file_.get(), NULL)) |
| 475 return false; | 437 return false; |
| 476 | 438 |
| 477 ++chunks_written_; | 439 ++chunks_written_; |
| 478 add_prefixes_added_ += add_prefixes_.size(); | |
| 479 | 440 |
| 480 // Clear everything to save memory. | 441 // Clear everything to save memory. |
| 481 return ClearChunkBuffers(); | 442 return ClearChunkBuffers(); |
| 482 } | 443 } |
| 483 | 444 |
| 484 bool SafeBrowsingStoreFile::DoUpdate( | 445 bool SafeBrowsingStoreFile::DoUpdate( |
| 485 const std::vector<SBAddFullHash>& pending_adds, | 446 const std::vector<SBAddFullHash>& pending_adds, |
| 486 const std::set<SBPrefix>& prefix_misses, | 447 const std::set<SBPrefix>& prefix_misses, |
| 487 std::vector<SBAddPrefix>* add_prefixes_result, | 448 SBAddPrefixContainer* add_prefixes_result, |
| 488 std::vector<SBAddFullHash>* add_full_hashes_result) { | 449 std::vector<SBAddFullHash>* add_full_hashes_result) { |
| 489 DCHECK(file_.get() || empty_); | 450 DCHECK(file_.get() || empty_); |
| 490 DCHECK(new_file_.get()); | 451 DCHECK(new_file_.get()); |
| 491 CHECK(add_prefixes_result); | 452 CHECK(add_prefixes_result); |
| 492 CHECK(add_full_hashes_result); | 453 CHECK(add_full_hashes_result); |
| 493 | 454 |
| 494 std::vector<SBAddPrefix> add_prefixes; | 455 SBAddPrefixContainer add_prefixes; |
| 495 std::vector<SBSubPrefix> sub_prefixes; | 456 std::vector<SBSubPrefix> sub_prefixes; |
| 496 std::vector<SBAddFullHash> add_full_hashes; | 457 std::vector<SBAddFullHash> add_full_hashes; |
| 497 std::vector<SBSubFullHash> sub_full_hashes; | 458 std::vector<SBSubFullHash> sub_full_hashes; |
| 498 | 459 |
| 499 // Read original data into the vectors. | 460 // Read original data into the vectors. |
| 500 if (!empty_) { | 461 if (!empty_) { |
| 501 DCHECK(file_.get()); | 462 DCHECK(file_.get()); |
| 502 | 463 |
| 503 if (!FileRewind(file_.get())) | 464 if (!FileRewind(file_.get())) |
| 504 return OnCorruptDatabase(); | 465 return OnCorruptDatabase(); |
| 505 | 466 |
| 506 base::MD5Context context; | 467 base::MD5Context context; |
| 507 base::MD5Init(&context); | 468 base::MD5Init(&context); |
| 508 | 469 |
| 509 // Read the file header and make sure it looks right. | 470 // Read the file header and make sure it looks right. |
| 510 FileHeader header; | 471 FileHeader header; |
| 511 if (!ReadAndVerifyHeader(filename_, file_.get(), &header, &context)) | 472 if (!ReadAndVerifyHeader(filename_, file_.get(), &header, &context)) |
| 512 return OnCorruptDatabase(); | 473 return OnCorruptDatabase(); |
| 513 | 474 |
| 514 // Re-read the chunks-seen data to get to the later data in the | 475 // Re-read the chunks-seen data to get to the later data in the |
| 515 // file and calculate the checksum. No new elements should be | 476 // file and calculate the checksum. No new elements should be |
| 516 // added to the sets. | 477 // added to the sets. |
| 517 if (!ReadToChunkSet(&add_chunks_cache_, header.add_chunk_count, | 478 if (!ReadToContainer(&add_chunks_cache_, header.add_chunk_count, |
| 518 file_.get(), &context) || | 479 file_.get(), &context) || |
| 519 !ReadToChunkSet(&sub_chunks_cache_, header.sub_chunk_count, | 480 !ReadToContainer(&sub_chunks_cache_, header.sub_chunk_count, |
| 520 file_.get(), &context)) | 481 file_.get(), &context)) |
| 521 return OnCorruptDatabase(); | 482 return OnCorruptDatabase(); |
| 522 | 483 |
| 523 add_prefixes.reserve(header.add_prefix_count + add_prefixes_added_); | 484 if (!ReadToContainer(&add_prefixes, header.add_prefix_count, |
| 524 | 485 file_.get(), &context) || |
| 525 if (!ReadToVector(&add_prefixes, header.add_prefix_count, | 486 !ReadToContainer(&sub_prefixes, header.sub_prefix_count, |
| 526 file_.get(), &context) || | 487 file_.get(), &context) || |
| 527 !ReadToVector(&sub_prefixes, header.sub_prefix_count, | 488 !ReadToContainer(&add_full_hashes, header.add_hash_count, |
| 528 file_.get(), &context) || | 489 file_.get(), &context) || |
| 529 !ReadToVector(&add_full_hashes, header.add_hash_count, | 490 !ReadToContainer(&sub_full_hashes, header.sub_hash_count, |
| 530 file_.get(), &context) || | 491 file_.get(), &context)) |
| 531 !ReadToVector(&sub_full_hashes, header.sub_hash_count, | |
| 532 file_.get(), &context)) | |
| 533 return OnCorruptDatabase(); | 492 return OnCorruptDatabase(); |
| 534 | 493 |
| 535 // Calculate the digest to this point. | 494 // Calculate the digest to this point. |
| 536 base::MD5Digest calculated_digest; | 495 base::MD5Digest calculated_digest; |
| 537 base::MD5Final(&calculated_digest, &context); | 496 base::MD5Final(&calculated_digest, &context); |
| 538 | 497 |
| 539 // Read the stored checksum and verify it. | 498 // Read the stored checksum and verify it. |
| 540 base::MD5Digest file_digest; | 499 base::MD5Digest file_digest; |
| 541 if (!ReadArray(&file_digest, 1, file_.get(), NULL)) | 500 if (!ReadItem(&file_digest, file_.get(), NULL)) |
| 542 return OnCorruptDatabase(); | 501 return OnCorruptDatabase(); |
| 543 | 502 |
| 544 if (0 != memcmp(&file_digest, &calculated_digest, sizeof(file_digest))) | 503 if (0 != memcmp(&file_digest, &calculated_digest, sizeof(file_digest))) |
| 545 return OnCorruptDatabase(); | 504 return OnCorruptDatabase(); |
| 546 | 505 |
| 547 // Close the file so we can later rename over it. | 506 // Close the file so we can later rename over it. |
| 548 file_.reset(); | 507 file_.reset(); |
| 549 } else { | |
| 550 add_prefixes.reserve(add_prefixes_added_); | |
|
jar (doing other things)
2011/10/21 19:10:45
What happened to this line?
FWIW: I had a heck of
Scott Hess - ex-Googler
2011/10/21 23:39:32
The reserve() was my earlier change to have the ve
| |
| 551 } | 508 } |
| 552 DCHECK(!file_.get()); | 509 DCHECK(!file_.get()); |
| 553 | 510 |
| 554 // Rewind the temporary storage. | 511 // Rewind the temporary storage. |
| 555 if (!FileRewind(new_file_.get())) | 512 if (!FileRewind(new_file_.get())) |
| 556 return false; | 513 return false; |
| 557 | 514 |
| 558 // Get chunk file's size for validating counts. | 515 // Get chunk file's size for validating counts. |
| 559 int64 size = 0; | 516 int64 size = 0; |
| 560 if (!file_util::GetFileSize(TemporaryFileForFilename(filename_), &size)) | 517 if (!file_util::GetFileSize(TemporaryFileForFilename(filename_), &size)) |
| 561 return OnCorruptDatabase(); | 518 return OnCorruptDatabase(); |
| 562 | 519 |
| 563 // Track update size to answer questions at http://crbug.com/72216 . | 520 // Track update size to answer questions at http://crbug.com/72216 . |
| 564 // Log small updates as 1k so that the 0 (underflow) bucket can be | 521 // Log small updates as 1k so that the 0 (underflow) bucket can be |
| 565 // used for "empty" in SafeBrowsingDatabase. | 522 // used for "empty" in SafeBrowsingDatabase. |
| 566 UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", | 523 UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", |
| 567 std::max(static_cast<int>(size / 1024), 1)); | 524 std::max(static_cast<int>(size / 1024), 1)); |
| 568 | 525 |
| 569 // TODO(shess): For SB2.AddPrefixesReallocs histogram. | |
| 570 int add_prefixes_reallocs = 0; | |
| 571 | |
| 572 // Append the accumulated chunks onto the vectors read from |file_|. | 526 // Append the accumulated chunks onto the vectors read from |file_|. |
| 573 for (int i = 0; i < chunks_written_; ++i) { | 527 for (int i = 0; i < chunks_written_; ++i) { |
| 574 ChunkHeader header; | 528 ChunkHeader header; |
| 575 | 529 |
| 576 int64 ofs = ftell(new_file_.get()); | 530 int64 ofs = ftell(new_file_.get()); |
| 577 if (ofs == -1) | 531 if (ofs == -1) |
| 578 return false; | 532 return false; |
| 579 | 533 |
| 580 if (!ReadArray(&header, 1, new_file_.get(), NULL)) | 534 if (!ReadItem(&header, new_file_.get(), NULL)) |
| 581 return false; | 535 return false; |
| 582 | 536 |
| 583 // As a safety measure, make sure that the header describes a sane | 537 // As a safety measure, make sure that the header describes a sane |
| 584 // chunk, given the remaining file size. | 538 // chunk, given the remaining file size. |
| 585 int64 expected_size = ofs + sizeof(ChunkHeader); | 539 int64 expected_size = ofs + sizeof(ChunkHeader); |
| 586 expected_size += header.add_prefix_count * sizeof(SBAddPrefix); | 540 expected_size += header.add_prefix_count * sizeof(SBAddPrefix); |
| 587 expected_size += header.sub_prefix_count * sizeof(SBSubPrefix); | 541 expected_size += header.sub_prefix_count * sizeof(SBSubPrefix); |
| 588 expected_size += header.add_hash_count * sizeof(SBAddFullHash); | 542 expected_size += header.add_hash_count * sizeof(SBAddFullHash); |
| 589 expected_size += header.sub_hash_count * sizeof(SBSubFullHash); | 543 expected_size += header.sub_hash_count * sizeof(SBSubFullHash); |
| 590 if (expected_size > size) | 544 if (expected_size > size) |
| 591 return false; | 545 return false; |
| 592 | 546 |
| 593 // TODO(shess): For SB2.AddPrefixesReallocs histogram. | |
| 594 SBAddPrefix* add_prefixes_old_start = | |
| 595 (add_prefixes.size() ? &add_prefixes[0] : NULL); | |
| 596 | |
| 597 // TODO(shess): If the vectors were kept sorted, then this code | 547 // TODO(shess): If the vectors were kept sorted, then this code |
| 598 // could use std::inplace_merge() to merge everything together in | 548 // could use std::inplace_merge() to merge everything together in |
| 599 // sorted order. That might still be slower than just sorting at | 549 // sorted order. That might still be slower than just sorting at |
| 600 // the end if there were a large number of chunks. In that case | 550 // the end if there were a large number of chunks. In that case |
| 601 // some sort of recursive binary merge might be in order (merge | 551 // some sort of recursive binary merge might be in order (merge |
| 602 // chunks pairwise, merge those chunks pairwise, and so on, then | 552 // chunks pairwise, merge those chunks pairwise, and so on, then |
| 603 // merge the result with the main list). | 553 // merge the result with the main list). |
| 604 if (!ReadToVector(&add_prefixes, header.add_prefix_count, | 554 if (!ReadToContainer(&add_prefixes, header.add_prefix_count, |
| 605 new_file_.get(), NULL) || | 555 new_file_.get(), NULL) || |
| 606 !ReadToVector(&sub_prefixes, header.sub_prefix_count, | 556 !ReadToContainer(&sub_prefixes, header.sub_prefix_count, |
| 607 new_file_.get(), NULL) || | 557 new_file_.get(), NULL) || |
| 608 !ReadToVector(&add_full_hashes, header.add_hash_count, | 558 !ReadToContainer(&add_full_hashes, header.add_hash_count, |
| 609 new_file_.get(), NULL) || | 559 new_file_.get(), NULL) || |
| 610 !ReadToVector(&sub_full_hashes, header.sub_hash_count, | 560 !ReadToContainer(&sub_full_hashes, header.sub_hash_count, |
| 611 new_file_.get(), NULL)) | 561 new_file_.get(), NULL)) |
| 612 return false; | 562 return false; |
| 613 | |
| 614 // Determine if the vector data moved to a new location. This | |
| 615 // won't track cases where the malloc library was able to expand | |
| 616 // the storage in place, but those cases shouldn't cause memory | |
| 617 // fragmentation anyhow. | |
| 618 SBAddPrefix* add_prefixes_new_start = | |
| 619 (add_prefixes.size() ? &add_prefixes[0] : NULL); | |
| 620 if (add_prefixes_old_start != add_prefixes_new_start) | |
| 621 ++add_prefixes_reallocs; | |
| 622 } | 563 } |
| 623 | 564 |
| 624 // Track the number of times this number of re-allocations occurred. | |
| 625 UMA_HISTOGRAM_COUNTS_100("SB2.AddPrefixesReallocs", add_prefixes_reallocs); | |
| 626 | |
| 627 // Append items from |pending_adds|. | 565 // Append items from |pending_adds|. |
| 628 add_full_hashes.insert(add_full_hashes.end(), | 566 add_full_hashes.insert(add_full_hashes.end(), |
| 629 pending_adds.begin(), pending_adds.end()); | 567 pending_adds.begin(), pending_adds.end()); |
| 630 | 568 |
| 631 // Check how often a prefix was checked which wasn't in the | 569 // Check how often a prefix was checked which wasn't in the |
| 632 // database. | 570 // database. |
| 633 SBCheckPrefixMisses(add_prefixes, prefix_misses); | 571 SBCheckPrefixMisses(add_prefixes, prefix_misses); |
| 634 | 572 |
| 635 // Knock the subs from the adds and process deleted chunks. | 573 // Knock the subs from the adds and process deleted chunks. |
| 636 SBProcessSubs(&add_prefixes, &sub_prefixes, | 574 SBProcessSubs(&add_prefixes, &sub_prefixes, |
| (...skipping 14 matching lines...) Expand all Loading... | |
| 651 // Write a file header. | 589 // Write a file header. |
| 652 FileHeader header; | 590 FileHeader header; |
| 653 header.magic = kFileMagic; | 591 header.magic = kFileMagic; |
| 654 header.version = kFileVersion; | 592 header.version = kFileVersion; |
| 655 header.add_chunk_count = add_chunks_cache_.size(); | 593 header.add_chunk_count = add_chunks_cache_.size(); |
| 656 header.sub_chunk_count = sub_chunks_cache_.size(); | 594 header.sub_chunk_count = sub_chunks_cache_.size(); |
| 657 header.add_prefix_count = add_prefixes.size(); | 595 header.add_prefix_count = add_prefixes.size(); |
| 658 header.sub_prefix_count = sub_prefixes.size(); | 596 header.sub_prefix_count = sub_prefixes.size(); |
| 659 header.add_hash_count = add_full_hashes.size(); | 597 header.add_hash_count = add_full_hashes.size(); |
| 660 header.sub_hash_count = sub_full_hashes.size(); | 598 header.sub_hash_count = sub_full_hashes.size(); |
| 661 if (!WriteArray(&header, 1, new_file_.get(), &context)) | 599 if (!WriteItem(header, new_file_.get(), &context)) |
| 662 return false; | 600 return false; |
| 663 | 601 |
| 664 // Write all the chunk data. | 602 // Write all the chunk data. |
| 665 if (!WriteChunkSet(add_chunks_cache_, new_file_.get(), &context) || | 603 if (!WriteContainer(add_chunks_cache_, new_file_.get(), &context) || |
| 666 !WriteChunkSet(sub_chunks_cache_, new_file_.get(), &context) || | 604 !WriteContainer(sub_chunks_cache_, new_file_.get(), &context) || |
| 667 !WriteVector(add_prefixes, new_file_.get(), &context) || | 605 !WriteContainer(add_prefixes, new_file_.get(), &context) || |
| 668 !WriteVector(sub_prefixes, new_file_.get(), &context) || | 606 !WriteContainer(sub_prefixes, new_file_.get(), &context) || |
| 669 !WriteVector(add_full_hashes, new_file_.get(), &context) || | 607 !WriteContainer(add_full_hashes, new_file_.get(), &context) || |
| 670 !WriteVector(sub_full_hashes, new_file_.get(), &context)) | 608 !WriteContainer(sub_full_hashes, new_file_.get(), &context)) |
| 671 return false; | 609 return false; |
| 672 | 610 |
| 673 // Write the checksum at the end. | 611 // Write the checksum at the end. |
| 674 base::MD5Digest digest; | 612 base::MD5Digest digest; |
| 675 base::MD5Final(&digest, &context); | 613 base::MD5Final(&digest, &context); |
| 676 if (!WriteArray(&digest, 1, new_file_.get(), NULL)) | 614 if (!WriteItem(digest, new_file_.get(), NULL)) |
| 677 return false; | 615 return false; |
| 678 | 616 |
| 679 // Trim any excess left over from the temporary chunk data. | 617 // Trim any excess left over from the temporary chunk data. |
| 680 if (!file_util::TruncateFile(new_file_.get())) | 618 if (!file_util::TruncateFile(new_file_.get())) |
| 681 return false; | 619 return false; |
| 682 | 620 |
| 683 // Close the file handle and swizzle the file into place. | 621 // Close the file handle and swizzle the file into place. |
| 684 new_file_.reset(); | 622 new_file_.reset(); |
| 685 if (!file_util::Delete(filename_, false) && | 623 if (!file_util::Delete(filename_, false) && |
| 686 file_util::PathExists(filename_)) | 624 file_util::PathExists(filename_)) |
| (...skipping 10 matching lines...) Expand all Loading... | |
| 697 // Pass the resulting data off to the caller. | 635 // Pass the resulting data off to the caller. |
| 698 add_prefixes_result->swap(add_prefixes); | 636 add_prefixes_result->swap(add_prefixes); |
| 699 add_full_hashes_result->swap(add_full_hashes); | 637 add_full_hashes_result->swap(add_full_hashes); |
| 700 | 638 |
| 701 return true; | 639 return true; |
| 702 } | 640 } |
| 703 | 641 |
| 704 bool SafeBrowsingStoreFile::FinishUpdate( | 642 bool SafeBrowsingStoreFile::FinishUpdate( |
| 705 const std::vector<SBAddFullHash>& pending_adds, | 643 const std::vector<SBAddFullHash>& pending_adds, |
| 706 const std::set<SBPrefix>& prefix_misses, | 644 const std::set<SBPrefix>& prefix_misses, |
| 707 std::vector<SBAddPrefix>* add_prefixes_result, | 645 SBAddPrefixContainer* add_prefixes_result, |
| 708 std::vector<SBAddFullHash>* add_full_hashes_result) { | 646 std::vector<SBAddFullHash>* add_full_hashes_result) { |
| 709 DCHECK(add_prefixes_result); | 647 DCHECK(add_prefixes_result); |
| 710 DCHECK(add_full_hashes_result); | 648 DCHECK(add_full_hashes_result); |
| 711 | 649 |
| 712 bool ret = DoUpdate(pending_adds, prefix_misses, | 650 bool ret = DoUpdate(pending_adds, prefix_misses, |
| 713 add_prefixes_result, add_full_hashes_result); | 651 add_prefixes_result, add_full_hashes_result); |
| 714 | 652 |
| 715 if (!ret) { | 653 if (!ret) { |
| 716 CancelUpdate(); | 654 CancelUpdate(); |
| 717 return false; | 655 return false; |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 753 out->insert(out->end(), sub_chunks_cache_.begin(), sub_chunks_cache_.end()); | 691 out->insert(out->end(), sub_chunks_cache_.begin(), sub_chunks_cache_.end()); |
| 754 } | 692 } |
| 755 | 693 |
| 756 void SafeBrowsingStoreFile::DeleteAddChunk(int32 chunk_id) { | 694 void SafeBrowsingStoreFile::DeleteAddChunk(int32 chunk_id) { |
| 757 add_del_cache_.insert(chunk_id); | 695 add_del_cache_.insert(chunk_id); |
| 758 } | 696 } |
| 759 | 697 |
| 760 void SafeBrowsingStoreFile::DeleteSubChunk(int32 chunk_id) { | 698 void SafeBrowsingStoreFile::DeleteSubChunk(int32 chunk_id) { |
| 761 sub_del_cache_.insert(chunk_id); | 699 sub_del_cache_.insert(chunk_id); |
| 762 } | 700 } |
| OLD | NEW |