Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(258)

Side by Side Diff: chrome/browser/safe_browsing/safe_browsing_store_file.cc

Issue 8349018: Safe-browsing SBAddPrefix from vector to deque. (Closed) Base URL: http://git.chromium.org/chromium/src.git@master
Patch Set: Created 9 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. 1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h" 5 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
6 6
7 #include "base/md5.h" 7 #include "base/md5.h"
8 #include "base/metrics/histogram.h" 8 #include "base/metrics/histogram.h"
9 9
10 namespace { 10 namespace {
(...skipping 30 matching lines...) Expand all
41 // Although fseek takes negative values, for this case, we only want 41 // Although fseek takes negative values, for this case, we only want
42 // to skip forward. 42 // to skip forward.
43 DCHECK(static_cast<long>(bytes) >= 0); 43 DCHECK(static_cast<long>(bytes) >= 0);
44 if (static_cast<long>(bytes) < 0) 44 if (static_cast<long>(bytes) < 0)
45 return false; 45 return false;
46 int rv = fseek(fp, static_cast<long>(bytes), SEEK_CUR); 46 int rv = fseek(fp, static_cast<long>(bytes), SEEK_CUR);
47 DCHECK_EQ(rv, 0); 47 DCHECK_EQ(rv, 0);
48 return rv == 0; 48 return rv == 0;
49 } 49 }
50 50
51 // Read an array of |nmemb| items from |fp| into |ptr|, and fold the 51 // Read from |fp| into |item|, and fold the input data into the
52 // input data into the checksum in |context|, if non-NULL. Return 52 // checksum in |context|, if non-NULL. Return true on success.
53 // true on success.
54 template <class T> 53 template <class T>
55 bool ReadArray(T* ptr, size_t nmemb, FILE* fp, base::MD5Context* context) { 54 bool ReadItem(T* item, FILE* fp, base::MD5Context* context) {
56 const size_t ret = fread(ptr, sizeof(T), nmemb, fp); 55 const size_t ret = fread(item, sizeof(T), 1, fp);
57 if (ret != nmemb) 56 if (ret != 1)
58 return false; 57 return false;
59 58
60 if (context) { 59 if (context) {
61 base::MD5Update(context, 60 base::MD5Update(context,
62 base::StringPiece(reinterpret_cast<char*>(ptr), 61 base::StringPiece(reinterpret_cast<char*>(item),
63 sizeof(T) * nmemb)); 62 sizeof(T)));
64 } 63 }
65 return true; 64 return true;
66 } 65 }
67 66
68 // Write an array of |nmemb| items from |ptr| to |fp|, and fold the 67 // Write |item| to |fp|, and fold the output data into the checksum in
69 // output data into the checksum in |context|, if non-NULL. Return 68 // |context|, if non-NULL. Return true on success.
70 // true on success.
71 template <class T> 69 template <class T>
72 bool WriteArray(const T* ptr, size_t nmemb, FILE* fp, 70 bool WriteItem(const T& item, FILE* fp, base::MD5Context* context) {
73 base::MD5Context* context) { 71 const size_t ret = fwrite(&item, sizeof(T), 1, fp);
74 const size_t ret = fwrite(ptr, sizeof(T), nmemb, fp); 72 if (ret != 1)
75 if (ret != nmemb)
76 return false; 73 return false;
77 74
78 if (context) { 75 if (context) {
79 base::MD5Update(context, 76 base::MD5Update(context,
80 base::StringPiece(reinterpret_cast<const char*>(ptr), 77 base::StringPiece(reinterpret_cast<const char*>(&item),
81 sizeof(T) * nmemb)); 78 sizeof(T)));
82 } 79 }
83 80
84 return true; 81 return true;
85 } 82 }
86 83
87 // Expand |values| to fit |count| new items, read those items from 84 // Read |count| items into |values| from |fp|, and fold them into the
88 // |fp| and fold them into the checksum in |context|. Returns true on 85 // checksum in |context|. Returns true on success.
89 // success. 86 template <typename CT>
90 template <class T> 87 bool ReadToContainer(CT* values, size_t count, FILE* fp,
91 bool ReadToVector(std::vector<T>* values, size_t count, FILE* fp, 88 base::MD5Context* context) {
92 base::MD5Context* context) {
93 // Pointers into an empty vector may not be valid.
94 if (!count) 89 if (!count)
95 return true; 90 return true;
96 91
97 // Grab the size for purposes of finding where to read to. The 92 for (size_t i = 0; i < count; ++i) {
98 // resize could invalidate any iterator captured here. 93 typename CT::value_type value;
99 const size_t original_size = values->size(); 94 if (!ReadItem(&value, fp, context))
100 values->resize(original_size + count); 95 return false;
101 96 values->insert(values->end(), value);
jar (doing other things) 2011/10/21 19:10:45 It might be clearer to: values->push_back(value);
Scott Hess - ex-Googler 2011/10/21 23:39:32 That is the magic which allows the function to wor
102 // Sayeth Herb Sutter: Vectors are guaranteed to be contiguous. So
103 // get a pointer to where to read the data to.
104 T* ptr = &((*values)[original_size]);
105 if (!ReadArray(ptr, count, fp, context)) {
106 values->resize(original_size);
107 return false;
108 } 97 }
109 98
110 return true; 99 return true;
111 } 100 }
112 101
113 // Write all of |values| to |fp|, and fold the data into the checksum 102 // Write all of |values| to |fp|, and fold the data into the checksum
114 // in |context|, if non-NULL. Returns true on succsess. 103 // in |context|, if non-NULL. Returns true on succsess.
115 template <class T> 104 template <typename CT>
116 bool WriteVector(const std::vector<T>& values, FILE* fp, 105 bool WriteContainer(const CT& values, FILE* fp,
117 base::MD5Context* context) { 106 base::MD5Context* context) {
118 // Pointers into empty vectors may not be valid.
119 if (values.empty()) 107 if (values.empty())
120 return true; 108 return true;
121 109
122 // Sayeth Herb Sutter: Vectors are guaranteed to be contiguous. So 110 for (typename CT::const_iterator iter = values.begin();
123 // get a pointer to where to write from. 111 iter != values.end(); ++iter) {
124 const T* ptr = &(values[0]); 112 if (!WriteItem(*iter, fp, context))
125 return WriteArray(ptr, values.size(), fp, context); 113 return false;
126 } 114 }
127
128 // Read an array of |count| integers and add them to |values|.
129 // Returns true on success.
130 bool ReadToChunkSet(std::set<int32>* values, size_t count, FILE* fp,
131 base::MD5Context* context) {
132 if (!count)
133 return true;
134
135 std::vector<int32> flat_values;
136 if (!ReadToVector(&flat_values, count, fp, context))
137 return false;
138
139 values->insert(flat_values.begin(), flat_values.end());
140 return true; 115 return true;
141 } 116 }
142 117
143 // Write the contents of |values| as an array of integers. Returns
144 // true on success.
145 bool WriteChunkSet(const std::set<int32>& values, FILE* fp,
146 base::MD5Context* context) {
147 if (values.empty())
148 return true;
149
150 const std::vector<int32> flat_values(values.begin(), values.end());
151 return WriteVector(flat_values, fp, context);
152 }
153
154 // Delete the chunks in |deleted| from |chunks|. 118 // Delete the chunks in |deleted| from |chunks|.
155 void DeleteChunksFromSet(const base::hash_set<int32>& deleted, 119 void DeleteChunksFromSet(const base::hash_set<int32>& deleted,
156 std::set<int32>* chunks) { 120 std::set<int32>* chunks) {
157 for (std::set<int32>::iterator iter = chunks->begin(); 121 for (std::set<int32>::iterator iter = chunks->begin();
158 iter != chunks->end();) { 122 iter != chunks->end();) {
159 std::set<int32>::iterator prev = iter++; 123 std::set<int32>::iterator prev = iter++;
160 if (deleted.count(*prev) > 0) 124 if (deleted.count(*prev) > 0)
161 chunks->erase(prev); 125 chunks->erase(prev);
162 } 126 }
163 } 127 }
(...skipping 20 matching lines...) Expand all
184 148
185 return true; 149 return true;
186 } 150 }
187 151
188 // This a helper function that reads header to |header|. Returns true if the 152 // This a helper function that reads header to |header|. Returns true if the
189 // magic number is correct and santiy check passes. 153 // magic number is correct and santiy check passes.
190 bool ReadAndVerifyHeader(const FilePath& filename, 154 bool ReadAndVerifyHeader(const FilePath& filename,
191 FILE* fp, 155 FILE* fp,
192 FileHeader* header, 156 FileHeader* header,
193 base::MD5Context* context) { 157 base::MD5Context* context) {
194 if (!ReadArray(header, 1, fp, context)) 158 if (!ReadItem(header, fp, context))
195 return false; 159 return false;
196 if (header->magic != kFileMagic || header->version != kFileVersion) 160 if (header->magic != kFileMagic || header->version != kFileVersion)
197 return false; 161 return false;
198 if (!FileHeaderSanityCheck(filename, *header)) 162 if (!FileHeaderSanityCheck(filename, *header))
199 return false; 163 return false;
200 return true; 164 return true;
201 } 165 }
202 166
203 } // namespace 167 } // namespace
204 168
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after
286 250
287 bool SafeBrowsingStoreFile::BeginChunk() { 251 bool SafeBrowsingStoreFile::BeginChunk() {
288 return ClearChunkBuffers(); 252 return ClearChunkBuffers();
289 } 253 }
290 254
291 bool SafeBrowsingStoreFile::WriteAddPrefix(int32 chunk_id, SBPrefix prefix) { 255 bool SafeBrowsingStoreFile::WriteAddPrefix(int32 chunk_id, SBPrefix prefix) {
292 add_prefixes_.push_back(SBAddPrefix(chunk_id, prefix)); 256 add_prefixes_.push_back(SBAddPrefix(chunk_id, prefix));
293 return true; 257 return true;
294 } 258 }
295 259
296 bool SafeBrowsingStoreFile::GetAddPrefixes( 260 bool SafeBrowsingStoreFile::GetAddPrefixes(SBAddPrefixContainer* add_prefixes) {
297 std::vector<SBAddPrefix>* add_prefixes) {
298 add_prefixes->clear(); 261 add_prefixes->clear();
299 262
300 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); 263 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb"));
301 if (file.get() == NULL) return false; 264 if (file.get() == NULL) return false;
302 265
303 FileHeader header; 266 FileHeader header;
304 if (!ReadAndVerifyHeader(filename_, file.get(), &header, NULL)) 267 if (!ReadAndVerifyHeader(filename_, file.get(), &header, NULL))
305 return OnCorruptDatabase(); 268 return OnCorruptDatabase();
306 269
307 size_t add_prefix_offset = header.add_chunk_count * sizeof(int32) + 270 size_t add_prefix_offset = header.add_chunk_count * sizeof(int32) +
308 header.sub_chunk_count * sizeof(int32); 271 header.sub_chunk_count * sizeof(int32);
309 if (!FileSkip(add_prefix_offset, file.get())) 272 if (!FileSkip(add_prefix_offset, file.get()))
310 return false; 273 return false;
311 274
312 if (!ReadToVector(add_prefixes, header.add_prefix_count, file.get(), NULL)) 275 if (!ReadToContainer(add_prefixes, header.add_prefix_count, file.get(), NULL))
313 return false; 276 return false;
314 277
315 return true; 278 return true;
316 } 279 }
317 280
318 bool SafeBrowsingStoreFile::GetAddFullHashes( 281 bool SafeBrowsingStoreFile::GetAddFullHashes(
319 std::vector<SBAddFullHash>* add_full_hashes) { 282 std::vector<SBAddFullHash>* add_full_hashes) {
320 add_full_hashes->clear(); 283 add_full_hashes->clear();
321 284
322 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); 285 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb"));
323 if (file.get() == NULL) return false; 286 if (file.get() == NULL) return false;
324 287
325 FileHeader header; 288 FileHeader header;
326 if (!ReadAndVerifyHeader(filename_, file.get(), &header, NULL)) 289 if (!ReadAndVerifyHeader(filename_, file.get(), &header, NULL))
327 return OnCorruptDatabase(); 290 return OnCorruptDatabase();
328 291
329 size_t offset = 292 size_t offset =
330 header.add_chunk_count * sizeof(int32) + 293 header.add_chunk_count * sizeof(int32) +
331 header.sub_chunk_count * sizeof(int32) + 294 header.sub_chunk_count * sizeof(int32) +
332 header.add_prefix_count * sizeof(SBAddPrefix) + 295 header.add_prefix_count * sizeof(SBAddPrefix) +
333 header.sub_prefix_count * sizeof(SBSubPrefix); 296 header.sub_prefix_count * sizeof(SBSubPrefix);
334 if (!FileSkip(offset, file.get())) 297 if (!FileSkip(offset, file.get()))
335 return false; 298 return false;
336 299
337 return ReadToVector(add_full_hashes, 300 return ReadToContainer(add_full_hashes,
338 header.add_hash_count, 301 header.add_hash_count,
339 file.get(), 302 file.get(),
340 NULL); 303 NULL);
341 } 304 }
342 305
343 bool SafeBrowsingStoreFile::WriteAddHash(int32 chunk_id, 306 bool SafeBrowsingStoreFile::WriteAddHash(int32 chunk_id,
344 base::Time receive_time, 307 base::Time receive_time,
345 const SBFullHash& full_hash) { 308 const SBFullHash& full_hash) {
346 add_hashes_.push_back(SBAddFullHash(chunk_id, receive_time, full_hash)); 309 add_hashes_.push_back(SBAddFullHash(chunk_id, receive_time, full_hash));
347 return true; 310 return true;
348 } 311 }
349 312
350 bool SafeBrowsingStoreFile::WriteSubPrefix(int32 chunk_id, 313 bool SafeBrowsingStoreFile::WriteSubPrefix(int32 chunk_id,
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
386 // Structures should all be clear unless something bad happened. 349 // Structures should all be clear unless something bad happened.
387 DCHECK(add_chunks_cache_.empty()); 350 DCHECK(add_chunks_cache_.empty());
388 DCHECK(sub_chunks_cache_.empty()); 351 DCHECK(sub_chunks_cache_.empty());
389 DCHECK(add_del_cache_.empty()); 352 DCHECK(add_del_cache_.empty());
390 DCHECK(sub_del_cache_.empty()); 353 DCHECK(sub_del_cache_.empty());
391 DCHECK(add_prefixes_.empty()); 354 DCHECK(add_prefixes_.empty());
392 DCHECK(sub_prefixes_.empty()); 355 DCHECK(sub_prefixes_.empty());
393 DCHECK(add_hashes_.empty()); 356 DCHECK(add_hashes_.empty());
394 DCHECK(sub_hashes_.empty()); 357 DCHECK(sub_hashes_.empty());
395 DCHECK_EQ(chunks_written_, 0); 358 DCHECK_EQ(chunks_written_, 0);
396 add_prefixes_added_ = 0;
397 359
398 // Since the following code will already hit the profile looking for 360 // Since the following code will already hit the profile looking for
399 // database files, this is a reasonable to time delete any old 361 // database files, this is a reasonable to time delete any old
400 // files. 362 // files.
401 CheckForOriginalAndDelete(filename_); 363 CheckForOriginalAndDelete(filename_);
402 364
403 corruption_seen_ = false; 365 corruption_seen_ = false;
404 366
405 const FilePath new_filename = TemporaryFileForFilename(filename_); 367 const FilePath new_filename = TemporaryFileForFilename(filename_);
406 file_util::ScopedFILE new_file(file_util::OpenFile(new_filename, "wb+")); 368 file_util::ScopedFILE new_file(file_util::OpenFile(new_filename, "wb+"));
407 if (new_file.get() == NULL) 369 if (new_file.get() == NULL)
408 return false; 370 return false;
409 371
410 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb")); 372 file_util::ScopedFILE file(file_util::OpenFile(filename_, "rb"));
411 empty_ = (file.get() == NULL); 373 empty_ = (file.get() == NULL);
412 if (empty_) { 374 if (empty_) {
413 // If the file exists but cannot be opened, try to delete it (not 375 // If the file exists but cannot be opened, try to delete it (not
414 // deleting directly, the bloom filter needs to be deleted, too). 376 // deleting directly, the bloom filter needs to be deleted, too).
415 if (file_util::PathExists(filename_)) 377 if (file_util::PathExists(filename_))
416 return OnCorruptDatabase(); 378 return OnCorruptDatabase();
417 379
418 new_file_.swap(new_file); 380 new_file_.swap(new_file);
419 return true; 381 return true;
420 } 382 }
421 383
422 FileHeader header; 384 FileHeader header;
423 if (!ReadArray(&header, 1, file.get(), NULL)) 385 if (!ReadItem(&header, file.get(), NULL))
424 return OnCorruptDatabase(); 386 return OnCorruptDatabase();
425 387
426 if (header.magic != kFileMagic || header.version != kFileVersion) { 388 if (header.magic != kFileMagic || header.version != kFileVersion) {
427 if (!strcmp(reinterpret_cast<char*>(&header.magic), "SQLite format 3")) { 389 if (!strcmp(reinterpret_cast<char*>(&header.magic), "SQLite format 3")) {
428 RecordFormatEvent(FORMAT_EVENT_FOUND_SQLITE); 390 RecordFormatEvent(FORMAT_EVENT_FOUND_SQLITE);
429 } else { 391 } else {
430 RecordFormatEvent(FORMAT_EVENT_FOUND_UNKNOWN); 392 RecordFormatEvent(FORMAT_EVENT_FOUND_UNKNOWN);
431 } 393 }
432 394
433 // Close the file so that it can be deleted. 395 // Close the file so that it can be deleted.
434 file.reset(); 396 file.reset();
435 397
436 return OnCorruptDatabase(); 398 return OnCorruptDatabase();
437 } 399 }
438 400
439 // TODO(shess): Under POSIX it is possible that this could size a 401 // TODO(shess): Under POSIX it is possible that this could size a
440 // file different from the file which was opened. 402 // file different from the file which was opened.
441 if (!FileHeaderSanityCheck(filename_, header)) 403 if (!FileHeaderSanityCheck(filename_, header))
442 return OnCorruptDatabase(); 404 return OnCorruptDatabase();
443 405
444 // Pull in the chunks-seen data for purposes of implementing 406 // Pull in the chunks-seen data for purposes of implementing
445 // |GetAddChunks()| and |GetSubChunks()|. This data is sent up to 407 // |GetAddChunks()| and |GetSubChunks()|. This data is sent up to
446 // the server at the beginning of an update. 408 // the server at the beginning of an update.
447 if (!ReadToChunkSet(&add_chunks_cache_, header.add_chunk_count, 409 if (!ReadToContainer(&add_chunks_cache_, header.add_chunk_count,
448 file.get(), NULL) || 410 file.get(), NULL) ||
449 !ReadToChunkSet(&sub_chunks_cache_, header.sub_chunk_count, 411 !ReadToContainer(&sub_chunks_cache_, header.sub_chunk_count,
450 file.get(), NULL)) 412 file.get(), NULL))
451 return OnCorruptDatabase(); 413 return OnCorruptDatabase();
452 414
453 file_.swap(file); 415 file_.swap(file);
454 new_file_.swap(new_file); 416 new_file_.swap(new_file);
455 return true; 417 return true;
456 } 418 }
457 419
458 bool SafeBrowsingStoreFile::FinishChunk() { 420 bool SafeBrowsingStoreFile::FinishChunk() {
459 if (!add_prefixes_.size() && !sub_prefixes_.size() && 421 if (!add_prefixes_.size() && !sub_prefixes_.size() &&
460 !add_hashes_.size() && !sub_hashes_.size()) 422 !add_hashes_.size() && !sub_hashes_.size())
461 return true; 423 return true;
462 424
463 ChunkHeader header; 425 ChunkHeader header;
464 header.add_prefix_count = add_prefixes_.size(); 426 header.add_prefix_count = add_prefixes_.size();
465 header.sub_prefix_count = sub_prefixes_.size(); 427 header.sub_prefix_count = sub_prefixes_.size();
466 header.add_hash_count = add_hashes_.size(); 428 header.add_hash_count = add_hashes_.size();
467 header.sub_hash_count = sub_hashes_.size(); 429 header.sub_hash_count = sub_hashes_.size();
468 if (!WriteArray(&header, 1, new_file_.get(), NULL)) 430 if (!WriteItem(header, new_file_.get(), NULL))
469 return false; 431 return false;
470 432
471 if (!WriteVector(add_prefixes_, new_file_.get(), NULL) || 433 if (!WriteContainer(add_prefixes_, new_file_.get(), NULL) ||
472 !WriteVector(sub_prefixes_, new_file_.get(), NULL) || 434 !WriteContainer(sub_prefixes_, new_file_.get(), NULL) ||
473 !WriteVector(add_hashes_, new_file_.get(), NULL) || 435 !WriteContainer(add_hashes_, new_file_.get(), NULL) ||
474 !WriteVector(sub_hashes_, new_file_.get(), NULL)) 436 !WriteContainer(sub_hashes_, new_file_.get(), NULL))
475 return false; 437 return false;
476 438
477 ++chunks_written_; 439 ++chunks_written_;
478 add_prefixes_added_ += add_prefixes_.size();
479 440
480 // Clear everything to save memory. 441 // Clear everything to save memory.
481 return ClearChunkBuffers(); 442 return ClearChunkBuffers();
482 } 443 }
483 444
484 bool SafeBrowsingStoreFile::DoUpdate( 445 bool SafeBrowsingStoreFile::DoUpdate(
485 const std::vector<SBAddFullHash>& pending_adds, 446 const std::vector<SBAddFullHash>& pending_adds,
486 const std::set<SBPrefix>& prefix_misses, 447 const std::set<SBPrefix>& prefix_misses,
487 std::vector<SBAddPrefix>* add_prefixes_result, 448 SBAddPrefixContainer* add_prefixes_result,
488 std::vector<SBAddFullHash>* add_full_hashes_result) { 449 std::vector<SBAddFullHash>* add_full_hashes_result) {
489 DCHECK(file_.get() || empty_); 450 DCHECK(file_.get() || empty_);
490 DCHECK(new_file_.get()); 451 DCHECK(new_file_.get());
491 CHECK(add_prefixes_result); 452 CHECK(add_prefixes_result);
492 CHECK(add_full_hashes_result); 453 CHECK(add_full_hashes_result);
493 454
494 std::vector<SBAddPrefix> add_prefixes; 455 SBAddPrefixContainer add_prefixes;
495 std::vector<SBSubPrefix> sub_prefixes; 456 std::vector<SBSubPrefix> sub_prefixes;
496 std::vector<SBAddFullHash> add_full_hashes; 457 std::vector<SBAddFullHash> add_full_hashes;
497 std::vector<SBSubFullHash> sub_full_hashes; 458 std::vector<SBSubFullHash> sub_full_hashes;
498 459
499 // Read original data into the vectors. 460 // Read original data into the vectors.
500 if (!empty_) { 461 if (!empty_) {
501 DCHECK(file_.get()); 462 DCHECK(file_.get());
502 463
503 if (!FileRewind(file_.get())) 464 if (!FileRewind(file_.get()))
504 return OnCorruptDatabase(); 465 return OnCorruptDatabase();
505 466
506 base::MD5Context context; 467 base::MD5Context context;
507 base::MD5Init(&context); 468 base::MD5Init(&context);
508 469
509 // Read the file header and make sure it looks right. 470 // Read the file header and make sure it looks right.
510 FileHeader header; 471 FileHeader header;
511 if (!ReadAndVerifyHeader(filename_, file_.get(), &header, &context)) 472 if (!ReadAndVerifyHeader(filename_, file_.get(), &header, &context))
512 return OnCorruptDatabase(); 473 return OnCorruptDatabase();
513 474
514 // Re-read the chunks-seen data to get to the later data in the 475 // Re-read the chunks-seen data to get to the later data in the
515 // file and calculate the checksum. No new elements should be 476 // file and calculate the checksum. No new elements should be
516 // added to the sets. 477 // added to the sets.
517 if (!ReadToChunkSet(&add_chunks_cache_, header.add_chunk_count, 478 if (!ReadToContainer(&add_chunks_cache_, header.add_chunk_count,
518 file_.get(), &context) || 479 file_.get(), &context) ||
519 !ReadToChunkSet(&sub_chunks_cache_, header.sub_chunk_count, 480 !ReadToContainer(&sub_chunks_cache_, header.sub_chunk_count,
520 file_.get(), &context)) 481 file_.get(), &context))
521 return OnCorruptDatabase(); 482 return OnCorruptDatabase();
522 483
523 add_prefixes.reserve(header.add_prefix_count + add_prefixes_added_); 484 if (!ReadToContainer(&add_prefixes, header.add_prefix_count,
524 485 file_.get(), &context) ||
525 if (!ReadToVector(&add_prefixes, header.add_prefix_count, 486 !ReadToContainer(&sub_prefixes, header.sub_prefix_count,
526 file_.get(), &context) || 487 file_.get(), &context) ||
527 !ReadToVector(&sub_prefixes, header.sub_prefix_count, 488 !ReadToContainer(&add_full_hashes, header.add_hash_count,
528 file_.get(), &context) || 489 file_.get(), &context) ||
529 !ReadToVector(&add_full_hashes, header.add_hash_count, 490 !ReadToContainer(&sub_full_hashes, header.sub_hash_count,
530 file_.get(), &context) || 491 file_.get(), &context))
531 !ReadToVector(&sub_full_hashes, header.sub_hash_count,
532 file_.get(), &context))
533 return OnCorruptDatabase(); 492 return OnCorruptDatabase();
534 493
535 // Calculate the digest to this point. 494 // Calculate the digest to this point.
536 base::MD5Digest calculated_digest; 495 base::MD5Digest calculated_digest;
537 base::MD5Final(&calculated_digest, &context); 496 base::MD5Final(&calculated_digest, &context);
538 497
539 // Read the stored checksum and verify it. 498 // Read the stored checksum and verify it.
540 base::MD5Digest file_digest; 499 base::MD5Digest file_digest;
541 if (!ReadArray(&file_digest, 1, file_.get(), NULL)) 500 if (!ReadItem(&file_digest, file_.get(), NULL))
542 return OnCorruptDatabase(); 501 return OnCorruptDatabase();
543 502
544 if (0 != memcmp(&file_digest, &calculated_digest, sizeof(file_digest))) 503 if (0 != memcmp(&file_digest, &calculated_digest, sizeof(file_digest)))
545 return OnCorruptDatabase(); 504 return OnCorruptDatabase();
546 505
547 // Close the file so we can later rename over it. 506 // Close the file so we can later rename over it.
548 file_.reset(); 507 file_.reset();
549 } else {
550 add_prefixes.reserve(add_prefixes_added_);
jar (doing other things) 2011/10/21 19:10:45 What happened to this line? FWIW: I had a heck of
Scott Hess - ex-Googler 2011/10/21 23:39:32 The reserve() was my earlier change to have the ve
551 } 508 }
552 DCHECK(!file_.get()); 509 DCHECK(!file_.get());
553 510
554 // Rewind the temporary storage. 511 // Rewind the temporary storage.
555 if (!FileRewind(new_file_.get())) 512 if (!FileRewind(new_file_.get()))
556 return false; 513 return false;
557 514
558 // Get chunk file's size for validating counts. 515 // Get chunk file's size for validating counts.
559 int64 size = 0; 516 int64 size = 0;
560 if (!file_util::GetFileSize(TemporaryFileForFilename(filename_), &size)) 517 if (!file_util::GetFileSize(TemporaryFileForFilename(filename_), &size))
561 return OnCorruptDatabase(); 518 return OnCorruptDatabase();
562 519
563 // Track update size to answer questions at http://crbug.com/72216 . 520 // Track update size to answer questions at http://crbug.com/72216 .
564 // Log small updates as 1k so that the 0 (underflow) bucket can be 521 // Log small updates as 1k so that the 0 (underflow) bucket can be
565 // used for "empty" in SafeBrowsingDatabase. 522 // used for "empty" in SafeBrowsingDatabase.
566 UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", 523 UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes",
567 std::max(static_cast<int>(size / 1024), 1)); 524 std::max(static_cast<int>(size / 1024), 1));
568 525
569 // TODO(shess): For SB2.AddPrefixesReallocs histogram.
570 int add_prefixes_reallocs = 0;
571
572 // Append the accumulated chunks onto the vectors read from |file_|. 526 // Append the accumulated chunks onto the vectors read from |file_|.
573 for (int i = 0; i < chunks_written_; ++i) { 527 for (int i = 0; i < chunks_written_; ++i) {
574 ChunkHeader header; 528 ChunkHeader header;
575 529
576 int64 ofs = ftell(new_file_.get()); 530 int64 ofs = ftell(new_file_.get());
577 if (ofs == -1) 531 if (ofs == -1)
578 return false; 532 return false;
579 533
580 if (!ReadArray(&header, 1, new_file_.get(), NULL)) 534 if (!ReadItem(&header, new_file_.get(), NULL))
581 return false; 535 return false;
582 536
583 // As a safety measure, make sure that the header describes a sane 537 // As a safety measure, make sure that the header describes a sane
584 // chunk, given the remaining file size. 538 // chunk, given the remaining file size.
585 int64 expected_size = ofs + sizeof(ChunkHeader); 539 int64 expected_size = ofs + sizeof(ChunkHeader);
586 expected_size += header.add_prefix_count * sizeof(SBAddPrefix); 540 expected_size += header.add_prefix_count * sizeof(SBAddPrefix);
587 expected_size += header.sub_prefix_count * sizeof(SBSubPrefix); 541 expected_size += header.sub_prefix_count * sizeof(SBSubPrefix);
588 expected_size += header.add_hash_count * sizeof(SBAddFullHash); 542 expected_size += header.add_hash_count * sizeof(SBAddFullHash);
589 expected_size += header.sub_hash_count * sizeof(SBSubFullHash); 543 expected_size += header.sub_hash_count * sizeof(SBSubFullHash);
590 if (expected_size > size) 544 if (expected_size > size)
591 return false; 545 return false;
592 546
593 // TODO(shess): For SB2.AddPrefixesReallocs histogram.
594 SBAddPrefix* add_prefixes_old_start =
595 (add_prefixes.size() ? &add_prefixes[0] : NULL);
596
597 // TODO(shess): If the vectors were kept sorted, then this code 547 // TODO(shess): If the vectors were kept sorted, then this code
598 // could use std::inplace_merge() to merge everything together in 548 // could use std::inplace_merge() to merge everything together in
599 // sorted order. That might still be slower than just sorting at 549 // sorted order. That might still be slower than just sorting at
600 // the end if there were a large number of chunks. In that case 550 // the end if there were a large number of chunks. In that case
601 // some sort of recursive binary merge might be in order (merge 551 // some sort of recursive binary merge might be in order (merge
602 // chunks pairwise, merge those chunks pairwise, and so on, then 552 // chunks pairwise, merge those chunks pairwise, and so on, then
603 // merge the result with the main list). 553 // merge the result with the main list).
604 if (!ReadToVector(&add_prefixes, header.add_prefix_count, 554 if (!ReadToContainer(&add_prefixes, header.add_prefix_count,
605 new_file_.get(), NULL) || 555 new_file_.get(), NULL) ||
606 !ReadToVector(&sub_prefixes, header.sub_prefix_count, 556 !ReadToContainer(&sub_prefixes, header.sub_prefix_count,
607 new_file_.get(), NULL) || 557 new_file_.get(), NULL) ||
608 !ReadToVector(&add_full_hashes, header.add_hash_count, 558 !ReadToContainer(&add_full_hashes, header.add_hash_count,
609 new_file_.get(), NULL) || 559 new_file_.get(), NULL) ||
610 !ReadToVector(&sub_full_hashes, header.sub_hash_count, 560 !ReadToContainer(&sub_full_hashes, header.sub_hash_count,
611 new_file_.get(), NULL)) 561 new_file_.get(), NULL))
612 return false; 562 return false;
613
614 // Determine if the vector data moved to a new location. This
615 // won't track cases where the malloc library was able to expand
616 // the storage in place, but those cases shouldn't cause memory
617 // fragmentation anyhow.
618 SBAddPrefix* add_prefixes_new_start =
619 (add_prefixes.size() ? &add_prefixes[0] : NULL);
620 if (add_prefixes_old_start != add_prefixes_new_start)
621 ++add_prefixes_reallocs;
622 } 563 }
623 564
624 // Track the number of times this number of re-allocations occurred.
625 UMA_HISTOGRAM_COUNTS_100("SB2.AddPrefixesReallocs", add_prefixes_reallocs);
626
627 // Append items from |pending_adds|. 565 // Append items from |pending_adds|.
628 add_full_hashes.insert(add_full_hashes.end(), 566 add_full_hashes.insert(add_full_hashes.end(),
629 pending_adds.begin(), pending_adds.end()); 567 pending_adds.begin(), pending_adds.end());
630 568
631 // Check how often a prefix was checked which wasn't in the 569 // Check how often a prefix was checked which wasn't in the
632 // database. 570 // database.
633 SBCheckPrefixMisses(add_prefixes, prefix_misses); 571 SBCheckPrefixMisses(add_prefixes, prefix_misses);
634 572
635 // Knock the subs from the adds and process deleted chunks. 573 // Knock the subs from the adds and process deleted chunks.
636 SBProcessSubs(&add_prefixes, &sub_prefixes, 574 SBProcessSubs(&add_prefixes, &sub_prefixes,
(...skipping 14 matching lines...) Expand all
651 // Write a file header. 589 // Write a file header.
652 FileHeader header; 590 FileHeader header;
653 header.magic = kFileMagic; 591 header.magic = kFileMagic;
654 header.version = kFileVersion; 592 header.version = kFileVersion;
655 header.add_chunk_count = add_chunks_cache_.size(); 593 header.add_chunk_count = add_chunks_cache_.size();
656 header.sub_chunk_count = sub_chunks_cache_.size(); 594 header.sub_chunk_count = sub_chunks_cache_.size();
657 header.add_prefix_count = add_prefixes.size(); 595 header.add_prefix_count = add_prefixes.size();
658 header.sub_prefix_count = sub_prefixes.size(); 596 header.sub_prefix_count = sub_prefixes.size();
659 header.add_hash_count = add_full_hashes.size(); 597 header.add_hash_count = add_full_hashes.size();
660 header.sub_hash_count = sub_full_hashes.size(); 598 header.sub_hash_count = sub_full_hashes.size();
661 if (!WriteArray(&header, 1, new_file_.get(), &context)) 599 if (!WriteItem(header, new_file_.get(), &context))
662 return false; 600 return false;
663 601
664 // Write all the chunk data. 602 // Write all the chunk data.
665 if (!WriteChunkSet(add_chunks_cache_, new_file_.get(), &context) || 603 if (!WriteContainer(add_chunks_cache_, new_file_.get(), &context) ||
666 !WriteChunkSet(sub_chunks_cache_, new_file_.get(), &context) || 604 !WriteContainer(sub_chunks_cache_, new_file_.get(), &context) ||
667 !WriteVector(add_prefixes, new_file_.get(), &context) || 605 !WriteContainer(add_prefixes, new_file_.get(), &context) ||
668 !WriteVector(sub_prefixes, new_file_.get(), &context) || 606 !WriteContainer(sub_prefixes, new_file_.get(), &context) ||
669 !WriteVector(add_full_hashes, new_file_.get(), &context) || 607 !WriteContainer(add_full_hashes, new_file_.get(), &context) ||
670 !WriteVector(sub_full_hashes, new_file_.get(), &context)) 608 !WriteContainer(sub_full_hashes, new_file_.get(), &context))
671 return false; 609 return false;
672 610
673 // Write the checksum at the end. 611 // Write the checksum at the end.
674 base::MD5Digest digest; 612 base::MD5Digest digest;
675 base::MD5Final(&digest, &context); 613 base::MD5Final(&digest, &context);
676 if (!WriteArray(&digest, 1, new_file_.get(), NULL)) 614 if (!WriteItem(digest, new_file_.get(), NULL))
677 return false; 615 return false;
678 616
679 // Trim any excess left over from the temporary chunk data. 617 // Trim any excess left over from the temporary chunk data.
680 if (!file_util::TruncateFile(new_file_.get())) 618 if (!file_util::TruncateFile(new_file_.get()))
681 return false; 619 return false;
682 620
683 // Close the file handle and swizzle the file into place. 621 // Close the file handle and swizzle the file into place.
684 new_file_.reset(); 622 new_file_.reset();
685 if (!file_util::Delete(filename_, false) && 623 if (!file_util::Delete(filename_, false) &&
686 file_util::PathExists(filename_)) 624 file_util::PathExists(filename_))
(...skipping 10 matching lines...) Expand all
697 // Pass the resulting data off to the caller. 635 // Pass the resulting data off to the caller.
698 add_prefixes_result->swap(add_prefixes); 636 add_prefixes_result->swap(add_prefixes);
699 add_full_hashes_result->swap(add_full_hashes); 637 add_full_hashes_result->swap(add_full_hashes);
700 638
701 return true; 639 return true;
702 } 640 }
703 641
704 bool SafeBrowsingStoreFile::FinishUpdate( 642 bool SafeBrowsingStoreFile::FinishUpdate(
705 const std::vector<SBAddFullHash>& pending_adds, 643 const std::vector<SBAddFullHash>& pending_adds,
706 const std::set<SBPrefix>& prefix_misses, 644 const std::set<SBPrefix>& prefix_misses,
707 std::vector<SBAddPrefix>* add_prefixes_result, 645 SBAddPrefixContainer* add_prefixes_result,
708 std::vector<SBAddFullHash>* add_full_hashes_result) { 646 std::vector<SBAddFullHash>* add_full_hashes_result) {
709 DCHECK(add_prefixes_result); 647 DCHECK(add_prefixes_result);
710 DCHECK(add_full_hashes_result); 648 DCHECK(add_full_hashes_result);
711 649
712 bool ret = DoUpdate(pending_adds, prefix_misses, 650 bool ret = DoUpdate(pending_adds, prefix_misses,
713 add_prefixes_result, add_full_hashes_result); 651 add_prefixes_result, add_full_hashes_result);
714 652
715 if (!ret) { 653 if (!ret) {
716 CancelUpdate(); 654 CancelUpdate();
717 return false; 655 return false;
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
753 out->insert(out->end(), sub_chunks_cache_.begin(), sub_chunks_cache_.end()); 691 out->insert(out->end(), sub_chunks_cache_.begin(), sub_chunks_cache_.end());
754 } 692 }
755 693
756 void SafeBrowsingStoreFile::DeleteAddChunk(int32 chunk_id) { 694 void SafeBrowsingStoreFile::DeleteAddChunk(int32 chunk_id) {
757 add_del_cache_.insert(chunk_id); 695 add_del_cache_.insert(chunk_id);
758 } 696 }
759 697
760 void SafeBrowsingStoreFile::DeleteSubChunk(int32 chunk_id) { 698 void SafeBrowsingStoreFile::DeleteSubChunk(int32 chunk_id) {
761 sub_del_cache_.insert(chunk_id); 699 sub_del_cache_.insert(chunk_id);
762 } 700 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698