| OLD | NEW |
| 1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "chrome/browser/safe_browsing/safe_browsing_database.h" | 5 #include "chrome/browser/safe_browsing/safe_browsing_database.h" |
| 6 | 6 |
| 7 #include <algorithm> | 7 #include <algorithm> |
| 8 #include <iterator> | 8 #include <iterator> |
| 9 | 9 |
| 10 #include "base/file_util.h" | 10 #include "base/file_util.h" |
| (...skipping 15 matching lines...) Expand all Loading... |
| 26 | 26 |
| 27 namespace { | 27 namespace { |
| 28 | 28 |
| 29 // Filename suffix for the bloom filter. | 29 // Filename suffix for the bloom filter. |
| 30 const FilePath::CharType kBloomFilterFile[] = FILE_PATH_LITERAL(" Filter 2"); | 30 const FilePath::CharType kBloomFilterFile[] = FILE_PATH_LITERAL(" Filter 2"); |
| 31 // Filename suffix for download store. | 31 // Filename suffix for download store. |
| 32 const FilePath::CharType kDownloadDBFile[] = FILE_PATH_LITERAL(" Download"); | 32 const FilePath::CharType kDownloadDBFile[] = FILE_PATH_LITERAL(" Download"); |
| 33 // Filename suffix for client-side phishing detection whitelist store. | 33 // Filename suffix for client-side phishing detection whitelist store. |
| 34 const FilePath::CharType kCsdWhitelistDBFile[] = | 34 const FilePath::CharType kCsdWhitelistDBFile[] = |
| 35 FILE_PATH_LITERAL(" Csd Whitelist"); | 35 FILE_PATH_LITERAL(" Csd Whitelist"); |
| 36 // Filename suffix for the download whitelist store. |
| 37 const FilePath::CharType kDownloadWhitelistDBFile[] = |
| 38 FILE_PATH_LITERAL(" Download Whitelist"); |
| 36 // Filename suffix for browse store. | 39 // Filename suffix for browse store. |
| 37 // TODO(lzheng): change to a better name when we change the file format. | 40 // TODO(lzheng): change to a better name when we change the file format. |
| 38 const FilePath::CharType kBrowseDBFile[] = FILE_PATH_LITERAL(" Bloom"); | 41 const FilePath::CharType kBrowseDBFile[] = FILE_PATH_LITERAL(" Bloom"); |
| 39 | 42 |
| 40 // The maximum staleness for a cached entry. | 43 // The maximum staleness for a cached entry. |
| 41 const int kMaxStalenessMinutes = 45; | 44 const int kMaxStalenessMinutes = 45; |
| 42 | 45 |
| 43 // Maximum number of entries we allow in the client-side phishing detection | 46 // Maximum number of entries we allow in any of the whitelists. |
| 44 // whitelist. If the whitelist on disk contains more entries then | 47 // If a whitelist on disk contains more entries then all lookups to |
| 45 // ContainsCsdWhitelistedUrl will always return true. | 48 // the whitelist will be considered a match. |
| 46 const size_t kMaxCsdWhitelistSize = 5000; | 49 const size_t kMaxWhitelistSize = 5000; |
| 47 | 50 |
| 48 // If the hash of this exact expression is on the csd whitelist then | 51 // If the hash of this exact expression is on a whitelist then all |
| 49 // ContainsCsdWhitelistedUrl will always return true. | 52 // lookups to this whitelist will be considered a match. |
| 50 const char kCsdKillSwitchUrl[] = | 53 const char kWhitelistKillSwitchUrl[] = |
| 51 "sb-ssl.google.com/safebrowsing/csd/killswitch"; | 54 "sb-ssl.google.com/safebrowsing/csd/killswitch"; // Don't change this! |
| 52 | 55 |
| 53 // To save space, the incoming |chunk_id| and |list_id| are combined | 56 // To save space, the incoming |chunk_id| and |list_id| are combined |
| 54 // into an |encoded_chunk_id| for storage by shifting the |list_id| | 57 // into an |encoded_chunk_id| for storage by shifting the |list_id| |
| 55 // into the low-order bits. These functions decode that information. | 58 // into the low-order bits. These functions decode that information. |
| 56 // TODO(lzheng): It was reasonable when database is saved in sqlite, but | 59 // TODO(lzheng): It was reasonable when database is saved in sqlite, but |
| 57 // there should be better ways to save chunk_id and list_id after we use | 60 // there should be better ways to save chunk_id and list_id after we use |
| 58 // SafeBrowsingStoreFile. | 61 // SafeBrowsingStoreFile. |
| 59 int GetListIdBit(const int encoded_chunk_id) { | 62 int GetListIdBit(const int encoded_chunk_id) { |
| 60 return encoded_chunk_id & 1; | 63 return encoded_chunk_id & 1; |
| 61 } | 64 } |
| 62 int DecodeChunkId(int encoded_chunk_id) { | 65 int DecodeChunkId(int encoded_chunk_id) { |
| 63 return encoded_chunk_id >> 1; | 66 return encoded_chunk_id >> 1; |
| 64 } | 67 } |
| 65 int EncodeChunkId(const int chunk, const int list_id) { | 68 int EncodeChunkId(const int chunk, const int list_id) { |
| 66 DCHECK_NE(list_id, safe_browsing_util::INVALID); | 69 DCHECK_NE(list_id, safe_browsing_util::INVALID); |
| 67 return chunk << 1 | list_id % 2; | 70 return chunk << 1 | list_id % 2; |
| 68 } | 71 } |
| 69 | 72 |
| 70 // Generate the set of full hashes to check for |url|. If | 73 // Generate the set of full hashes to check for |url|. If |
| 71 // |include_whitelist_hashes| is true we will generate additional path-prefixes | 74 // |include_whitelist_hashes| is true we will generate additional path-prefixes |
| 72 // to match against the csd whitelist. E.g., if the path-prefix /foo is on the | 75 // to match against the csd whitelist. E.g., if the path-prefix /foo is on the |
| 73 // whitelist it should also match /foo/bar which is not the case for all the | 76 // whitelist it should also match /foo/bar which is not the case for all the |
| 74 // other lists. | 77 // other lists. We'll also always add a pattern for the empty path. |
| 75 // TODO(shess): This function is almost the same as | 78 // TODO(shess): This function is almost the same as |
| 76 // |CompareFullHashes()| in safe_browsing_util.cc, except that code | 79 // |CompareFullHashes()| in safe_browsing_util.cc, except that code |
| 77 // does an early exit on match. Since match should be the infrequent | 80 // does an early exit on match. Since match should be the infrequent |
| 78 // case (phishing or malware found), consider combining this function | 81 // case (phishing or malware found), consider combining this function |
| 79 // with that one. | 82 // with that one. |
| 80 void BrowseFullHashesToCheck(const GURL& url, | 83 void BrowseFullHashesToCheck(const GURL& url, |
| 81 bool include_whitelist_hashes, | 84 bool include_whitelist_hashes, |
| 82 std::vector<SBFullHash>* full_hashes) { | 85 std::vector<SBFullHash>* full_hashes) { |
| 83 std::vector<std::string> hosts; | 86 std::vector<std::string> hosts; |
| 84 if (url.HostIsIPAddress()) { | 87 if (url.HostIsIPAddress()) { |
| (...skipping 303 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 388 return prefix_set.release(); | 391 return prefix_set.release(); |
| 389 } | 392 } |
| 390 | 393 |
| 391 } // namespace | 394 } // namespace |
| 392 | 395 |
| 393 // The default SafeBrowsingDatabaseFactory. | 396 // The default SafeBrowsingDatabaseFactory. |
| 394 class SafeBrowsingDatabaseFactoryImpl : public SafeBrowsingDatabaseFactory { | 397 class SafeBrowsingDatabaseFactoryImpl : public SafeBrowsingDatabaseFactory { |
| 395 public: | 398 public: |
| 396 virtual SafeBrowsingDatabase* CreateSafeBrowsingDatabase( | 399 virtual SafeBrowsingDatabase* CreateSafeBrowsingDatabase( |
| 397 bool enable_download_protection, | 400 bool enable_download_protection, |
| 398 bool enable_client_side_whitelist) { | 401 bool enable_client_side_whitelist, |
| 402 bool enable_download_whitelist) { |
| 399 return new SafeBrowsingDatabaseNew( | 403 return new SafeBrowsingDatabaseNew( |
| 400 new SafeBrowsingStoreFile, | 404 new SafeBrowsingStoreFile, |
| 401 enable_download_protection ? new SafeBrowsingStoreFile : NULL, | 405 enable_download_protection ? new SafeBrowsingStoreFile : NULL, |
| 402 enable_client_side_whitelist ? new SafeBrowsingStoreFile : NULL); | 406 enable_client_side_whitelist ? new SafeBrowsingStoreFile : NULL, |
| 407 enable_download_whitelist ? new SafeBrowsingStoreFile : NULL); |
| 403 } | 408 } |
| 404 | 409 |
| 405 SafeBrowsingDatabaseFactoryImpl() { } | 410 SafeBrowsingDatabaseFactoryImpl() { } |
| 406 | 411 |
| 407 private: | 412 private: |
| 408 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseFactoryImpl); | 413 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseFactoryImpl); |
| 409 }; | 414 }; |
| 410 | 415 |
| 411 // static | 416 // static |
| 412 SafeBrowsingDatabaseFactory* SafeBrowsingDatabase::factory_ = NULL; | 417 SafeBrowsingDatabaseFactory* SafeBrowsingDatabase::factory_ = NULL; |
| 413 | 418 |
| 414 // Factory method, non-thread safe. Caller has to make sure this s called | 419 // Factory method, non-thread safe. Caller has to make sure this s called |
| 415 // on SafeBrowsing Thread. | 420 // on SafeBrowsing Thread. |
| 416 // TODO(shess): There's no need for a factory any longer. Convert | 421 // TODO(shess): There's no need for a factory any longer. Convert |
| 417 // SafeBrowsingDatabaseNew to SafeBrowsingDatabase, and have Create() | 422 // SafeBrowsingDatabaseNew to SafeBrowsingDatabase, and have Create() |
| 418 // callers just construct things directly. | 423 // callers just construct things directly. |
| 419 SafeBrowsingDatabase* SafeBrowsingDatabase::Create( | 424 SafeBrowsingDatabase* SafeBrowsingDatabase::Create( |
| 420 bool enable_download_protection, | 425 bool enable_download_protection, |
| 421 bool enable_client_side_whitelist) { | 426 bool enable_client_side_whitelist, |
| 427 bool enable_download_whitelist) { |
| 422 if (!factory_) | 428 if (!factory_) |
| 423 factory_ = new SafeBrowsingDatabaseFactoryImpl(); | 429 factory_ = new SafeBrowsingDatabaseFactoryImpl(); |
| 424 return factory_->CreateSafeBrowsingDatabase(enable_download_protection, | 430 return factory_->CreateSafeBrowsingDatabase(enable_download_protection, |
| 425 enable_client_side_whitelist); | 431 enable_client_side_whitelist, |
| 432 enable_download_whitelist); |
| 426 } | 433 } |
| 427 | 434 |
| 428 SafeBrowsingDatabase::~SafeBrowsingDatabase() { | 435 SafeBrowsingDatabase::~SafeBrowsingDatabase() { |
| 429 } | 436 } |
| 430 | 437 |
| 431 // static | 438 // static |
| 432 FilePath SafeBrowsingDatabase::BrowseDBFilename( | 439 FilePath SafeBrowsingDatabase::BrowseDBFilename( |
| 433 const FilePath& db_base_filename) { | 440 const FilePath& db_base_filename) { |
| 434 return FilePath(db_base_filename.value() + kBrowseDBFile); | 441 return FilePath(db_base_filename.value() + kBrowseDBFile); |
| 435 } | 442 } |
| 436 | 443 |
| 437 // static | 444 // static |
| 438 FilePath SafeBrowsingDatabase::DownloadDBFilename( | 445 FilePath SafeBrowsingDatabase::DownloadDBFilename( |
| 439 const FilePath& db_base_filename) { | 446 const FilePath& db_base_filename) { |
| 440 return FilePath(db_base_filename.value() + kDownloadDBFile); | 447 return FilePath(db_base_filename.value() + kDownloadDBFile); |
| 441 } | 448 } |
| 442 | 449 |
| 443 // static | 450 // static |
| 444 FilePath SafeBrowsingDatabase::BloomFilterForFilename( | 451 FilePath SafeBrowsingDatabase::BloomFilterForFilename( |
| 445 const FilePath& db_filename) { | 452 const FilePath& db_filename) { |
| 446 return FilePath(db_filename.value() + kBloomFilterFile); | 453 return FilePath(db_filename.value() + kBloomFilterFile); |
| 447 } | 454 } |
| 448 | 455 |
| 449 // static | 456 // static |
| 450 FilePath SafeBrowsingDatabase::CsdWhitelistDBFilename( | 457 FilePath SafeBrowsingDatabase::CsdWhitelistDBFilename( |
| 451 const FilePath& db_filename) { | 458 const FilePath& db_filename) { |
| 452 return FilePath(db_filename.value() + kCsdWhitelistDBFile); | 459 return FilePath(db_filename.value() + kCsdWhitelistDBFile); |
| 453 } | 460 } |
| 454 | 461 |
| 462 // static |
| 463 FilePath SafeBrowsingDatabase::DownloadWhitelistDBFilename( |
| 464 const FilePath& db_filename) { |
| 465 return FilePath(db_filename.value() + kDownloadWhitelistDBFile); |
| 466 } |
| 467 |
| 455 SafeBrowsingStore* SafeBrowsingDatabaseNew::GetStore(const int list_id) { | 468 SafeBrowsingStore* SafeBrowsingDatabaseNew::GetStore(const int list_id) { |
| 456 if (list_id == safe_browsing_util::PHISH || | 469 if (list_id == safe_browsing_util::PHISH || |
| 457 list_id == safe_browsing_util::MALWARE) { | 470 list_id == safe_browsing_util::MALWARE) { |
| 458 return browse_store_.get(); | 471 return browse_store_.get(); |
| 459 } else if (list_id == safe_browsing_util::BINURL || | 472 } else if (list_id == safe_browsing_util::BINURL || |
| 460 list_id == safe_browsing_util::BINHASH) { | 473 list_id == safe_browsing_util::BINHASH) { |
| 461 return download_store_.get(); | 474 return download_store_.get(); |
| 462 } else if (list_id == safe_browsing_util::CSDWHITELIST) { | 475 } else if (list_id == safe_browsing_util::CSDWHITELIST) { |
| 463 return csd_whitelist_store_.get(); | 476 return csd_whitelist_store_.get(); |
| 477 } else if (list_id == safe_browsing_util::DOWNLOADWHITELIST) { |
| 478 return download_whitelist_store_.get(); |
| 464 } | 479 } |
| 465 return NULL; | 480 return NULL; |
| 466 } | 481 } |
| 467 | 482 |
| 468 // static | 483 // static |
| 469 void SafeBrowsingDatabase::RecordFailure(FailureType failure_type) { | 484 void SafeBrowsingDatabase::RecordFailure(FailureType failure_type) { |
| 470 UMA_HISTOGRAM_ENUMERATION("SB2.DatabaseFailure", failure_type, | 485 UMA_HISTOGRAM_ENUMERATION("SB2.DatabaseFailure", failure_type, |
| 471 FAILURE_DATABASE_MAX); | 486 FAILURE_DATABASE_MAX); |
| 472 } | 487 } |
| 473 | 488 |
| 474 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew() | 489 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew() |
| 475 : creation_loop_(MessageLoop::current()), | 490 : creation_loop_(MessageLoop::current()), |
| 476 browse_store_(new SafeBrowsingStoreFile), | 491 browse_store_(new SafeBrowsingStoreFile), |
| 477 download_store_(NULL), | 492 download_store_(NULL), |
| 478 csd_whitelist_store_(NULL), | 493 csd_whitelist_store_(NULL), |
| 494 download_whitelist_store_(NULL), |
| 479 ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)) { | 495 ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)) { |
| 480 DCHECK(browse_store_.get()); | 496 DCHECK(browse_store_.get()); |
| 481 DCHECK(!download_store_.get()); | 497 DCHECK(!download_store_.get()); |
| 482 DCHECK(!csd_whitelist_store_.get()); | 498 DCHECK(!csd_whitelist_store_.get()); |
| 499 DCHECK(!download_whitelist_store_.get()); |
| 483 } | 500 } |
| 484 | 501 |
| 485 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew( | 502 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew( |
| 486 SafeBrowsingStore* browse_store, | 503 SafeBrowsingStore* browse_store, |
| 487 SafeBrowsingStore* download_store, | 504 SafeBrowsingStore* download_store, |
| 488 SafeBrowsingStore* csd_whitelist_store) | 505 SafeBrowsingStore* csd_whitelist_store, |
| 506 SafeBrowsingStore* download_whitelist_store) |
| 489 : creation_loop_(MessageLoop::current()), | 507 : creation_loop_(MessageLoop::current()), |
| 490 browse_store_(browse_store), | 508 browse_store_(browse_store), |
| 491 download_store_(download_store), | 509 download_store_(download_store), |
| 492 csd_whitelist_store_(csd_whitelist_store), | 510 csd_whitelist_store_(csd_whitelist_store), |
| 511 download_whitelist_store_(download_whitelist_store), |
| 493 ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)), | 512 ALLOW_THIS_IN_INITIALIZER_LIST(reset_factory_(this)), |
| 494 corruption_detected_(false) { | 513 corruption_detected_(false) { |
| 495 DCHECK(browse_store_.get()); | 514 DCHECK(browse_store_.get()); |
| 496 } | 515 } |
| 497 | 516 |
| 498 SafeBrowsingDatabaseNew::~SafeBrowsingDatabaseNew() { | 517 SafeBrowsingDatabaseNew::~SafeBrowsingDatabaseNew() { |
| 499 DCHECK_EQ(creation_loop_, MessageLoop::current()); | 518 DCHECK_EQ(creation_loop_, MessageLoop::current()); |
| 500 } | 519 } |
| 501 | 520 |
| 502 void SafeBrowsingDatabaseNew::Init(const FilePath& filename_base) { | 521 void SafeBrowsingDatabaseNew::Init(const FilePath& filename_base) { |
| 503 DCHECK_EQ(creation_loop_, MessageLoop::current()); | 522 DCHECK_EQ(creation_loop_, MessageLoop::current()); |
| 504 // Ensure we haven't been run before. | 523 // Ensure we haven't been run before. |
| 505 DCHECK(browse_filename_.empty()); | 524 DCHECK(browse_filename_.empty()); |
| 506 DCHECK(download_filename_.empty()); | 525 DCHECK(download_filename_.empty()); |
| 507 DCHECK(csd_whitelist_filename_.empty()); | 526 DCHECK(csd_whitelist_filename_.empty()); |
| 527 DCHECK(download_whitelist_filename_.empty()); |
| 508 | 528 |
| 509 browse_filename_ = BrowseDBFilename(filename_base); | 529 browse_filename_ = BrowseDBFilename(filename_base); |
| 510 bloom_filter_filename_ = BloomFilterForFilename(browse_filename_); | 530 bloom_filter_filename_ = BloomFilterForFilename(browse_filename_); |
| 511 | 531 |
| 512 browse_store_->Init( | 532 browse_store_->Init( |
| 513 browse_filename_, | 533 browse_filename_, |
| 514 NewCallback(this, &SafeBrowsingDatabaseNew::HandleCorruptDatabase)); | 534 NewCallback(this, &SafeBrowsingDatabaseNew::HandleCorruptDatabase)); |
| 515 DVLOG(1) << "Init browse store: " << browse_filename_.value(); | 535 DVLOG(1) << "Init browse store: " << browse_filename_.value(); |
| 516 | 536 |
| 517 { | 537 { |
| (...skipping 16 matching lines...) Expand all Loading... |
| 534 } | 554 } |
| 535 | 555 |
| 536 if (csd_whitelist_store_.get()) { | 556 if (csd_whitelist_store_.get()) { |
| 537 csd_whitelist_filename_ = CsdWhitelistDBFilename(filename_base); | 557 csd_whitelist_filename_ = CsdWhitelistDBFilename(filename_base); |
| 538 csd_whitelist_store_->Init( | 558 csd_whitelist_store_->Init( |
| 539 csd_whitelist_filename_, | 559 csd_whitelist_filename_, |
| 540 NewCallback(this, &SafeBrowsingDatabaseNew::HandleCorruptDatabase)); | 560 NewCallback(this, &SafeBrowsingDatabaseNew::HandleCorruptDatabase)); |
| 541 DVLOG(1) << "Init csd whitelist store: " << csd_whitelist_filename_.value(); | 561 DVLOG(1) << "Init csd whitelist store: " << csd_whitelist_filename_.value(); |
| 542 std::vector<SBAddFullHash> full_hashes; | 562 std::vector<SBAddFullHash> full_hashes; |
| 543 if (csd_whitelist_store_->GetAddFullHashes(&full_hashes)) { | 563 if (csd_whitelist_store_->GetAddFullHashes(&full_hashes)) { |
| 544 LoadCsdWhitelist(full_hashes); | 564 LoadWhitelist(full_hashes, &csd_whitelist_); |
| 545 } else { | 565 } else { |
| 546 CsdWhitelistAllUrls(); | 566 WhitelistEverything(&csd_whitelist_); |
| 547 } | 567 } |
| 548 } else { | 568 } else { |
| 549 CsdWhitelistAllUrls(); // Just to be safe. | 569 WhitelistEverything(&csd_whitelist_); // Just to be safe. |
| 570 } |
| 571 |
| 572 if (download_whitelist_store_.get()) { |
| 573 download_whitelist_filename_ = DownloadWhitelistDBFilename(filename_base); |
| 574 download_whitelist_store_->Init( |
| 575 download_whitelist_filename_, |
| 576 NewCallback(this, &SafeBrowsingDatabaseNew::HandleCorruptDatabase)); |
| 577 DVLOG(1) << "Init download whitelist store: " |
| 578 << download_whitelist_filename_.value(); |
| 579 std::vector<SBAddFullHash> full_hashes; |
| 580 if (download_whitelist_store_->GetAddFullHashes(&full_hashes)) { |
| 581 LoadWhitelist(full_hashes, &download_whitelist_); |
| 582 } else { |
| 583 WhitelistEverything(&download_whitelist_); |
| 584 } |
| 585 } else { |
| 586 WhitelistEverything(&download_whitelist_); // Just to be safe. |
| 550 } | 587 } |
| 551 } | 588 } |
| 552 | 589 |
| 553 bool SafeBrowsingDatabaseNew::ResetDatabase() { | 590 bool SafeBrowsingDatabaseNew::ResetDatabase() { |
| 554 DCHECK_EQ(creation_loop_, MessageLoop::current()); | 591 DCHECK_EQ(creation_loop_, MessageLoop::current()); |
| 555 | 592 |
| 556 // Delete files on disk. | 593 // Delete files on disk. |
| 557 // TODO(shess): Hard to see where one might want to delete without a | 594 // TODO(shess): Hard to see where one might want to delete without a |
| 558 // reset. Perhaps inline |Delete()|? | 595 // reset. Perhaps inline |Delete()|? |
| 559 if (!Delete()) | 596 if (!Delete()) |
| 560 return false; | 597 return false; |
| 561 | 598 |
| 562 // Reset objects in memory. | 599 // Reset objects in memory. |
| 563 { | 600 { |
| 564 base::AutoLock locked(lookup_lock_); | 601 base::AutoLock locked(lookup_lock_); |
| 565 full_browse_hashes_.clear(); | 602 full_browse_hashes_.clear(); |
| 566 pending_browse_hashes_.clear(); | 603 pending_browse_hashes_.clear(); |
| 567 prefix_miss_cache_.clear(); | 604 prefix_miss_cache_.clear(); |
| 568 // TODO(shess): This could probably be |bloom_filter_.reset()|. | 605 // TODO(shess): This could probably be |bloom_filter_.reset()|. |
| 569 browse_bloom_filter_ = new BloomFilter(BloomFilter::kBloomFilterMinSize * | 606 browse_bloom_filter_ = new BloomFilter(BloomFilter::kBloomFilterMinSize * |
| 570 BloomFilter::kBloomFilterSizeRatio); | 607 BloomFilter::kBloomFilterSizeRatio); |
| 571 // TODO(shess): It is simpler for the code to assume that presence | 608 // TODO(shess): It is simpler for the code to assume that presence |
| 572 // of a bloom filter always implies presence of a prefix set. | 609 // of a bloom filter always implies presence of a prefix set. |
| 573 prefix_set_.reset(new safe_browsing::PrefixSet(std::vector<SBPrefix>())); | 610 prefix_set_.reset(new safe_browsing::PrefixSet(std::vector<SBPrefix>())); |
| 574 } | 611 } |
| 575 // Wants to acquire the lock itself. | 612 // Wants to acquire the lock itself. |
| 576 CsdWhitelistAllUrls(); | 613 WhitelistEverything(&csd_whitelist_); |
| 614 WhitelistEverything(&download_whitelist_); |
| 577 | 615 |
| 578 return true; | 616 return true; |
| 579 } | 617 } |
| 580 | 618 |
| 581 // TODO(lzheng): Remove matching_list, it is not used anywhere. | 619 // TODO(lzheng): Remove matching_list, it is not used anywhere. |
| 582 bool SafeBrowsingDatabaseNew::ContainsBrowseUrl( | 620 bool SafeBrowsingDatabaseNew::ContainsBrowseUrl( |
| 583 const GURL& url, | 621 const GURL& url, |
| 584 std::string* matching_list, | 622 std::string* matching_list, |
| 585 std::vector<SBPrefix>* prefix_hits, | 623 std::vector<SBPrefix>* prefix_hits, |
| 586 std::vector<SBFullHashResult>* full_hits, | 624 std::vector<SBFullHashResult>* full_hits, |
| (...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 706 std::vector<SBPrefix> prefix_hits; | 744 std::vector<SBPrefix> prefix_hits; |
| 707 return MatchDownloadAddPrefixes(safe_browsing_util::BINHASH % 2, | 745 return MatchDownloadAddPrefixes(safe_browsing_util::BINHASH % 2, |
| 708 prefixes, | 746 prefixes, |
| 709 &prefix_hits); | 747 &prefix_hits); |
| 710 } | 748 } |
| 711 | 749 |
| 712 bool SafeBrowsingDatabaseNew::ContainsCsdWhitelistedUrl(const GURL& url) { | 750 bool SafeBrowsingDatabaseNew::ContainsCsdWhitelistedUrl(const GURL& url) { |
| 713 // This method is theoretically thread-safe but we expect all calls to | 751 // This method is theoretically thread-safe but we expect all calls to |
| 714 // originate from the IO thread. | 752 // originate from the IO thread. |
| 715 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 753 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
| 716 base::AutoLock l(lookup_lock_); | |
| 717 if (csd_whitelist_all_urls_) | |
| 718 return true; | |
| 719 | |
| 720 std::vector<SBFullHash> full_hashes; | 754 std::vector<SBFullHash> full_hashes; |
| 721 BrowseFullHashesToCheck(url, true, &full_hashes); | 755 BrowseFullHashesToCheck(url, true, &full_hashes); |
| 722 for (std::vector<SBFullHash>::const_iterator it = full_hashes.begin(); | 756 return ContainsWhitelistedHashes(csd_whitelist_, full_hashes); |
| 723 it != full_hashes.end(); ++it) { | 757 } |
| 724 if (std::binary_search(csd_whitelist_.begin(), csd_whitelist_.end(), *it)) | 758 |
| 759 bool SafeBrowsingDatabaseNew::ContainsDownloadWhitelistedUrl(const GURL& url) { |
| 760 std::vector<SBFullHash> full_hashes; |
| 761 BrowseFullHashesToCheck(url, true, &full_hashes); |
| 762 return ContainsWhitelistedHashes(download_whitelist_, full_hashes); |
| 763 } |
| 764 |
| 765 bool SafeBrowsingDatabaseNew::ContainsDownloadWhitelistedString( |
| 766 const std::string& str) { |
| 767 SBFullHash hash; |
| 768 crypto::SHA256HashString(str, &hash, sizeof(hash)); |
| 769 std::vector<SBFullHash> hashes; |
| 770 hashes.push_back(hash); |
| 771 return ContainsWhitelistedHashes(download_whitelist_, hashes); |
| 772 } |
| 773 |
| 774 bool SafeBrowsingDatabaseNew::ContainsWhitelistedHashes( |
| 775 const SBWhitelist& whitelist, |
| 776 const std::vector<SBFullHash>& hashes) { |
| 777 base::AutoLock l(lookup_lock_); |
| 778 if (whitelist.second) |
| 779 return true; |
| 780 for (std::vector<SBFullHash>::const_iterator it = hashes.begin(); |
| 781 it != hashes.end(); ++it) { |
| 782 if (std::binary_search(whitelist.first.begin(), whitelist.first.end(), *it)) |
| 725 return true; | 783 return true; |
| 726 } | 784 } |
| 727 return false; | 785 return false; |
| 728 } | 786 } |
| 729 | 787 |
| 730 // Helper to insert entries for all of the prefixes or full hashes in | 788 // Helper to insert entries for all of the prefixes or full hashes in |
| 731 // |entry| into the store. | 789 // |entry| into the store. |
| 732 void SafeBrowsingDatabaseNew::InsertAdd(int chunk_id, SBPrefix host, | 790 void SafeBrowsingDatabaseNew::InsertAdd(int chunk_id, SBPrefix host, |
| 733 const SBEntry* entry, int list_id) { | 791 const SBEntry* entry, int list_id) { |
| 734 DCHECK_EQ(creation_loop_, MessageLoop::current()); | 792 DCHECK_EQ(creation_loop_, MessageLoop::current()); |
| (...skipping 237 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 972 return false; | 1030 return false; |
| 973 } | 1031 } |
| 974 | 1032 |
| 975 if (download_store_.get() && !download_store_->BeginUpdate()) { | 1033 if (download_store_.get() && !download_store_->BeginUpdate()) { |
| 976 RecordFailure(FAILURE_DOWNLOAD_DATABASE_UPDATE_BEGIN); | 1034 RecordFailure(FAILURE_DOWNLOAD_DATABASE_UPDATE_BEGIN); |
| 977 HandleCorruptDatabase(); | 1035 HandleCorruptDatabase(); |
| 978 return false; | 1036 return false; |
| 979 } | 1037 } |
| 980 | 1038 |
| 981 if (csd_whitelist_store_.get() && !csd_whitelist_store_->BeginUpdate()) { | 1039 if (csd_whitelist_store_.get() && !csd_whitelist_store_->BeginUpdate()) { |
| 982 RecordFailure(FAILURE_CSD_WHITELIST_DATABASE_UPDATE_BEGIN); | 1040 RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN); |
| 983 HandleCorruptDatabase(); | 1041 HandleCorruptDatabase(); |
| 984 return false; | 1042 return false; |
| 985 } | 1043 } |
| 1044 |
| 1045 if (download_whitelist_store_.get() && |
| 1046 !download_whitelist_store_->BeginUpdate()) { |
| 1047 RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN); |
| 1048 HandleCorruptDatabase(); |
| 1049 return false; |
| 1050 } |
| 986 | 1051 |
| 987 std::vector<std::string> browse_listnames; | 1052 std::vector<std::string> browse_listnames; |
| 988 browse_listnames.push_back(safe_browsing_util::kMalwareList); | 1053 browse_listnames.push_back(safe_browsing_util::kMalwareList); |
| 989 browse_listnames.push_back(safe_browsing_util::kPhishingList); | 1054 browse_listnames.push_back(safe_browsing_util::kPhishingList); |
| 990 UpdateChunkRanges(browse_store_.get(), browse_listnames, lists); | 1055 UpdateChunkRanges(browse_store_.get(), browse_listnames, lists); |
| 991 | 1056 |
| 992 if (download_store_.get()) { | 1057 if (download_store_.get()) { |
| 993 std::vector<std::string> download_listnames; | 1058 std::vector<std::string> download_listnames; |
| 994 download_listnames.push_back(safe_browsing_util::kBinUrlList); | 1059 download_listnames.push_back(safe_browsing_util::kBinUrlList); |
| 995 download_listnames.push_back(safe_browsing_util::kBinHashList); | 1060 download_listnames.push_back(safe_browsing_util::kBinHashList); |
| 996 UpdateChunkRanges(download_store_.get(), download_listnames, lists); | 1061 UpdateChunkRanges(download_store_.get(), download_listnames, lists); |
| 997 } | 1062 } |
| 998 | 1063 |
| 999 if (csd_whitelist_store_.get()) { | 1064 if (csd_whitelist_store_.get()) { |
| 1000 std::vector<std::string> csd_whitelist_listnames; | 1065 std::vector<std::string> csd_whitelist_listnames; |
| 1001 csd_whitelist_listnames.push_back(safe_browsing_util::kCsdWhiteList); | 1066 csd_whitelist_listnames.push_back(safe_browsing_util::kCsdWhiteList); |
| 1002 UpdateChunkRanges(csd_whitelist_store_.get(), | 1067 UpdateChunkRanges(csd_whitelist_store_.get(), |
| 1003 csd_whitelist_listnames, lists); | 1068 csd_whitelist_listnames, lists); |
| 1004 } | 1069 } |
| 1005 | 1070 |
| 1071 if (download_whitelist_store_.get()) { |
| 1072 std::vector<std::string> download_whitelist_listnames; |
| 1073 download_whitelist_listnames.push_back( |
| 1074 safe_browsing_util::kDownloadWhiteList); |
| 1075 UpdateChunkRanges(download_whitelist_store_.get(), |
| 1076 download_whitelist_listnames, lists); |
| 1077 } |
| 1078 |
| 1006 corruption_detected_ = false; | 1079 corruption_detected_ = false; |
| 1007 change_detected_ = false; | 1080 change_detected_ = false; |
| 1008 return true; | 1081 return true; |
| 1009 } | 1082 } |
| 1010 | 1083 |
| 1011 void SafeBrowsingDatabaseNew::UpdateFinished(bool update_succeeded) { | 1084 void SafeBrowsingDatabaseNew::UpdateFinished(bool update_succeeded) { |
| 1012 DCHECK_EQ(creation_loop_, MessageLoop::current()); | 1085 DCHECK_EQ(creation_loop_, MessageLoop::current()); |
| 1013 if (corruption_detected_) | 1086 if (corruption_detected_) |
| 1014 return; | 1087 return; |
| 1015 | 1088 |
| 1016 // Unroll the transaction if there was a protocol error or if the | 1089 // Unroll the transaction if there was a protocol error or if the |
| 1017 // transaction was empty. This will leave the bloom filter, the | 1090 // transaction was empty. This will leave the bloom filter, the |
| 1018 // pending hashes, and the prefix miss cache in place. | 1091 // pending hashes, and the prefix miss cache in place. |
| 1019 if (!update_succeeded || !change_detected_) { | 1092 if (!update_succeeded || !change_detected_) { |
| 1020 // Track empty updates to answer questions at http://crbug.com/72216 . | 1093 // Track empty updates to answer questions at http://crbug.com/72216 . |
| 1021 if (update_succeeded && !change_detected_) | 1094 if (update_succeeded && !change_detected_) |
| 1022 UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", 0); | 1095 UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", 0); |
| 1023 browse_store_->CancelUpdate(); | 1096 browse_store_->CancelUpdate(); |
| 1024 if (download_store_.get()) | 1097 if (download_store_.get()) |
| 1025 download_store_->CancelUpdate(); | 1098 download_store_->CancelUpdate(); |
| 1026 if (csd_whitelist_store_.get()) | 1099 if (csd_whitelist_store_.get()) |
| 1027 csd_whitelist_store_->CancelUpdate(); | 1100 csd_whitelist_store_->CancelUpdate(); |
| 1101 if (download_whitelist_store_.get()) |
| 1102 download_whitelist_store_->CancelUpdate(); |
| 1028 return; | 1103 return; |
| 1029 } | 1104 } |
| 1030 | 1105 |
| 1031 // for download | 1106 // for download |
| 1032 UpdateDownloadStore(); | 1107 UpdateDownloadStore(); |
| 1033 // for browsing | 1108 // for browsing |
| 1034 UpdateBrowseStore(); | 1109 UpdateBrowseStore(); |
| 1035 // for csd whitelist | 1110 // for csd and download whitelists. |
| 1036 UpdateCsdWhitelistStore(); | 1111 UpdateWhitelistStore(csd_whitelist_filename_, |
| 1112 csd_whitelist_store_.get(), |
| 1113 &csd_whitelist_); |
| 1114 UpdateWhitelistStore(download_whitelist_filename_, |
| 1115 download_whitelist_store_.get(), |
| 1116 &download_whitelist_); |
| 1037 } | 1117 } |
| 1038 | 1118 |
| 1039 void SafeBrowsingDatabaseNew::UpdateCsdWhitelistStore() { | 1119 void SafeBrowsingDatabaseNew::UpdateWhitelistStore( |
| 1040 if (!csd_whitelist_store_.get()) | 1120 const FilePath& store_filename, |
| 1121 SafeBrowsingStore* store, |
| 1122 SBWhitelist* whitelist) { |
| 1123 if (!store) |
| 1041 return; | 1124 return; |
| 1042 | 1125 |
| 1043 // For the csd whitelist, we don't cache and save full hashes since all | 1126 // For the whitelists, we don't cache and save full hashes since all |
| 1044 // hashes are already full. | 1127 // hashes are already full. |
| 1045 std::vector<SBAddFullHash> empty_add_hashes; | 1128 std::vector<SBAddFullHash> empty_add_hashes; |
| 1046 | 1129 |
| 1047 // Not needed for the csd whitelist. | 1130 // Not needed for the whitelists. |
| 1048 std::set<SBPrefix> empty_miss_cache; | 1131 std::set<SBPrefix> empty_miss_cache; |
| 1049 | 1132 |
| 1050 // Note: prefixes will not be empty. The current data store implementation | 1133 // Note: prefixes will not be empty. The current data store implementation |
| 1051 // stores all full-length hashes as both full and prefix hashes. | 1134 // stores all full-length hashes as both full and prefix hashes. |
| 1052 std::vector<SBAddPrefix> prefixes; | 1135 std::vector<SBAddPrefix> prefixes; |
| 1053 std::vector<SBAddFullHash> full_hashes; | 1136 std::vector<SBAddFullHash> full_hashes; |
| 1054 if (!csd_whitelist_store_->FinishUpdate(empty_add_hashes, | 1137 if (!store->FinishUpdate(empty_add_hashes, empty_miss_cache, &prefixes, |
| 1055 empty_miss_cache, | 1138 &full_hashes)) { |
| 1056 &prefixes, | 1139 RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_FINISH); |
| 1057 &full_hashes)) { | 1140 WhitelistEverything(whitelist); |
| 1058 RecordFailure(FAILURE_CSD_WHITELIST_DATABASE_UPDATE_FINISH); | |
| 1059 CsdWhitelistAllUrls(); | |
| 1060 return; | 1141 return; |
| 1061 } | 1142 } |
| 1062 | 1143 |
| 1063 #if defined(OS_MACOSX) | 1144 #if defined(OS_MACOSX) |
| 1064 base::mac::SetFileBackupExclusion(csd_whitelist_filename_); | 1145 base::mac::SetFileBackupExclusion(store_filename); |
| 1065 #endif | 1146 #endif |
| 1066 | 1147 |
| 1067 LoadCsdWhitelist(full_hashes); | 1148 LoadWhitelist(full_hashes, whitelist); |
| 1068 } | 1149 } |
| 1069 | 1150 |
| 1070 void SafeBrowsingDatabaseNew::UpdateDownloadStore() { | 1151 void SafeBrowsingDatabaseNew::UpdateDownloadStore() { |
| 1071 if (!download_store_.get()) | 1152 if (!download_store_.get()) |
| 1072 return; | 1153 return; |
| 1073 | 1154 |
| 1074 // For download, we don't cache and save full hashes. | 1155 // For download, we don't cache and save full hashes. |
| 1075 std::vector<SBAddFullHash> empty_add_hashes; | 1156 std::vector<SBAddFullHash> empty_add_hashes; |
| 1076 | 1157 |
| 1077 // For download, backend lookup happens only if a prefix is in add list. | 1158 // For download, backend lookup happens only if a prefix is in add list. |
| (...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1272 | 1353 |
| 1273 const bool r2 = download_store_.get() ? download_store_->Delete() : true; | 1354 const bool r2 = download_store_.get() ? download_store_->Delete() : true; |
| 1274 if (!r2) | 1355 if (!r2) |
| 1275 RecordFailure(FAILURE_DATABASE_STORE_DELETE); | 1356 RecordFailure(FAILURE_DATABASE_STORE_DELETE); |
| 1276 | 1357 |
| 1277 const bool r3 = csd_whitelist_store_.get() ? | 1358 const bool r3 = csd_whitelist_store_.get() ? |
| 1278 csd_whitelist_store_->Delete() : true; | 1359 csd_whitelist_store_->Delete() : true; |
| 1279 if (!r3) | 1360 if (!r3) |
| 1280 RecordFailure(FAILURE_DATABASE_STORE_DELETE); | 1361 RecordFailure(FAILURE_DATABASE_STORE_DELETE); |
| 1281 | 1362 |
| 1282 const bool r4 = file_util::Delete(bloom_filter_filename_, false); | 1363 const bool r4 = download_whitelist_store_.get() ? |
| 1364 download_whitelist_store_->Delete() : true; |
| 1283 if (!r4) | 1365 if (!r4) |
| 1366 RecordFailure(FAILURE_DATABASE_STORE_DELETE); |
| 1367 |
| 1368 const bool r5 = file_util::Delete(bloom_filter_filename_, false); |
| 1369 if (!r5) |
| 1284 RecordFailure(FAILURE_DATABASE_FILTER_DELETE); | 1370 RecordFailure(FAILURE_DATABASE_FILTER_DELETE); |
| 1285 return r1 && r2 && r3 && r4; | 1371 return r1 && r2 && r3 && r4 && r5; |
| 1286 } | 1372 } |
| 1287 | 1373 |
| 1288 void SafeBrowsingDatabaseNew::WriteBloomFilter() { | 1374 void SafeBrowsingDatabaseNew::WriteBloomFilter() { |
| 1289 DCHECK_EQ(creation_loop_, MessageLoop::current()); | 1375 DCHECK_EQ(creation_loop_, MessageLoop::current()); |
| 1290 | 1376 |
| 1291 if (!browse_bloom_filter_.get()) | 1377 if (!browse_bloom_filter_.get()) |
| 1292 return; | 1378 return; |
| 1293 | 1379 |
| 1294 const base::TimeTicks before = base::TimeTicks::Now(); | 1380 const base::TimeTicks before = base::TimeTicks::Now(); |
| 1295 const bool write_ok = browse_bloom_filter_->WriteFile(bloom_filter_filename_); | 1381 const bool write_ok = browse_bloom_filter_->WriteFile(bloom_filter_filename_); |
| 1296 DVLOG(1) << "SafeBrowsingDatabaseNew wrote bloom filter in " | 1382 DVLOG(1) << "SafeBrowsingDatabaseNew wrote bloom filter in " |
| 1297 << (base::TimeTicks::Now() - before).InMilliseconds() << " ms"; | 1383 << (base::TimeTicks::Now() - before).InMilliseconds() << " ms"; |
| 1298 | 1384 |
| 1299 if (!write_ok) | 1385 if (!write_ok) |
| 1300 RecordFailure(FAILURE_DATABASE_FILTER_WRITE); | 1386 RecordFailure(FAILURE_DATABASE_FILTER_WRITE); |
| 1301 | 1387 |
| 1302 #if defined(OS_MACOSX) | 1388 #if defined(OS_MACOSX) |
| 1303 base::mac::SetFileBackupExclusion(bloom_filter_filename_); | 1389 base::mac::SetFileBackupExclusion(bloom_filter_filename_); |
| 1304 #endif | 1390 #endif |
| 1305 } | 1391 } |
| 1306 | 1392 |
| 1307 void SafeBrowsingDatabaseNew::CsdWhitelistAllUrls() { | 1393 void SafeBrowsingDatabaseNew::WhitelistEverything(SBWhitelist* whitelist) { |
| 1308 base::AutoLock locked(lookup_lock_); | 1394 base::AutoLock locked(lookup_lock_); |
| 1309 csd_whitelist_all_urls_ = true; | 1395 whitelist->second = true; |
| 1310 csd_whitelist_.clear(); | 1396 whitelist->first.clear(); |
| 1311 } | 1397 } |
| 1312 | 1398 |
| 1313 void SafeBrowsingDatabaseNew::LoadCsdWhitelist( | 1399 void SafeBrowsingDatabaseNew::LoadWhitelist( |
| 1314 const std::vector<SBAddFullHash>& full_hashes) { | 1400 const std::vector<SBAddFullHash>& full_hashes, |
| 1401 SBWhitelist* whitelist) { |
| 1315 DCHECK_EQ(creation_loop_, MessageLoop::current()); | 1402 DCHECK_EQ(creation_loop_, MessageLoop::current()); |
| 1316 if (full_hashes.size() > kMaxCsdWhitelistSize) { | 1403 if (full_hashes.size() > kMaxWhitelistSize) { |
| 1317 CsdWhitelistAllUrls(); | 1404 WhitelistEverything(whitelist); |
| 1318 return; | 1405 return; |
| 1319 } | 1406 } |
| 1320 | 1407 |
| 1321 std::vector<SBFullHash> new_csd_whitelist; | 1408 std::vector<SBFullHash> new_whitelist; |
| 1322 for (std::vector<SBAddFullHash>::const_iterator it = full_hashes.begin(); | 1409 for (std::vector<SBAddFullHash>::const_iterator it = full_hashes.begin(); |
| 1323 it != full_hashes.end(); ++it) { | 1410 it != full_hashes.end(); ++it) { |
| 1324 new_csd_whitelist.push_back(it->full_hash); | 1411 new_whitelist.push_back(it->full_hash); |
| 1325 } | 1412 } |
| 1326 std::sort(new_csd_whitelist.begin(), new_csd_whitelist.end()); | 1413 std::sort(new_whitelist.begin(), new_whitelist.end()); |
| 1327 | 1414 |
| 1328 SBFullHash kill_switch; | 1415 SBFullHash kill_switch; |
| 1329 crypto::SHA256HashString(kCsdKillSwitchUrl, &kill_switch, | 1416 crypto::SHA256HashString(kWhitelistKillSwitchUrl, &kill_switch, |
| 1330 sizeof(kill_switch)); | 1417 sizeof(kill_switch)); |
| 1331 if (std::binary_search(new_csd_whitelist.begin(), new_csd_whitelist.end(), | 1418 if (std::binary_search(new_whitelist.begin(), new_whitelist.end(), |
| 1332 kill_switch)) { | 1419 kill_switch)) { |
| 1333 // The kill switch is whitelisted hence we whitelist all URLs. | 1420 // The kill switch is whitelisted hence we whitelist all URLs. |
| 1334 CsdWhitelistAllUrls(); | 1421 WhitelistEverything(whitelist); |
| 1335 } else { | 1422 } else { |
| 1336 base::AutoLock locked(lookup_lock_); | 1423 base::AutoLock locked(lookup_lock_); |
| 1337 csd_whitelist_all_urls_ = false; | 1424 whitelist->second = false; |
| 1338 csd_whitelist_.swap(new_csd_whitelist); | 1425 whitelist->first.swap(new_whitelist); |
| 1339 } | 1426 } |
| 1340 } | 1427 } |
| OLD | NEW |