| OLD | NEW |
| 1 // Copyright 2016 The Chromium Authors. All rights reserved. | 1 // Copyright 2016 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "base/base64.h" | 5 #include "base/base64.h" |
| 6 #include "base/bind.h" | 6 #include "base/bind.h" |
| 7 #include "base/files/file_util.h" | 7 #include "base/files/file_util.h" |
| 8 #include "base/memory/ptr_util.h" | 8 #include "base/memory/ptr_util.h" |
| 9 #include "base/metrics/histogram_macros.h" | 9 #include "base/metrics/histogram_macros.h" |
| 10 #include "base/metrics/sparse_histogram.h" | 10 #include "base/metrics/sparse_histogram.h" |
| (...skipping 199 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 210 state_ = ""; | 210 state_ = ""; |
| 211 } | 211 } |
| 212 | 212 |
| 213 ApplyUpdateResult V4Store::ProcessPartialUpdateAndWriteToDisk( | 213 ApplyUpdateResult V4Store::ProcessPartialUpdateAndWriteToDisk( |
| 214 const std::string& metric, | 214 const std::string& metric, |
| 215 const HashPrefixMap& hash_prefix_map_old, | 215 const HashPrefixMap& hash_prefix_map_old, |
| 216 std::unique_ptr<ListUpdateResponse> response) { | 216 std::unique_ptr<ListUpdateResponse> response) { |
| 217 DCHECK(response->has_response_type()); | 217 DCHECK(response->has_response_type()); |
| 218 DCHECK_EQ(ListUpdateResponse::PARTIAL_UPDATE, response->response_type()); | 218 DCHECK_EQ(ListUpdateResponse::PARTIAL_UPDATE, response->response_type()); |
| 219 | 219 |
| 220 ApplyUpdateResult result = | 220 ApplyUpdateResult result = ProcessUpdate( |
| 221 ProcessUpdate(metric, hash_prefix_map_old, response); | 221 metric, hash_prefix_map_old, response, false /* delay_checksum check */); |
| 222 if (result == APPLY_UPDATE_SUCCESS) { | 222 if (result == APPLY_UPDATE_SUCCESS) { |
| 223 Checksum checksum = response->checksum(); | 223 Checksum checksum = response->checksum(); |
| 224 response.reset(); | 224 response.reset(); |
| 225 RecordStoreWriteResult(WriteToDisk(checksum)); | 225 RecordStoreWriteResult(WriteToDisk(checksum)); |
| 226 } | 226 } |
| 227 return result; | 227 return result; |
| 228 } | 228 } |
| 229 | 229 |
| 230 ApplyUpdateResult V4Store::ProcessFullUpdateAndWriteToDisk( | 230 ApplyUpdateResult V4Store::ProcessFullUpdateAndWriteToDisk( |
| 231 const std::string& metric, | 231 const std::string& metric, |
| 232 std::unique_ptr<ListUpdateResponse> response) { | 232 std::unique_ptr<ListUpdateResponse> response) { |
| 233 ApplyUpdateResult result = ProcessFullUpdate(metric, response); | 233 ApplyUpdateResult result = |
| 234 ProcessFullUpdate(metric, response, false /* delay_checksum check */); |
| 234 if (result == APPLY_UPDATE_SUCCESS) { | 235 if (result == APPLY_UPDATE_SUCCESS) { |
| 235 Checksum checksum = response->checksum(); | 236 Checksum checksum = response->checksum(); |
| 236 response.reset(); | 237 response.reset(); |
| 237 RecordStoreWriteResult(WriteToDisk(checksum)); | 238 RecordStoreWriteResult(WriteToDisk(checksum)); |
| 238 } | 239 } |
| 239 return result; | 240 return result; |
| 240 } | 241 } |
| 241 | 242 |
| 242 ApplyUpdateResult V4Store::ProcessFullUpdate( | 243 ApplyUpdateResult V4Store::ProcessFullUpdate( |
| 243 const std::string& metric, | 244 const std::string& metric, |
| 244 const std::unique_ptr<ListUpdateResponse>& response) { | 245 const std::unique_ptr<ListUpdateResponse>& response, |
| 246 bool delay_checksum_check) { |
| 245 DCHECK(response->has_response_type()); | 247 DCHECK(response->has_response_type()); |
| 246 DCHECK_EQ(ListUpdateResponse::FULL_UPDATE, response->response_type()); | 248 DCHECK_EQ(ListUpdateResponse::FULL_UPDATE, response->response_type()); |
| 247 // TODO(vakh): For a full update, we don't need to process the update in | 249 // TODO(vakh): For a full update, we don't need to process the update in |
| 248 // lexographical order to store it, but we do need to do that for calculating | 250 // lexographical order to store it, but we do need to do that for calculating |
| 249 // checksum. It might save some CPU cycles to store the full update as-is and | 251 // checksum. It might save some CPU cycles to store the full update as-is and |
| 250 // walk the list of hash prefixes in lexographical order only for checksum | 252 // walk the list of hash prefixes in lexographical order only for checksum |
| 251 // calculation. | 253 // calculation. |
| 252 return ProcessUpdate(metric, HashPrefixMap(), response); | 254 return ProcessUpdate(metric, HashPrefixMap(), response, delay_checksum_check); |
| 253 } | 255 } |
| 254 | 256 |
| 255 ApplyUpdateResult V4Store::ProcessUpdate( | 257 ApplyUpdateResult V4Store::ProcessUpdate( |
| 256 const std::string& metric, | 258 const std::string& metric, |
| 257 const HashPrefixMap& hash_prefix_map_old, | 259 const HashPrefixMap& hash_prefix_map_old, |
| 258 const std::unique_ptr<ListUpdateResponse>& response) { | 260 const std::unique_ptr<ListUpdateResponse>& response, |
| 261 bool delay_checksum_check) { |
| 259 const RepeatedField<int32>* raw_removals = nullptr; | 262 const RepeatedField<int32>* raw_removals = nullptr; |
| 260 RepeatedField<int32> rice_removals; | 263 RepeatedField<int32> rice_removals; |
| 261 size_t removals_size = response->removals_size(); | 264 size_t removals_size = response->removals_size(); |
| 262 DCHECK_LE(removals_size, 1u); | 265 DCHECK_LE(removals_size, 1u); |
| 263 if (removals_size == 1) { | 266 if (removals_size == 1) { |
| 264 const ThreatEntrySet& removal = response->removals().Get(0); | 267 const ThreatEntrySet& removal = response->removals().Get(0); |
| 265 const CompressionType compression_type = removal.compression_type(); | 268 const CompressionType compression_type = removal.compression_type(); |
| 266 if (compression_type == RAW) { | 269 if (compression_type == RAW) { |
| 267 raw_removals = &removal.raw_indices().indices(); | 270 raw_removals = &removal.raw_indices().indices(); |
| 268 } else if (compression_type == RICE) { | 271 } else if (compression_type == RICE) { |
| (...skipping 24 matching lines...) Expand all Loading... |
| 293 if (apply_update_result != APPLY_UPDATE_SUCCESS) { | 296 if (apply_update_result != APPLY_UPDATE_SUCCESS) { |
| 294 return apply_update_result; | 297 return apply_update_result; |
| 295 } | 298 } |
| 296 | 299 |
| 297 std::string expected_checksum; | 300 std::string expected_checksum; |
| 298 if (response->has_checksum() && response->checksum().has_sha256()) { | 301 if (response->has_checksum() && response->checksum().has_sha256()) { |
| 299 expected_checksum = response->checksum().sha256(); | 302 expected_checksum = response->checksum().sha256(); |
| 300 } | 303 } |
| 301 | 304 |
| 302 TimeTicks before = TimeTicks::Now(); | 305 TimeTicks before = TimeTicks::Now(); |
| 303 apply_update_result = MergeUpdate(hash_prefix_map_old, hash_prefix_map, | 306 if (delay_checksum_check) { |
| 304 raw_removals, expected_checksum); | 307 DCHECK(hash_prefix_map_old.empty()); |
| 305 if (apply_update_result != APPLY_UPDATE_SUCCESS) { | 308 DCHECK(!raw_removals); |
| 306 return apply_update_result; | 309 // We delay the checksum check at startup to be able to load the DB |
| 310 // quickly. In this case, the |hash_prefix_map_old| should be empty, so just |
| 311 // copy over the |hash_prefix_map|. |
| 312 hash_prefix_map_ = hash_prefix_map; |
| 313 |
| 314 // Calculate the checksum asynchronously later and if it doesn't match, |
| 315 // reset the store. |
| 316 expected_checksum_ = expected_checksum; |
| 317 |
| 318 apply_update_result = APPLY_UPDATE_SUCCESS; |
| 319 } else { |
| 320 apply_update_result = MergeUpdate(hash_prefix_map_old, hash_prefix_map, |
| 321 raw_removals, expected_checksum); |
| 322 if (apply_update_result != APPLY_UPDATE_SUCCESS) { |
| 323 return apply_update_result; |
| 324 } |
| 307 } | 325 } |
| 308 RecordMergeUpdateTime(metric, TimeTicks::Now() - before, store_path_); | 326 RecordMergeUpdateTime(metric, TimeTicks::Now() - before, store_path_); |
| 309 | 327 |
| 310 state_ = response->new_client_state(); | 328 state_ = response->new_client_state(); |
| 311 return APPLY_UPDATE_SUCCESS; | 329 return APPLY_UPDATE_SUCCESS; |
| 312 } | 330 } |
| 313 | 331 |
| 314 void V4Store::ApplyUpdate( | 332 void V4Store::ApplyUpdate( |
| 315 std::unique_ptr<ListUpdateResponse> response, | 333 std::unique_ptr<ListUpdateResponse> response, |
| 316 const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, | 334 const scoped_refptr<base::SingleThreadTaskRunner>& callback_task_runner, |
| (...skipping 172 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 489 const std::string& expected_checksum) { | 507 const std::string& expected_checksum) { |
| 490 DCHECK(task_runner_->RunsTasksOnCurrentThread()); | 508 DCHECK(task_runner_->RunsTasksOnCurrentThread()); |
| 491 DCHECK(hash_prefix_map_.empty()); | 509 DCHECK(hash_prefix_map_.empty()); |
| 492 | 510 |
| 493 bool calculate_checksum = !expected_checksum.empty(); | 511 bool calculate_checksum = !expected_checksum.empty(); |
| 494 if (calculate_checksum && | 512 if (calculate_checksum && |
| 495 (expected_checksum.size() != crypto::kSHA256Length)) { | 513 (expected_checksum.size() != crypto::kSHA256Length)) { |
| 496 return CHECKSUM_MISMATCH_FAILURE; | 514 return CHECKSUM_MISMATCH_FAILURE; |
| 497 } | 515 } |
| 498 | 516 |
| 499 if (old_prefixes_map.empty()) { | |
| 500 // If the old map is empty, which it is at startup, then just copy over the | |
| 501 // additions map. | |
| 502 DCHECK(!raw_removals); | |
| 503 hash_prefix_map_ = additions_map; | |
| 504 | |
| 505 // Calculate the checksum asynchronously later and if it doesn't match, | |
| 506 // reset the store. | |
| 507 expected_checksum_ = expected_checksum; | |
| 508 | |
| 509 return APPLY_UPDATE_SUCCESS; | |
| 510 } | |
| 511 | |
| 512 hash_prefix_map_.clear(); | 517 hash_prefix_map_.clear(); |
| 513 ReserveSpaceInPrefixMap(old_prefixes_map, &hash_prefix_map_); | 518 ReserveSpaceInPrefixMap(old_prefixes_map, &hash_prefix_map_); |
| 514 ReserveSpaceInPrefixMap(additions_map, &hash_prefix_map_); | 519 ReserveSpaceInPrefixMap(additions_map, &hash_prefix_map_); |
| 515 | 520 |
| 516 IteratorMap old_iterator_map; | 521 IteratorMap old_iterator_map; |
| 517 HashPrefix next_smallest_prefix_old; | 522 HashPrefix next_smallest_prefix_old; |
| 518 InitializeIteratorMap(old_prefixes_map, &old_iterator_map); | 523 InitializeIteratorMap(old_prefixes_map, &old_iterator_map); |
| 519 bool old_has_unmerged = GetNextSmallestUnmergedPrefix( | 524 bool old_has_unmerged = GetNextSmallestUnmergedPrefix( |
| 520 old_prefixes_map, old_iterator_map, &next_smallest_prefix_old); | 525 old_prefixes_map, old_iterator_map, &next_smallest_prefix_old); |
| 521 | 526 |
| (...skipping 141 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 663 if (file_format.version_number() != kFileVersion) { | 668 if (file_format.version_number() != kFileVersion) { |
| 664 return FILE_VERSION_INCOMPATIBLE_FAILURE; | 669 return FILE_VERSION_INCOMPATIBLE_FAILURE; |
| 665 } | 670 } |
| 666 | 671 |
| 667 if (!file_format.has_list_update_response()) { | 672 if (!file_format.has_list_update_response()) { |
| 668 return HASH_PREFIX_INFO_MISSING_FAILURE; | 673 return HASH_PREFIX_INFO_MISSING_FAILURE; |
| 669 } | 674 } |
| 670 | 675 |
| 671 std::unique_ptr<ListUpdateResponse> response(new ListUpdateResponse); | 676 std::unique_ptr<ListUpdateResponse> response(new ListUpdateResponse); |
| 672 response->Swap(file_format.mutable_list_update_response()); | 677 response->Swap(file_format.mutable_list_update_response()); |
| 673 ApplyUpdateResult apply_update_result = | 678 ApplyUpdateResult apply_update_result = ProcessFullUpdate( |
| 674 ProcessFullUpdate(kReadFromDisk, response); | 679 kReadFromDisk, response, true /* delay_checksum check */); |
| 675 RecordApplyUpdateResult(kReadFromDisk, apply_update_result, store_path_); | 680 RecordApplyUpdateResult(kReadFromDisk, apply_update_result, store_path_); |
| 676 if (apply_update_result != APPLY_UPDATE_SUCCESS) { | 681 if (apply_update_result != APPLY_UPDATE_SUCCESS) { |
| 677 hash_prefix_map_.clear(); | 682 hash_prefix_map_.clear(); |
| 678 return HASH_PREFIX_MAP_GENERATION_FAILURE; | 683 return HASH_PREFIX_MAP_GENERATION_FAILURE; |
| 679 } | 684 } |
| 680 RecordApplyUpdateTime(kReadFromDisk, TimeTicks::Now() - before, store_path_); | 685 RecordApplyUpdateTime(kReadFromDisk, TimeTicks::Now() - before, store_path_); |
| 681 | 686 |
| 682 return READ_SUCCESS; | 687 return READ_SUCCESS; |
| 683 } | 688 } |
| 684 | 689 |
| (...skipping 119 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 804 << "; expected: " << expected_checksum_b64 | 809 << "; expected: " << expected_checksum_b64 |
| 805 << "; store: " << *this; | 810 << "; store: " << *this; |
| 806 #endif | 811 #endif |
| 807 return false; | 812 return false; |
| 808 } | 813 } |
| 809 } | 814 } |
| 810 return true; | 815 return true; |
| 811 } | 816 } |
| 812 | 817 |
| 813 } // namespace safe_browsing | 818 } // namespace safe_browsing |
| OLD | NEW |