| OLD | NEW |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "components/ntp_snippets/ntp_snippets_service.h" | 5 #include "components/ntp_snippets/ntp_snippets_service.h" |
| 6 | 6 |
| 7 #include <algorithm> | 7 #include <algorithm> |
| 8 #include <iterator> | 8 #include <iterator> |
| 9 #include <utility> | 9 #include <utility> |
| 10 | 10 |
| (...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 43 using suggestions::SuggestionsService; | 43 using suggestions::SuggestionsService; |
| 44 | 44 |
| 45 namespace ntp_snippets { | 45 namespace ntp_snippets { |
| 46 | 46 |
| 47 namespace { | 47 namespace { |
| 48 | 48 |
| 49 // Number of snippets requested to the server. Consider replacing sparse UMA | 49 // Number of snippets requested to the server. Consider replacing sparse UMA |
| 50 // histograms with COUNTS() if this number increases beyond 50. | 50 // histograms with COUNTS() if this number increases beyond 50. |
| 51 const int kMaxSnippetCount = 10; | 51 const int kMaxSnippetCount = 10; |
| 52 | 52 |
| 53 // Number of archived snippets we keep around in memory. |
| 54 const int kMaxArchivedSnippetCount = 200; |
| 55 |
| 53 // Default values for snippets fetching intervals - once per day only. | 56 // Default values for snippets fetching intervals - once per day only. |
| 54 const int kDefaultFetchingIntervalWifiSeconds = 0; | 57 const int kDefaultFetchingIntervalWifiSeconds = 0; |
| 55 const int kDefaultFetchingIntervalFallbackSeconds = 24 * 60 * 60; | 58 const int kDefaultFetchingIntervalFallbackSeconds = 24 * 60 * 60; |
| 56 | 59 |
| 57 // Variation parameters than can override the default fetching intervals. | 60 // Variation parameters than can override the default fetching intervals. |
| 58 const char kFetchingIntervalWifiParamName[] = | 61 const char kFetchingIntervalWifiParamName[] = |
| 59 "fetching_interval_wifi_seconds"; | 62 "fetching_interval_wifi_seconds"; |
| 60 const char kFetchingIntervalFallbackParamName[] = | 63 const char kFetchingIntervalFallbackParamName[] = |
| 61 "fetching_interval_fallback_seconds"; | 64 "fetching_interval_fallback_seconds"; |
| 62 | 65 |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 111 std::set<std::string> hosts; | 114 std::set<std::string> hosts; |
| 112 for (int i = 0; i < suggestions.suggestions_size(); ++i) { | 115 for (int i = 0; i < suggestions.suggestions_size(); ++i) { |
| 113 const ChromeSuggestion& suggestion = suggestions.suggestions(i); | 116 const ChromeSuggestion& suggestion = suggestions.suggestions(i); |
| 114 GURL url(suggestion.url()); | 117 GURL url(suggestion.url()); |
| 115 if (url.is_valid()) | 118 if (url.is_valid()) |
| 116 hosts.insert(url.host()); | 119 hosts.insert(url.host()); |
| 117 } | 120 } |
| 118 return hosts; | 121 return hosts; |
| 119 } | 122 } |
| 120 | 123 |
| 121 void InsertAllIDs(const NTPSnippet::PtrVector& snippets, | 124 std::set<std::string> GetAllIDs(const NTPSnippet::PtrVector& snippets) { |
| 122 std::set<std::string>* ids) { | 125 std::set<std::string> ids; |
| 123 for (const std::unique_ptr<NTPSnippet>& snippet : snippets) { | 126 for (const std::unique_ptr<NTPSnippet>& snippet : snippets) { |
| 124 ids->insert(snippet->id()); | 127 ids.insert(snippet->id()); |
| 125 for (const SnippetSource& source : snippet->sources()) | 128 for (const SnippetSource& source : snippet->sources()) |
| 126 ids->insert(source.url.spec()); | 129 ids.insert(source.url.spec()); |
| 127 } | 130 } |
| 131 return ids; |
| 132 } |
| 133 |
| 134 std::set<std::string> GetMainIDs(const NTPSnippet::PtrVector& snippets) { |
| 135 std::set<std::string> ids; |
| 136 for (const std::unique_ptr<NTPSnippet>& snippet : snippets) |
| 137 ids.insert(snippet->id()); |
| 138 return ids; |
| 139 } |
| 140 |
| 141 bool IsSnippetInSet(const std::unique_ptr<NTPSnippet>& snippet, |
| 142 const std::set<std::string>& ids, |
| 143 bool match_all_ids) { |
| 144 if (ids.count(snippet->id())) |
| 145 return true; |
| 146 if (!match_all_ids) |
| 147 return false; |
| 148 for (const SnippetSource& source : snippet->sources()) { |
| 149 if (ids.count(source.url.spec())) |
| 150 return true; |
| 151 } |
| 152 return false; |
| 153 } |
| 154 |
| 155 void EraseMatchingSnippets(NTPSnippet::PtrVector* snippets, |
| 156 const std::set<std::string>& matching_ids, |
| 157 bool match_all_ids) { |
| 158 snippets->erase( |
| 159 std::remove_if(snippets->begin(), snippets->end(), |
| 160 [&matching_ids, match_all_ids]( |
| 161 const std::unique_ptr<NTPSnippet>& snippet) { |
| 162 return IsSnippetInSet(snippet, matching_ids, |
| 163 match_all_ids); |
| 164 }), |
| 165 snippets->end()); |
| 128 } | 166 } |
| 129 | 167 |
| 130 void Compact(NTPSnippet::PtrVector* snippets) { | 168 void Compact(NTPSnippet::PtrVector* snippets) { |
| 131 snippets->erase( | 169 snippets->erase( |
| 132 std::remove_if( | 170 std::remove_if( |
| 133 snippets->begin(), snippets->end(), | 171 snippets->begin(), snippets->end(), |
| 134 [](const std::unique_ptr<NTPSnippet>& snippet) { return !snippet; }), | 172 [](const std::unique_ptr<NTPSnippet>& snippet) { return !snippet; }), |
| 135 snippets->end()); | 173 snippets->end()); |
| 136 } | 174 } |
| 137 | 175 |
| (...skipping 178 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 316 void NTPSnippetsService::ClearCachedSuggestions(Category category) { | 354 void NTPSnippetsService::ClearCachedSuggestions(Category category) { |
| 317 if (!initialized()) | 355 if (!initialized()) |
| 318 return; | 356 return; |
| 319 | 357 |
| 320 if (categories_.find(category) == categories_.end()) | 358 if (categories_.find(category) == categories_.end()) |
| 321 return; | 359 return; |
| 322 CategoryContent* content = &categories_[category]; | 360 CategoryContent* content = &categories_[category]; |
| 323 if (content->snippets.empty()) | 361 if (content->snippets.empty()) |
| 324 return; | 362 return; |
| 325 | 363 |
| 326 if (category == articles_category_) | 364 if (category == articles_category_) { |
| 327 database_->DeleteSnippets(content->snippets); | 365 database_->DeleteSnippets(content->snippets); |
| 366 database_->DeleteImages(content->snippets); |
| 367 } |
| 328 content->snippets.clear(); | 368 content->snippets.clear(); |
| 329 | 369 |
| 330 NotifyNewSuggestions(); | 370 NotifyNewSuggestions(); |
| 331 } | 371 } |
| 332 | 372 |
| 333 void NTPSnippetsService::GetDismissedSuggestionsForDebugging( | 373 void NTPSnippetsService::GetDismissedSuggestionsForDebugging( |
| 334 Category category, | 374 Category category, |
| 335 const DismissedSuggestionsCallback& callback) { | 375 const DismissedSuggestionsCallback& callback) { |
| 336 DCHECK(categories_.find(category) != categories_.end()); | 376 DCHECK(categories_.find(category) != categories_.end()); |
| 337 | 377 |
| (...skipping 20 matching lines...) Expand all Loading... |
| 358 Category category) { | 398 Category category) { |
| 359 DCHECK(categories_.find(category) != categories_.end()); | 399 DCHECK(categories_.find(category) != categories_.end()); |
| 360 | 400 |
| 361 if (!initialized()) | 401 if (!initialized()) |
| 362 return; | 402 return; |
| 363 | 403 |
| 364 CategoryContent* content = &categories_[category]; | 404 CategoryContent* content = &categories_[category]; |
| 365 if (content->dismissed.empty()) | 405 if (content->dismissed.empty()) |
| 366 return; | 406 return; |
| 367 | 407 |
| 368 if (category == articles_category_) | 408 if (category == articles_category_) { |
| 409 // The image got already deleted when the suggestion was dismissed. |
| 369 database_->DeleteSnippets(content->dismissed); | 410 database_->DeleteSnippets(content->dismissed); |
| 411 } |
| 370 content->dismissed.clear(); | 412 content->dismissed.clear(); |
| 371 } | 413 } |
| 372 | 414 |
| 373 std::set<std::string> NTPSnippetsService::GetSuggestionsHosts() const { | 415 std::set<std::string> NTPSnippetsService::GetSuggestionsHosts() const { |
| 374 // |suggestions_service_| can be null in tests. | 416 // |suggestions_service_| can be null in tests. |
| 375 if (!suggestions_service_) | 417 if (!suggestions_service_) |
| 376 return std::set<std::string>(); | 418 return std::set<std::string>(); |
| 377 | 419 |
| 378 // TODO(treib): This should just call GetSnippetHostsFromPrefs. | 420 // TODO(treib): This should just call GetSnippetHostsFromPrefs. |
| 379 return GetSuggestionsHostsImpl( | 421 return GetSuggestionsHostsImpl( |
| 380 suggestions_service_->GetSuggestionsDataFromCache()); | 422 suggestions_service_->GetSuggestionsDataFromCache()); |
| 381 } | 423 } |
| 382 | 424 |
| 383 // static | 425 // static |
| 384 int NTPSnippetsService::GetMaxSnippetCountForTesting() { | 426 int NTPSnippetsService::GetMaxSnippetCountForTesting() { |
| 385 return kMaxSnippetCount; | 427 return kMaxSnippetCount; |
| 386 } | 428 } |
| 387 | 429 |
| 388 //////////////////////////////////////////////////////////////////////////////// | 430 //////////////////////////////////////////////////////////////////////////////// |
| 389 // Private methods | 431 // Private methods |
| 390 | 432 |
| 433 GURL NTPSnippetsService::FindSnippetImageUrl( |
| 434 Category category, |
| 435 const std::string& snippet_id) const { |
| 436 DCHECK(categories_.find(category) != categories_.end()); |
| 437 |
| 438 const CategoryContent& content = categories_.at(category); |
| 439 // Search for the snippet in current and archived snippets. |
| 440 auto it = |
| 441 std::find_if(content.snippets.begin(), content.snippets.end(), |
| 442 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) { |
| 443 return snippet->id() == snippet_id; |
| 444 }); |
| 445 if (it != content.snippets.end()) |
| 446 return (*it)->salient_image_url(); |
| 447 |
| 448 it = std::find_if(content.archived.begin(), content.archived.end(), |
| 449 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) { |
| 450 return snippet->id() == snippet_id; |
| 451 }); |
| 452 if (it != content.archived.end()) |
| 453 return (*it)->salient_image_url(); |
| 454 |
| 455 return GURL(); |
| 456 } |
| 457 |
| 391 // image_fetcher::ImageFetcherDelegate implementation. | 458 // image_fetcher::ImageFetcherDelegate implementation. |
| 392 void NTPSnippetsService::OnImageDataFetched(const std::string& suggestion_id, | 459 void NTPSnippetsService::OnImageDataFetched(const std::string& suggestion_id, |
| 393 const std::string& image_data) { | 460 const std::string& image_data) { |
| 394 if (image_data.empty()) | 461 if (image_data.empty()) |
| 395 return; | 462 return; |
| 396 | 463 |
| 397 Category category = GetCategoryFromUniqueID(suggestion_id); | 464 Category category = GetCategoryFromUniqueID(suggestion_id); |
| 398 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id); | 465 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id); |
| 399 | 466 |
| 400 auto category_it = categories_.find(category); | 467 if (categories_.find(category) == categories_.end()) |
| 401 if (category_it == categories_.end()) | |
| 402 return; | 468 return; |
| 403 | 469 |
| 404 const CategoryContent& content = category_it->second; | |
| 405 | |
| 406 // Only save the image if the corresponding snippet still exists. | 470 // Only save the image if the corresponding snippet still exists. |
| 407 auto it = | 471 if (FindSnippetImageUrl(category, snippet_id).is_empty()) |
| 408 std::find_if(content.snippets.begin(), content.snippets.end(), | |
| 409 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) { | |
| 410 return snippet->id() == snippet_id; | |
| 411 }); | |
| 412 if (it == content.snippets.end()) | |
| 413 return; | 472 return; |
| 414 | 473 |
| 474 // Only cache the data in the DB, the actual serving is done in the callback |
| 475 // provided to |image_fetcher_| (OnSnippetImageDecodedFromNetwork()). |
| 415 database_->SaveImage(snippet_id, image_data); | 476 database_->SaveImage(snippet_id, image_data); |
| 416 } | 477 } |
| 417 | 478 |
| 418 void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) { | 479 void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) { |
| 419 if (state_ == State::ERROR_OCCURRED) | 480 if (state_ == State::ERROR_OCCURRED) |
| 420 return; | 481 return; |
| 421 DCHECK(state_ == State::NOT_INITED); | 482 DCHECK(state_ == State::NOT_INITED); |
| 422 DCHECK(categories_.size() == 1); // Only articles category, so far. | 483 DCHECK_EQ(1u, categories_.size()); // Only articles category, so far. |
| 423 DCHECK(categories_.find(articles_category_) != categories_.end()); | 484 DCHECK(categories_.find(articles_category_) != categories_.end()); |
| 424 | 485 |
| 425 // TODO(sfiera): support non-article categories in database. | 486 // TODO(sfiera): support non-article categories in database. |
| 426 CategoryContent* content = &categories_[articles_category_]; | 487 CategoryContent* content = &categories_[articles_category_]; |
| 427 for (std::unique_ptr<NTPSnippet>& snippet : snippets) { | 488 for (std::unique_ptr<NTPSnippet>& snippet : snippets) { |
| 428 if (snippet->is_dismissed()) | 489 if (snippet->is_dismissed()) |
| 429 content->dismissed.emplace_back(std::move(snippet)); | 490 content->dismissed.emplace_back(std::move(snippet)); |
| 430 else | 491 else |
| 431 content->snippets.emplace_back(std::move(snippet)); | 492 content->snippets.emplace_back(std::move(snippet)); |
| 432 } | 493 } |
| 433 | 494 |
| 434 std::sort(content->snippets.begin(), content->snippets.end(), | 495 std::sort(content->snippets.begin(), content->snippets.end(), |
| 435 [](const std::unique_ptr<NTPSnippet>& lhs, | 496 [](const std::unique_ptr<NTPSnippet>& lhs, |
| 436 const std::unique_ptr<NTPSnippet>& rhs) { | 497 const std::unique_ptr<NTPSnippet>& rhs) { |
| 437 return lhs->score() > rhs->score(); | 498 return lhs->score() > rhs->score(); |
| 438 }); | 499 }); |
| 439 | 500 |
| 440 ClearExpiredSnippets(); | 501 ClearExpiredDismissedSnippets(); |
| 502 ClearOrphanedImages(); |
| 441 FinishInitialization(); | 503 FinishInitialization(); |
| 442 } | 504 } |
| 443 | 505 |
| 444 void NTPSnippetsService::OnDatabaseError() { | 506 void NTPSnippetsService::OnDatabaseError() { |
| 445 EnterState(State::ERROR_OCCURRED); | 507 EnterState(State::ERROR_OCCURRED); |
| 446 UpdateAllCategoryStatus(CategoryStatus::LOADING_ERROR); | 508 UpdateAllCategoryStatus(CategoryStatus::LOADING_ERROR); |
| 447 } | 509 } |
| 448 | 510 |
| 449 // TODO(dgn): name clash between content suggestions and suggestions hosts. | 511 // TODO(dgn): name clash between content suggestions and suggestions hosts. |
| 450 // method name should be changed. | 512 // method name should be changed. |
| (...skipping 14 matching lines...) Expand all Loading... |
| 465 // and apply the same logic to them here. Maybe never? | 527 // and apply the same logic to them here. Maybe never? |
| 466 // | 528 // |
| 467 // First, move them over into |to_delete|. | 529 // First, move them over into |to_delete|. |
| 468 CategoryContent* content = &categories_[articles_category_]; | 530 CategoryContent* content = &categories_[articles_category_]; |
| 469 NTPSnippet::PtrVector to_delete; | 531 NTPSnippet::PtrVector to_delete; |
| 470 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) { | 532 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) { |
| 471 if (!hosts.count(snippet->best_source().url.host())) | 533 if (!hosts.count(snippet->best_source().url.host())) |
| 472 to_delete.emplace_back(std::move(snippet)); | 534 to_delete.emplace_back(std::move(snippet)); |
| 473 } | 535 } |
| 474 Compact(&content->snippets); | 536 Compact(&content->snippets); |
| 475 // Then delete the removed snippets from the database. | 537 ArchiveSnippets(articles_category_, &to_delete); |
| 476 database_->DeleteSnippets(to_delete); | |
| 477 | 538 |
| 478 StoreSnippetHostsToPrefs(hosts); | 539 StoreSnippetHostsToPrefs(hosts); |
| 479 | 540 |
| 480 // We removed some suggestions, so we want to let the client know about that. | 541 // We removed some suggestions, so we want to let the client know about that. |
| 481 // The fetch might take a long time or not complete so we don't want to wait | 542 // The fetch might take a long time or not complete so we don't want to wait |
| 482 // for its callback. | 543 // for its callback. |
| 483 NotifyNewSuggestions(); | 544 NotifyNewSuggestions(); |
| 484 | 545 |
| 485 FetchSnippetsFromHosts(hosts, /*force_request=*/false); | 546 FetchSnippetsFromHosts(hosts, /*force_request=*/false); |
| 486 } | 547 } |
| 487 | 548 |
| 488 void NTPSnippetsService::OnFetchFinished( | 549 void NTPSnippetsService::OnFetchFinished( |
| 489 NTPSnippetsFetcher::OptionalSnippets snippets) { | 550 NTPSnippetsFetcher::OptionalSnippets snippets) { |
| 490 if (!ready()) | 551 if (!ready()) |
| 491 return; | 552 return; |
| 492 | 553 |
| 493 for (auto& item : categories_) { | 554 for (auto& item : categories_) { |
| 494 CategoryContent* content = &item.second; | 555 CategoryContent* content = &item.second; |
| 495 content->provided_by_server = false; | 556 content->provided_by_server = false; |
| 496 } | 557 } |
| 497 | 558 |
| 559 // Clear up expired dismissed snippets before we use them to filter new ones. |
| 560 ClearExpiredDismissedSnippets(); |
| 561 |
| 498 // If snippets were fetched successfully, update our |categories_| from each | 562 // If snippets were fetched successfully, update our |categories_| from each |
| 499 // category provided by the server. | 563 // category provided by the server. |
| 500 if (snippets) { | 564 if (snippets) { |
| 565 // TODO(jkrcal): A bit hard to understand with so many variables called |
| 566 // "*categor*". Isn't here some room for simplification? |
| 501 for (NTPSnippetsFetcher::FetchedCategory& fetched_category : *snippets) { | 567 for (NTPSnippetsFetcher::FetchedCategory& fetched_category : *snippets) { |
| 502 Category category = fetched_category.category; | 568 Category category = fetched_category.category; |
| 503 | 569 |
| 504 // TODO(sfiera): Avoid hard-coding articles category checks in so many | 570 // TODO(sfiera): Avoid hard-coding articles category checks in so many |
| 505 // places. | 571 // places. |
| 506 if (category != articles_category_) { | 572 if (category != articles_category_) { |
| 507 // Only update titles from server-side provided categories. | 573 // Only update titles from server-side provided categories. |
| 508 categories_[category].localized_title = | 574 categories_[category].localized_title = |
| 509 fetched_category.localized_title; | 575 fetched_category.localized_title; |
| 510 } | 576 } |
| 577 categories_[category].provided_by_server = true; |
| 511 | 578 |
| 512 DCHECK_LE(snippets->size(), static_cast<size_t>(kMaxSnippetCount)); | 579 DCHECK_LE(snippets->size(), static_cast<size_t>(kMaxSnippetCount)); |
| 513 // TODO(sfiera): histograms for server categories. | 580 // TODO(sfiera): histograms for server categories. |
| 514 // Sparse histogram used because the number of snippets is small (bound by | 581 // Sparse histogram used because the number of snippets is small (bound by |
| 515 // kMaxSnippetCount). | 582 // kMaxSnippetCount). |
| 516 if (category == articles_category_) { | 583 if (category == articles_category_) { |
| 517 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticlesFetched", | 584 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticlesFetched", |
| 518 fetched_category.snippets.size()); | 585 fetched_category.snippets.size()); |
| 519 } | 586 } |
| 520 | 587 |
| 521 MergeSnippets(category, std::move(fetched_category.snippets)); | 588 ReplaceSnippets(category, std::move(fetched_category.snippets)); |
| 522 | |
| 523 // If there are more snippets than we want to show, delete the extra ones. | |
| 524 CategoryContent* content = &categories_[category]; | |
| 525 content->provided_by_server = true; | |
| 526 if (content->snippets.size() > kMaxSnippetCount) { | |
| 527 NTPSnippet::PtrVector to_delete( | |
| 528 std::make_move_iterator(content->snippets.begin() + | |
| 529 kMaxSnippetCount), | |
| 530 std::make_move_iterator(content->snippets.end())); | |
| 531 content->snippets.resize(kMaxSnippetCount); | |
| 532 if (category == articles_category_) | |
| 533 database_->DeleteSnippets(to_delete); | |
| 534 } | |
| 535 } | 589 } |
| 536 } | 590 } |
| 537 | 591 |
| 538 // Trigger expiration. This probably won't expire any current snippets (old | |
| 539 // ones should have already been expired by the timer, and new ones shouldn't | |
| 540 // have expired yet), but it will update the timer for the next run. | |
| 541 ClearExpiredSnippets(); | |
| 542 | |
| 543 for (const auto& item : categories_) { | 592 for (const auto& item : categories_) { |
| 544 Category category = item.first; | 593 Category category = item.first; |
| 545 UpdateCategoryStatus(category, CategoryStatus::AVAILABLE); | 594 UpdateCategoryStatus(category, CategoryStatus::AVAILABLE); |
| 546 } | 595 } |
| 547 | 596 |
| 548 // TODO(sfiera): equivalent metrics for non-articles. | 597 // TODO(sfiera): equivalent metrics for non-articles. |
| 549 const CategoryContent& content = categories_[articles_category_]; | 598 const CategoryContent& content = categories_[articles_category_]; |
| 550 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles", | 599 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles", |
| 551 content.snippets.size()); | 600 content.snippets.size()); |
| 552 if (content.snippets.empty() && !content.dismissed.empty()) { | 601 if (content.snippets.empty() && !content.dismissed.empty()) { |
| 553 UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded", | 602 UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded", |
| 554 content.dismissed.size()); | 603 content.dismissed.size()); |
| 555 } | 604 } |
| 556 | 605 |
| 557 // TODO(sfiera): notify only when a category changed above. | 606 // TODO(sfiera): notify only when a category changed above. |
| 558 NotifyNewSuggestions(); | 607 NotifyNewSuggestions(); |
| 559 | 608 |
| 560 // Reschedule after a successful fetch. This resets all currently scheduled | 609 // Reschedule after a successful fetch. This resets all currently scheduled |
| 561 // fetches, to make sure the fallback interval triggers only if no wifi fetch | 610 // fetches, to make sure the fallback interval triggers only if no wifi fetch |
| 562 // succeeded, and also that we don't do a background fetch immediately after | 611 // succeeded, and also that we don't do a background fetch immediately after |
| 563 // a user-initiated one. | 612 // a user-initiated one. |
| 564 if (snippets) | 613 if (snippets) |
| 565 RescheduleFetching(); | 614 RescheduleFetching(); |
| 566 } | 615 } |
| 567 | 616 |
| 568 void NTPSnippetsService::MergeSnippets(Category category, | 617 void NTPSnippetsService::ArchiveSnippets(Category category, |
| 569 NTPSnippet::PtrVector new_snippets) { | 618 NTPSnippet::PtrVector* to_archive) { |
| 619 CategoryContent* content = &categories_[category]; |
| 620 |
| 621 // TODO(sfiera): handle DB for non-articles too. |
| 622 if (category == articles_category_) { |
| 623 database_->DeleteSnippets(*to_archive); |
| 624 // Do not delete the thumbnail images as they are still handy on open NTPs. |
| 625 } |
| 626 |
| 627 // Archive previous snippets - move them at the beginning of the list. |
| 628 content->archived.insert(content->archived.begin(), |
| 629 std::make_move_iterator(to_archive->begin()), |
| 630 std::make_move_iterator(to_archive->end())); |
| 631 Compact(to_archive); |
| 632 |
| 633 // If there are more archived snippets than we want to keep, delete the |
| 634 // oldest ones by their fetch time (which are always in the back). |
| 635 if (content->archived.size() > kMaxArchivedSnippetCount) { |
| 636 NTPSnippet::PtrVector to_delete( |
| 637 std::make_move_iterator(content->archived.begin() + |
| 638 kMaxArchivedSnippetCount), |
| 639 std::make_move_iterator(content->archived.end())); |
| 640 content->archived.resize(kMaxArchivedSnippetCount); |
| 641 if (category == articles_category_) |
| 642 database_->DeleteImages(to_delete); |
| 643 } |
| 644 } |
| 645 |
| 646 void NTPSnippetsService::ReplaceSnippets(Category category, |
| 647 NTPSnippet::PtrVector new_snippets) { |
| 570 DCHECK(ready()); | 648 DCHECK(ready()); |
| 571 CategoryContent* content = &categories_[category]; | 649 CategoryContent* content = &categories_[category]; |
| 572 | 650 |
| 573 // Remove new snippets that we already have, or that have been dismissed. | 651 // Remove new snippets that have been dismissed. |
| 574 std::set<std::string> old_snippet_ids; | 652 EraseMatchingSnippets(&new_snippets, GetAllIDs(content->dismissed), |
| 575 InsertAllIDs(content->dismissed, &old_snippet_ids); | 653 /*match_all_ids=*/true); |
| 576 InsertAllIDs(content->snippets, &old_snippet_ids); | |
| 577 new_snippets.erase( | |
| 578 std::remove_if( | |
| 579 new_snippets.begin(), new_snippets.end(), | |
| 580 [&old_snippet_ids](const std::unique_ptr<NTPSnippet>& snippet) { | |
| 581 if (old_snippet_ids.count(snippet->id())) | |
| 582 return true; | |
| 583 for (const SnippetSource& source : snippet->sources()) { | |
| 584 if (old_snippet_ids.count(source.url.spec())) | |
| 585 return true; | |
| 586 } | |
| 587 return false; | |
| 588 }), | |
| 589 new_snippets.end()); | |
| 590 | 654 |
| 591 // Fill in default publish/expiry dates where required. | 655 // Fill in default publish/expiry dates where required. |
| 592 for (std::unique_ptr<NTPSnippet>& snippet : new_snippets) { | 656 for (std::unique_ptr<NTPSnippet>& snippet : new_snippets) { |
| 593 if (snippet->publish_date().is_null()) | 657 if (snippet->publish_date().is_null()) |
| 594 snippet->set_publish_date(base::Time::Now()); | 658 snippet->set_publish_date(base::Time::Now()); |
| 595 if (snippet->expiry_date().is_null()) { | 659 if (snippet->expiry_date().is_null()) { |
| 596 snippet->set_expiry_date( | 660 snippet->set_expiry_date( |
| 597 snippet->publish_date() + | 661 snippet->publish_date() + |
| 598 base::TimeDelta::FromMinutes(kDefaultExpiryTimeMins)); | 662 base::TimeDelta::FromMinutes(kDefaultExpiryTimeMins)); |
| 599 } | 663 } |
| (...skipping 14 matching lines...) Expand all Loading... |
| 614 new_snippets.end()); | 678 new_snippets.end()); |
| 615 int num_snippets_dismissed = num_new_snippets - new_snippets.size(); | 679 int num_snippets_dismissed = num_new_snippets - new_snippets.size(); |
| 616 UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch", | 680 UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch", |
| 617 num_snippets_dismissed > 0); | 681 num_snippets_dismissed > 0); |
| 618 if (num_snippets_dismissed > 0) { | 682 if (num_snippets_dismissed > 0) { |
| 619 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets", | 683 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets", |
| 620 num_snippets_dismissed); | 684 num_snippets_dismissed); |
| 621 } | 685 } |
| 622 } | 686 } |
| 623 | 687 |
| 624 // Save new articles to the DB. | 688 // Do not touch the current set of snippets if the newly fetched one is empty. |
| 625 // TODO(sfiera): save non-articles to DB too. | 689 if (new_snippets.empty()) |
| 626 if (category == articles_category_) | 690 return; |
| 691 |
| 692 // Remove current snippets that have been fetched again. We do not need to |
| 693 // archive those as they will be in the new current set. |
| 694 EraseMatchingSnippets(&content->snippets, GetMainIDs(new_snippets), |
| 695 /*match_all_ids=*/false); |
| 696 |
| 697 ArchiveSnippets(category, &content->snippets); |
| 698 |
| 699 // TODO(sfiera): handle DB for non-articles too. |
| 700 if (category == articles_category_) { |
| 701 // Save new articles to the DB. |
| 627 database_->SaveSnippets(new_snippets); | 702 database_->SaveSnippets(new_snippets); |
| 703 } |
| 628 | 704 |
| 629 // Insert the new snippets at the front. | 705 content->snippets = std::move(new_snippets); |
| 630 content->snippets.insert(content->snippets.begin(), | |
| 631 std::make_move_iterator(new_snippets.begin()), | |
| 632 std::make_move_iterator(new_snippets.end())); | |
| 633 } | 706 } |
| 634 | 707 |
| 635 std::set<std::string> NTPSnippetsService::GetSnippetHostsFromPrefs() const { | 708 std::set<std::string> NTPSnippetsService::GetSnippetHostsFromPrefs() const { |
| 636 std::set<std::string> hosts; | 709 std::set<std::string> hosts; |
| 637 const base::ListValue* list = pref_service_->GetList(prefs::kSnippetHosts); | 710 const base::ListValue* list = pref_service_->GetList(prefs::kSnippetHosts); |
| 638 for (const auto& value : *list) { | 711 for (const auto& value : *list) { |
| 639 std::string str; | 712 std::string str; |
| 640 bool success = value->GetAsString(&str); | 713 bool success = value->GetAsString(&str); |
| 641 DCHECK(success) << "Failed to parse snippet host from prefs"; | 714 DCHECK(success) << "Failed to parse snippet host from prefs"; |
| 642 hosts.insert(std::move(str)); | 715 hosts.insert(std::move(str)); |
| 643 } | 716 } |
| 644 return hosts; | 717 return hosts; |
| 645 } | 718 } |
| 646 | 719 |
| 647 void NTPSnippetsService::StoreSnippetHostsToPrefs( | 720 void NTPSnippetsService::StoreSnippetHostsToPrefs( |
| 648 const std::set<std::string>& hosts) { | 721 const std::set<std::string>& hosts) { |
| 649 base::ListValue list; | 722 base::ListValue list; |
| 650 for (const std::string& host : hosts) | 723 for (const std::string& host : hosts) |
| 651 list.AppendString(host); | 724 list.AppendString(host); |
| 652 pref_service_->Set(prefs::kSnippetHosts, list); | 725 pref_service_->Set(prefs::kSnippetHosts, list); |
| 653 } | 726 } |
| 654 | 727 |
| 655 void NTPSnippetsService::ClearExpiredSnippets() { | 728 void NTPSnippetsService::ClearExpiredDismissedSnippets() { |
| 656 std::vector<Category> categories_to_erase; | 729 std::vector<Category> categories_to_erase; |
| 657 | 730 |
| 658 const base::Time expiry = base::Time::Now(); | 731 const base::Time now = base::Time::Now(); |
| 659 base::Time next_expiry = base::Time::Max(); | |
| 660 | 732 |
| 661 for (auto& item : categories_) { | 733 for (auto& item : categories_) { |
| 662 Category category = item.first; | 734 Category category = item.first; |
| 663 CategoryContent* content = &item.second; | 735 CategoryContent* content = &item.second; |
| 664 | 736 |
| 665 // Move expired snippets over into |to_delete|. | |
| 666 NTPSnippet::PtrVector to_delete; | 737 NTPSnippet::PtrVector to_delete; |
| 667 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) { | 738 // Move expired dismissed snippets over into |to_delete|. |
| 668 if (snippet->expiry_date() <= expiry) | |
| 669 to_delete.emplace_back(std::move(snippet)); | |
| 670 } | |
| 671 Compact(&content->snippets); | |
| 672 | |
| 673 // Move expired dismissed snippets over into |to_delete| as well. | |
| 674 for (std::unique_ptr<NTPSnippet>& snippet : content->dismissed) { | 739 for (std::unique_ptr<NTPSnippet>& snippet : content->dismissed) { |
| 675 if (snippet->expiry_date() <= expiry) | 740 if (snippet->expiry_date() <= now) |
| 676 to_delete.emplace_back(std::move(snippet)); | 741 to_delete.emplace_back(std::move(snippet)); |
| 677 } | 742 } |
| 678 Compact(&content->dismissed); | 743 Compact(&content->dismissed); |
| 679 | 744 |
| 680 // Finally, actually delete the removed snippets from the DB. | 745 // Delete the removed article suggestions from the DB. |
| 681 if (category == articles_category_) | 746 if (category == articles_category_) { |
| 747 // The image got already deleted when the suggestion was dismissed. |
| 682 database_->DeleteSnippets(to_delete); | 748 database_->DeleteSnippets(to_delete); |
| 683 | |
| 684 if (content->snippets.empty() && content->dismissed.empty()) { | |
| 685 if ((category != articles_category_) && !content->provided_by_server) | |
| 686 categories_to_erase.push_back(category); | |
| 687 continue; | |
| 688 } | 749 } |
| 689 | 750 |
| 690 for (const auto& snippet : content->snippets) { | 751 if (content->snippets.empty() && content->dismissed.empty() && |
| 691 if (snippet->expiry_date() < next_expiry) | 752 category != articles_category_ && !content->provided_by_server) { |
| 692 next_expiry = snippet->expiry_date(); | 753 categories_to_erase.push_back(category); |
| 693 } | |
| 694 for (const auto& snippet : content->dismissed) { | |
| 695 if (snippet->expiry_date() < next_expiry) | |
| 696 next_expiry = snippet->expiry_date(); | |
| 697 } | 754 } |
| 698 } | 755 } |
| 699 | 756 |
| 700 for (Category category : categories_to_erase) { | 757 for (Category category : categories_to_erase) { |
| 701 UpdateCategoryStatus(category, CategoryStatus::NOT_PROVIDED); | 758 UpdateCategoryStatus(category, CategoryStatus::NOT_PROVIDED); |
| 702 categories_.erase(category); | 759 categories_.erase(category); |
| 703 } | 760 } |
| 761 } |
| 704 | 762 |
| 705 // Unless there are no snippets left, schedule a timer for the next expiry. | 763 void NTPSnippetsService::ClearOrphanedImages() { |
| 706 DCHECK_GT(next_expiry, expiry); | 764 // TODO(jkrcal): Implement. crbug.com/649009 |
| 707 if (next_expiry < base::Time::Max()) { | |
| 708 expiry_timer_.Start(FROM_HERE, next_expiry - expiry, | |
| 709 base::Bind(&NTPSnippetsService::ClearExpiredSnippets, | |
| 710 base::Unretained(this))); | |
| 711 } | |
| 712 } | 765 } |
| 713 | 766 |
| 714 void NTPSnippetsService::NukeAllSnippets() { | 767 void NTPSnippetsService::NukeAllSnippets() { |
| 715 std::vector<Category> categories_to_erase; | 768 std::vector<Category> categories_to_erase; |
| 716 | 769 |
| 717 // Empty the ARTICLES category and remove all others, since they may or may | 770 // Empty the ARTICLES category and remove all others, since they may or may |
| 718 // not be personalized. | 771 // not be personalized. |
| 719 for (const auto& item : categories_) { | 772 for (const auto& item : categories_) { |
| 720 Category category = item.first; | 773 Category category = item.first; |
| 721 | 774 |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 776 | 829 |
| 777 FetchSnippetImageFromNetwork(suggestion_id, callback); | 830 FetchSnippetImageFromNetwork(suggestion_id, callback); |
| 778 } | 831 } |
| 779 | 832 |
| 780 void NTPSnippetsService::FetchSnippetImageFromNetwork( | 833 void NTPSnippetsService::FetchSnippetImageFromNetwork( |
| 781 const std::string& suggestion_id, | 834 const std::string& suggestion_id, |
| 782 const ImageFetchedCallback& callback) { | 835 const ImageFetchedCallback& callback) { |
| 783 Category category = GetCategoryFromUniqueID(suggestion_id); | 836 Category category = GetCategoryFromUniqueID(suggestion_id); |
| 784 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id); | 837 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id); |
| 785 | 838 |
| 786 auto category_it = categories_.find(category); | 839 if (categories_.find(category) == categories_.end()) { |
| 787 if (category_it == categories_.end()) { | |
| 788 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image()); | 840 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image()); |
| 789 return; | 841 return; |
| 790 } | 842 } |
| 791 | 843 |
| 792 const CategoryContent& content = category_it->second; | 844 GURL image_url = FindSnippetImageUrl(category, snippet_id); |
| 793 auto it = | |
| 794 std::find_if(content.snippets.begin(), content.snippets.end(), | |
| 795 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) { | |
| 796 return snippet->id() == snippet_id; | |
| 797 }); | |
| 798 | 845 |
| 799 if (it == content.snippets.end() || | 846 if (image_url.is_empty() || |
| 800 !thumbnail_requests_throttler_.DemandQuotaForRequest( | 847 !thumbnail_requests_throttler_.DemandQuotaForRequest( |
| 801 /*interactive_request=*/true)) { | 848 /*interactive_request=*/true)) { |
| 802 // Return an empty image. Directly, this is never synchronous with the | 849 // Return an empty image. Directly, this is never synchronous with the |
| 803 // original FetchSuggestionImage() call - an asynchronous database query has | 850 // original FetchSuggestionImage() call - an asynchronous database query has |
| 804 // happened in the meantime. | 851 // happened in the meantime. |
| 805 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image()); | 852 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image()); |
| 806 return; | 853 return; |
| 807 } | 854 } |
| 808 | 855 |
| 809 const NTPSnippet& snippet = *it->get(); | |
| 810 | |
| 811 // TODO(jkrcal): We probably should rename OnImageDataFetched() to | |
| 812 // CacheImageData(). This would document that this is actually independent | |
| 813 // from the individual fetch-flow. | |
| 814 // The image fetcher calls OnImageDataFetched() with the raw data (this object | |
| 815 // is an ImageFetcherDelegate) and then also | |
| 816 // OnSnippetImageDecodedFromNetwork() after the raw data gets decoded. | |
| 817 image_fetcher_->StartOrQueueNetworkRequest( | 856 image_fetcher_->StartOrQueueNetworkRequest( |
| 818 suggestion_id, snippet.salient_image_url(), | 857 suggestion_id, image_url, |
| 819 base::Bind(&NTPSnippetsService::OnSnippetImageDecodedFromNetwork, | 858 base::Bind(&NTPSnippetsService::OnSnippetImageDecodedFromNetwork, |
| 820 base::Unretained(this), callback)); | 859 base::Unretained(this), callback)); |
| 821 } | 860 } |
| 822 | 861 |
| 823 void NTPSnippetsService::OnSnippetImageDecodedFromNetwork( | 862 void NTPSnippetsService::OnSnippetImageDecodedFromNetwork( |
| 824 const ImageFetchedCallback& callback, | 863 const ImageFetchedCallback& callback, |
| 825 const std::string& suggestion_id, | 864 const std::string& suggestion_id, |
| 826 const gfx::Image& image) { | 865 const gfx::Image& image) { |
| 827 callback.Run(image); | 866 callback.Run(image); |
| 828 } | 867 } |
| (...skipping 26 matching lines...) Expand all Loading... |
| 855 // |suggestions_service_| can be null in tests. | 894 // |suggestions_service_| can be null in tests. |
| 856 if (snippets_fetcher_->UsesHostRestrictions() && suggestions_service_) { | 895 if (snippets_fetcher_->UsesHostRestrictions() && suggestions_service_) { |
| 857 suggestions_service_subscription_ = | 896 suggestions_service_subscription_ = |
| 858 suggestions_service_->AddCallback(base::Bind( | 897 suggestions_service_->AddCallback(base::Bind( |
| 859 &NTPSnippetsService::OnSuggestionsChanged, base::Unretained(this))); | 898 &NTPSnippetsService::OnSuggestionsChanged, base::Unretained(this))); |
| 860 } | 899 } |
| 861 } | 900 } |
| 862 | 901 |
| 863 void NTPSnippetsService::EnterStateDisabled() { | 902 void NTPSnippetsService::EnterStateDisabled() { |
| 864 NukeAllSnippets(); | 903 NukeAllSnippets(); |
| 865 expiry_timer_.Stop(); | |
| 866 suggestions_service_subscription_.reset(); | 904 suggestions_service_subscription_.reset(); |
| 867 } | 905 } |
| 868 | 906 |
| 869 void NTPSnippetsService::EnterStateError() { | 907 void NTPSnippetsService::EnterStateError() { |
| 870 expiry_timer_.Stop(); | |
| 871 suggestions_service_subscription_.reset(); | 908 suggestions_service_subscription_.reset(); |
| 872 snippets_status_service_.reset(); | 909 snippets_status_service_.reset(); |
| 873 } | 910 } |
| 874 | 911 |
| 875 void NTPSnippetsService::FinishInitialization() { | 912 void NTPSnippetsService::FinishInitialization() { |
| 876 if (nuke_when_initialized_) { | 913 if (nuke_when_initialized_) { |
| 877 // We nuke here in addition to EnterStateReady, so that it happens even if | 914 // We nuke here in addition to EnterStateReady, so that it happens even if |
| 878 // we enter the DISABLED state below. | 915 // we enter the DISABLED state below. |
| 879 NukeAllSnippets(); | 916 NukeAllSnippets(); |
| 880 nuke_when_initialized_ = false; | 917 nuke_when_initialized_ = false; |
| (...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1012 } | 1049 } |
| 1013 | 1050 |
| 1014 NTPSnippetsService::CategoryContent::CategoryContent() = default; | 1051 NTPSnippetsService::CategoryContent::CategoryContent() = default; |
| 1015 NTPSnippetsService::CategoryContent::CategoryContent(CategoryContent&&) = | 1052 NTPSnippetsService::CategoryContent::CategoryContent(CategoryContent&&) = |
| 1016 default; | 1053 default; |
| 1017 NTPSnippetsService::CategoryContent::~CategoryContent() = default; | 1054 NTPSnippetsService::CategoryContent::~CategoryContent() = default; |
| 1018 NTPSnippetsService::CategoryContent& NTPSnippetsService::CategoryContent:: | 1055 NTPSnippetsService::CategoryContent& NTPSnippetsService::CategoryContent:: |
| 1019 operator=(CategoryContent&&) = default; | 1056 operator=(CategoryContent&&) = default; |
| 1020 | 1057 |
| 1021 } // namespace ntp_snippets | 1058 } // namespace ntp_snippets |
| OLD | NEW |