Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(199)

Side by Side Diff: components/ntp_snippets/ntp_snippets_service.cc

Issue 2355393002: New snippets now replace old snippets and do not merge (Closed)
Patch Set: Marc's comments #2 Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « components/ntp_snippets/ntp_snippets_service.h ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2015 The Chromium Authors. All rights reserved. 1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "components/ntp_snippets/ntp_snippets_service.h" 5 #include "components/ntp_snippets/ntp_snippets_service.h"
6 6
7 #include <algorithm> 7 #include <algorithm>
8 #include <iterator> 8 #include <iterator>
9 #include <utility> 9 #include <utility>
10 10
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after
43 using suggestions::SuggestionsService; 43 using suggestions::SuggestionsService;
44 44
45 namespace ntp_snippets { 45 namespace ntp_snippets {
46 46
47 namespace { 47 namespace {
48 48
49 // Number of snippets requested to the server. Consider replacing sparse UMA 49 // Number of snippets requested to the server. Consider replacing sparse UMA
50 // histograms with COUNTS() if this number increases beyond 50. 50 // histograms with COUNTS() if this number increases beyond 50.
51 const int kMaxSnippetCount = 10; 51 const int kMaxSnippetCount = 10;
52 52
53 // Number of archived snippets we keep around in memory.
54 const int kMaxArchivedSnippetCount = 200;
55
53 // Default values for snippets fetching intervals - once per day only. 56 // Default values for snippets fetching intervals - once per day only.
54 const int kDefaultFetchingIntervalWifiChargingSeconds = 0; 57 const int kDefaultFetchingIntervalWifiChargingSeconds = 0;
55 const int kDefaultFetchingIntervalWifiSeconds = 0; 58 const int kDefaultFetchingIntervalWifiSeconds = 0;
56 const int kDefaultFetchingIntervalFallbackSeconds = 24 * 60 * 60; 59 const int kDefaultFetchingIntervalFallbackSeconds = 24 * 60 * 60;
57 60
58 // Variation parameters than can override the default fetching intervals. 61 // Variation parameters than can override the default fetching intervals.
59 const char kFetchingIntervalWifiChargingParamName[] = 62 const char kFetchingIntervalWifiChargingParamName[] =
60 "fetching_interval_wifi_charging_seconds"; 63 "fetching_interval_wifi_charging_seconds";
61 const char kFetchingIntervalWifiParamName[] = 64 const char kFetchingIntervalWifiParamName[] =
62 "fetching_interval_wifi_seconds"; 65 "fetching_interval_wifi_seconds";
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after
161 std::set<std::string> hosts; 164 std::set<std::string> hosts;
162 for (int i = 0; i < suggestions.suggestions_size(); ++i) { 165 for (int i = 0; i < suggestions.suggestions_size(); ++i) {
163 const ChromeSuggestion& suggestion = suggestions.suggestions(i); 166 const ChromeSuggestion& suggestion = suggestions.suggestions(i);
164 GURL url(suggestion.url()); 167 GURL url(suggestion.url());
165 if (url.is_valid()) 168 if (url.is_valid())
166 hosts.insert(url.host()); 169 hosts.insert(url.host());
167 } 170 }
168 return hosts; 171 return hosts;
169 } 172 }
170 173
171 void InsertAllIDs(const NTPSnippet::PtrVector& snippets, 174 std::set<std::string> GetAllIDs(const NTPSnippet::PtrVector& snippets) {
172 std::set<std::string>* ids) { 175 std::set<std::string> ids;
173 for (const std::unique_ptr<NTPSnippet>& snippet : snippets) { 176 for (const std::unique_ptr<NTPSnippet>& snippet : snippets) {
174 ids->insert(snippet->id()); 177 ids.insert(snippet->id());
175 for (const SnippetSource& source : snippet->sources()) 178 for (const SnippetSource& source : snippet->sources())
176 ids->insert(source.url.spec()); 179 ids.insert(source.url.spec());
177 } 180 }
181 return ids;
182 }
183
184 std::set<std::string> GetMainIDs(const NTPSnippet::PtrVector& snippets) {
185 std::set<std::string> ids;
186 for (const std::unique_ptr<NTPSnippet>& snippet : snippets)
187 ids.insert(snippet->id());
188 return ids;
189 }
190
191 bool IsSnippetInSet(const std::unique_ptr<NTPSnippet>& snippet,
192 const std::set<std::string>& ids,
193 bool match_all_ids) {
194 if (ids.count(snippet->id()))
195 return true;
196 if (!match_all_ids)
197 return false;
198 for (const SnippetSource& source : snippet->sources()) {
199 if (ids.count(source.url.spec()))
200 return true;
201 }
202 return false;
203 }
204
205 void EraseMatchingSnippets(NTPSnippet::PtrVector* snippets,
206 const std::set<std::string>& matching_ids,
Marc Treib 2016/09/22 13:05:29 optional: You could also pass in the snippets to m
jkrcal 2016/09/22 14:07:58 I think the current way it is even clearer on the
207 bool match_all_ids) {
208 snippets->erase(
209 std::remove_if(snippets->begin(), snippets->end(),
210 [&matching_ids, match_all_ids](
211 const std::unique_ptr<NTPSnippet>& snippet) {
212 return IsSnippetInSet(snippet, matching_ids,
213 match_all_ids);
214 }),
215 snippets->end());
178 } 216 }
179 217
180 void Compact(NTPSnippet::PtrVector* snippets) { 218 void Compact(NTPSnippet::PtrVector* snippets) {
181 snippets->erase( 219 snippets->erase(
182 std::remove_if( 220 std::remove_if(
183 snippets->begin(), snippets->end(), 221 snippets->begin(), snippets->end(),
184 [](const std::unique_ptr<NTPSnippet>& snippet) { return !snippet; }), 222 [](const std::unique_ptr<NTPSnippet>& snippet) { return !snippet; }),
185 snippets->end()); 223 snippets->end());
186 } 224 }
187 225
(...skipping 174 matching lines...) Expand 10 before | Expand all | Expand 10 after
362 void NTPSnippetsService::ClearCachedSuggestions(Category category) { 400 void NTPSnippetsService::ClearCachedSuggestions(Category category) {
363 if (!initialized()) 401 if (!initialized())
364 return; 402 return;
365 403
366 if (categories_.find(category) == categories_.end()) 404 if (categories_.find(category) == categories_.end())
367 return; 405 return;
368 CategoryContent* content = &categories_[category]; 406 CategoryContent* content = &categories_[category];
369 if (content->snippets.empty()) 407 if (content->snippets.empty())
370 return; 408 return;
371 409
372 if (category == articles_category_) 410 if (category == articles_category_) {
373 database_->DeleteSnippets(content->snippets); 411 database_->DeleteSnippets(content->snippets);
412 database_->DeleteImages(content->snippets);
413 }
374 content->snippets.clear(); 414 content->snippets.clear();
375 415
376 NotifyNewSuggestions(); 416 NotifyNewSuggestions();
377 } 417 }
378 418
379 void NTPSnippetsService::GetDismissedSuggestionsForDebugging( 419 void NTPSnippetsService::GetDismissedSuggestionsForDebugging(
380 Category category, 420 Category category,
381 const DismissedSuggestionsCallback& callback) { 421 const DismissedSuggestionsCallback& callback) {
382 DCHECK(categories_.find(category) != categories_.end()); 422 DCHECK(categories_.find(category) != categories_.end());
383 423
(...skipping 20 matching lines...) Expand all
404 Category category) { 444 Category category) {
405 DCHECK(categories_.find(category) != categories_.end()); 445 DCHECK(categories_.find(category) != categories_.end());
406 446
407 if (!initialized()) 447 if (!initialized())
408 return; 448 return;
409 449
410 CategoryContent* content = &categories_[category]; 450 CategoryContent* content = &categories_[category];
411 if (content->dismissed.empty()) 451 if (content->dismissed.empty())
412 return; 452 return;
413 453
414 if (category == articles_category_) 454 if (category == articles_category_) {
455 // The image got already deleted when the suggestion was dismissed.
415 database_->DeleteSnippets(content->dismissed); 456 database_->DeleteSnippets(content->dismissed);
457 }
416 content->dismissed.clear(); 458 content->dismissed.clear();
417 } 459 }
418 460
419 std::set<std::string> NTPSnippetsService::GetSuggestionsHosts() const { 461 std::set<std::string> NTPSnippetsService::GetSuggestionsHosts() const {
420 // |suggestions_service_| can be null in tests. 462 // |suggestions_service_| can be null in tests.
421 if (!suggestions_service_) 463 if (!suggestions_service_)
422 return std::set<std::string>(); 464 return std::set<std::string>();
423 465
424 // TODO(treib): This should just call GetSnippetHostsFromPrefs. 466 // TODO(treib): This should just call GetSnippetHostsFromPrefs.
425 return GetSuggestionsHostsImpl( 467 return GetSuggestionsHostsImpl(
426 suggestions_service_->GetSuggestionsDataFromCache()); 468 suggestions_service_->GetSuggestionsDataFromCache());
427 } 469 }
428 470
429 // static 471 // static
430 int NTPSnippetsService::GetMaxSnippetCountForTesting() { 472 int NTPSnippetsService::GetMaxSnippetCountForTesting() {
431 return kMaxSnippetCount; 473 return kMaxSnippetCount;
432 } 474 }
433 475
434 //////////////////////////////////////////////////////////////////////////////// 476 ////////////////////////////////////////////////////////////////////////////////
435 // Private methods 477 // Private methods
436 478
479 GURL NTPSnippetsService::FindSnippetImageUrl(
480 Category category,
481 const std::string& snippet_id) const {
482 DCHECK(categories_.find(category) != categories_.end());
483
484 const CategoryContent& content = categories_.at(category);
485 // Search for the snippet in current and archived snippets.
486 auto it =
487 std::find_if(content.snippets.begin(), content.snippets.end(),
488 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
489 return snippet->id() == snippet_id;
490 });
491 if (it != content.snippets.end())
492 return (*it)->salient_image_url();
493
494 it = std::find_if(content.archived.begin(), content.archived.end(),
495 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
496 return snippet->id() == snippet_id;
497 });
498 if (it != content.archived.end())
499 return (*it)->salient_image_url();
500
501 return GURL();
502 }
503
437 // image_fetcher::ImageFetcherDelegate implementation. 504 // image_fetcher::ImageFetcherDelegate implementation.
438 void NTPSnippetsService::OnImageDataFetched(const std::string& suggestion_id, 505 void NTPSnippetsService::OnImageDataFetched(const std::string& suggestion_id,
439 const std::string& image_data) { 506 const std::string& image_data) {
440 if (image_data.empty()) 507 if (image_data.empty())
441 return; 508 return;
442 509
443 Category category = GetCategoryFromUniqueID(suggestion_id); 510 Category category = GetCategoryFromUniqueID(suggestion_id);
444 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id); 511 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id);
445 512
446 auto category_it = categories_.find(category); 513 if (categories_.find(category) == categories_.end())
447 if (category_it == categories_.end())
448 return; 514 return;
449 515
450 const CategoryContent& content = category_it->second;
451
452 // Only save the image if the corresponding snippet still exists. 516 // Only save the image if the corresponding snippet still exists.
453 auto it = 517 if (FindSnippetImageUrl(category, snippet_id).is_empty())
454 std::find_if(content.snippets.begin(), content.snippets.end(),
455 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
456 return snippet->id() == snippet_id;
457 });
458 if (it == content.snippets.end())
459 return; 518 return;
460 519
520 // Only cache the data in the DB, the actual serving is done in the callback
521 // provided to |image_fetcher_| (OnSnippetImageDecodedFromNetwork()).
461 database_->SaveImage(snippet_id, image_data); 522 database_->SaveImage(snippet_id, image_data);
462 } 523 }
463 524
464 void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) { 525 void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) {
465 if (state_ == State::ERROR_OCCURRED) 526 if (state_ == State::ERROR_OCCURRED)
466 return; 527 return;
467 DCHECK(state_ == State::NOT_INITED); 528 DCHECK(state_ == State::NOT_INITED);
468 DCHECK(categories_.size() == 1); // Only articles category, so far. 529 DCHECK_EQ(1u, categories_.size()); // Only articles category, so far.
469 DCHECK(categories_.find(articles_category_) != categories_.end()); 530 DCHECK(categories_.find(articles_category_) != categories_.end());
470 531
471 // TODO(sfiera): support non-article categories in database. 532 // TODO(sfiera): support non-article categories in database.
472 CategoryContent* content = &categories_[articles_category_]; 533 CategoryContent* content = &categories_[articles_category_];
473 for (std::unique_ptr<NTPSnippet>& snippet : snippets) { 534 for (std::unique_ptr<NTPSnippet>& snippet : snippets) {
474 if (snippet->is_dismissed()) 535 if (snippet->is_dismissed())
475 content->dismissed.emplace_back(std::move(snippet)); 536 content->dismissed.emplace_back(std::move(snippet));
476 else 537 else
477 content->snippets.emplace_back(std::move(snippet)); 538 content->snippets.emplace_back(std::move(snippet));
478 } 539 }
479 540
480 std::sort(content->snippets.begin(), content->snippets.end(), 541 std::sort(content->snippets.begin(), content->snippets.end(),
481 [](const std::unique_ptr<NTPSnippet>& lhs, 542 [](const std::unique_ptr<NTPSnippet>& lhs,
482 const std::unique_ptr<NTPSnippet>& rhs) { 543 const std::unique_ptr<NTPSnippet>& rhs) {
483 return lhs->score() > rhs->score(); 544 return lhs->score() > rhs->score();
484 }); 545 });
485 546
486 ClearExpiredSnippets(); 547 ClearExpiredDismissedSnippets();
548 ClearOrphanedImages();
487 FinishInitialization(); 549 FinishInitialization();
488 } 550 }
489 551
490 void NTPSnippetsService::OnDatabaseError() { 552 void NTPSnippetsService::OnDatabaseError() {
491 EnterState(State::ERROR_OCCURRED); 553 EnterState(State::ERROR_OCCURRED);
492 UpdateAllCategoryStatus(CategoryStatus::LOADING_ERROR); 554 UpdateAllCategoryStatus(CategoryStatus::LOADING_ERROR);
493 } 555 }
494 556
495 // TODO(dgn): name clash between content suggestions and suggestions hosts. 557 // TODO(dgn): name clash between content suggestions and suggestions hosts.
496 // method name should be changed. 558 // method name should be changed.
(...skipping 14 matching lines...) Expand all
511 // and apply the same logic to them here. Maybe never? 573 // and apply the same logic to them here. Maybe never?
512 // 574 //
513 // First, move them over into |to_delete|. 575 // First, move them over into |to_delete|.
514 CategoryContent* content = &categories_[articles_category_]; 576 CategoryContent* content = &categories_[articles_category_];
515 NTPSnippet::PtrVector to_delete; 577 NTPSnippet::PtrVector to_delete;
516 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) { 578 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) {
517 if (!hosts.count(snippet->best_source().url.host())) 579 if (!hosts.count(snippet->best_source().url.host()))
518 to_delete.emplace_back(std::move(snippet)); 580 to_delete.emplace_back(std::move(snippet));
519 } 581 }
520 Compact(&content->snippets); 582 Compact(&content->snippets);
521 // Then delete the removed snippets from the database. 583 ArchiveSnippets(articles_category_, &to_delete);
522 database_->DeleteSnippets(to_delete);
523 584
524 StoreSnippetHostsToPrefs(hosts); 585 StoreSnippetHostsToPrefs(hosts);
525 586
526 // We removed some suggestions, so we want to let the client know about that. 587 // We removed some suggestions, so we want to let the client know about that.
527 // The fetch might take a long time or not complete so we don't want to wait 588 // The fetch might take a long time or not complete so we don't want to wait
528 // for its callback. 589 // for its callback.
529 NotifyNewSuggestions(); 590 NotifyNewSuggestions();
530 591
531 FetchSnippetsFromHosts(hosts, /*force_request=*/false); 592 FetchSnippetsFromHosts(hosts, /*force_request=*/false);
532 } 593 }
533 594
534 void NTPSnippetsService::OnFetchFinished( 595 void NTPSnippetsService::OnFetchFinished(
535 NTPSnippetsFetcher::OptionalSnippets snippets) { 596 NTPSnippetsFetcher::OptionalSnippets snippets) {
536 if (!ready()) 597 if (!ready())
537 return; 598 return;
538 599
539 for (auto& item : categories_) { 600 for (auto& item : categories_) {
540 CategoryContent* content = &item.second; 601 CategoryContent* content = &item.second;
541 content->provided_by_server = false; 602 content->provided_by_server = false;
542 } 603 }
543 604
605 // Clear up expired dismissed snippets before we use them to filter new ones.
606 ClearExpiredDismissedSnippets();
607
544 // If snippets were fetched successfully, update our |categories_| from each 608 // If snippets were fetched successfully, update our |categories_| from each
545 // category provided by the server. 609 // category provided by the server.
546 if (snippets) { 610 if (snippets) {
611 // TODO(jkrcal): A bit hard to understand with so many variables called
612 // "*categor*". Isn't here some room for simplification?
547 for (NTPSnippetsFetcher::FetchedCategory& fetched_category : *snippets) { 613 for (NTPSnippetsFetcher::FetchedCategory& fetched_category : *snippets) {
548 Category category = fetched_category.category; 614 Category category = fetched_category.category;
549 615
550 // TODO(sfiera): Avoid hard-coding articles category checks in so many 616 // TODO(sfiera): Avoid hard-coding articles category checks in so many
551 // places. 617 // places.
552 if (category != articles_category_) { 618 if (category != articles_category_) {
553 // Only update titles from server-side provided categories. 619 // Only update titles from server-side provided categories.
554 categories_[category].localized_title = 620 categories_[category].localized_title =
555 fetched_category.localized_title; 621 fetched_category.localized_title;
556 } 622 }
623 categories_[category].provided_by_server = true;
557 624
558 DCHECK_LE(snippets->size(), static_cast<size_t>(kMaxSnippetCount)); 625 DCHECK_LE(snippets->size(), static_cast<size_t>(kMaxSnippetCount));
559 // TODO(sfiera): histograms for server categories. 626 // TODO(sfiera): histograms for server categories.
560 // Sparse histogram used because the number of snippets is small (bound by 627 // Sparse histogram used because the number of snippets is small (bound by
561 // kMaxSnippetCount). 628 // kMaxSnippetCount).
562 if (category == articles_category_) { 629 if (category == articles_category_) {
563 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticlesFetched", 630 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticlesFetched",
564 fetched_category.snippets.size()); 631 fetched_category.snippets.size());
565 } 632 }
566 633
567 MergeSnippets(category, std::move(fetched_category.snippets)); 634 ReplaceSnippets(category, std::move(fetched_category.snippets));
568
569 // If there are more snippets than we want to show, delete the extra ones.
570 CategoryContent* content = &categories_[category];
571 content->provided_by_server = true;
572 if (content->snippets.size() > kMaxSnippetCount) {
573 NTPSnippet::PtrVector to_delete(
574 std::make_move_iterator(content->snippets.begin() +
575 kMaxSnippetCount),
576 std::make_move_iterator(content->snippets.end()));
577 content->snippets.resize(kMaxSnippetCount);
578 if (category == articles_category_)
579 database_->DeleteSnippets(to_delete);
580 }
581 } 635 }
582 } 636 }
583 637
584 // Trigger expiration. This probably won't expire any current snippets (old
585 // ones should have already been expired by the timer, and new ones shouldn't
586 // have expired yet), but it will update the timer for the next run.
587 ClearExpiredSnippets();
588
589 for (const auto& item : categories_) { 638 for (const auto& item : categories_) {
590 Category category = item.first; 639 Category category = item.first;
591 UpdateCategoryStatus(category, CategoryStatus::AVAILABLE); 640 UpdateCategoryStatus(category, CategoryStatus::AVAILABLE);
592 } 641 }
593 642
594 // TODO(sfiera): equivalent metrics for non-articles. 643 // TODO(sfiera): equivalent metrics for non-articles.
595 const CategoryContent& content = categories_[articles_category_]; 644 const CategoryContent& content = categories_[articles_category_];
596 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles", 645 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles",
597 content.snippets.size()); 646 content.snippets.size());
598 if (content.snippets.empty() && !content.dismissed.empty()) { 647 if (content.snippets.empty() && !content.dismissed.empty()) {
599 UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded", 648 UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded",
600 content.dismissed.size()); 649 content.dismissed.size());
601 } 650 }
602 651
603 // TODO(sfiera): notify only when a category changed above. 652 // TODO(sfiera): notify only when a category changed above.
604 NotifyNewSuggestions(); 653 NotifyNewSuggestions();
605 } 654 }
606 655
607 void NTPSnippetsService::MergeSnippets(Category category, 656 void NTPSnippetsService::ArchiveSnippets(Category category,
608 NTPSnippet::PtrVector new_snippets) { 657 NTPSnippet::PtrVector* to_archive) {
658 CategoryContent* content = &categories_[category];
659
660 // TODO(sfiera): handle DB for non-articles too.
661 if (category == articles_category_) {
662 database_->DeleteSnippets(*to_archive);
663 // Do not delete the thumbnail images as they are still handy on open NTPs.
664 }
665
666 // Archive previous snippets - move them at the beginning of the list.
667 content->archived.insert(content->archived.begin(),
668 std::make_move_iterator(to_archive->begin()),
669 std::make_move_iterator(to_archive->end()));
670 Compact(to_archive);
671
672 // If there are more archived snippets than we want to keep, delete the
673 // oldest ones by their fetch time (which are always in the back).
674 if (content->archived.size() > kMaxArchivedSnippetCount) {
675 NTPSnippet::PtrVector to_delete(
676 std::make_move_iterator(content->archived.begin() +
677 kMaxArchivedSnippetCount),
678 std::make_move_iterator(content->archived.end()));
679 content->archived.resize(kMaxArchivedSnippetCount);
680 if (category == articles_category_)
681 database_->DeleteImages(to_delete);
682 }
683 }
684
685 void NTPSnippetsService::ReplaceSnippets(Category category,
686 NTPSnippet::PtrVector new_snippets) {
609 DCHECK(ready()); 687 DCHECK(ready());
610 CategoryContent* content = &categories_[category]; 688 CategoryContent* content = &categories_[category];
611 689
612 // Remove new snippets that we already have, or that have been dismissed. 690 // Remove new snippets that have been dismissed.
613 std::set<std::string> old_snippet_ids; 691 EraseMatchingSnippets(&new_snippets, GetAllIDs(content->dismissed),
614 InsertAllIDs(content->dismissed, &old_snippet_ids); 692 /*match_all_ids=*/true);
615 InsertAllIDs(content->snippets, &old_snippet_ids);
616 new_snippets.erase(
617 std::remove_if(
618 new_snippets.begin(), new_snippets.end(),
619 [&old_snippet_ids](const std::unique_ptr<NTPSnippet>& snippet) {
620 if (old_snippet_ids.count(snippet->id()))
621 return true;
622 for (const SnippetSource& source : snippet->sources()) {
623 if (old_snippet_ids.count(source.url.spec()))
624 return true;
625 }
626 return false;
627 }),
628 new_snippets.end());
629 693
630 // Fill in default publish/expiry dates where required. 694 // Fill in default publish/expiry dates where required.
631 for (std::unique_ptr<NTPSnippet>& snippet : new_snippets) { 695 for (std::unique_ptr<NTPSnippet>& snippet : new_snippets) {
632 if (snippet->publish_date().is_null()) 696 if (snippet->publish_date().is_null())
633 snippet->set_publish_date(base::Time::Now()); 697 snippet->set_publish_date(base::Time::Now());
634 if (snippet->expiry_date().is_null()) { 698 if (snippet->expiry_date().is_null()) {
635 snippet->set_expiry_date( 699 snippet->set_expiry_date(
636 snippet->publish_date() + 700 snippet->publish_date() +
637 base::TimeDelta::FromMinutes(kDefaultExpiryTimeMins)); 701 base::TimeDelta::FromMinutes(kDefaultExpiryTimeMins));
638 } 702 }
(...skipping 14 matching lines...) Expand all
653 new_snippets.end()); 717 new_snippets.end());
654 int num_snippets_dismissed = num_new_snippets - new_snippets.size(); 718 int num_snippets_dismissed = num_new_snippets - new_snippets.size();
655 UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch", 719 UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch",
656 num_snippets_dismissed > 0); 720 num_snippets_dismissed > 0);
657 if (num_snippets_dismissed > 0) { 721 if (num_snippets_dismissed > 0) {
658 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets", 722 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets",
659 num_snippets_dismissed); 723 num_snippets_dismissed);
660 } 724 }
661 } 725 }
662 726
663 // Save new articles to the DB. 727 // Do not touch the current set of snippets if the newly fetched one is empty.
664 // TODO(sfiera): save non-articles to DB too. 728 if (new_snippets.empty())
665 if (category == articles_category_) 729 return;
730
731 // Remove current snippets that have been fetched again. We do not need to
732 // archive those as they will be in the new current set.
733 EraseMatchingSnippets(&content->snippets, GetMainIDs(new_snippets),
734 /*match_all_ids=*/false);
735
736 ArchiveSnippets(category, &content->snippets);
737
738 // TODO(sfiera): handle DB for non-articles too.
739 if (category == articles_category_) {
740 // Save new articles to the DB.
666 database_->SaveSnippets(new_snippets); 741 database_->SaveSnippets(new_snippets);
742 }
667 743
668 // Insert the new snippets at the front. 744 // Insert new snippets.
Marc Treib 2016/09/22 13:05:29 nitty nit: "Insert" isn't correct anymore. I'd jus
jkrcal 2016/09/22 14:07:58 Done.
669 content->snippets.insert(content->snippets.begin(), 745 content->snippets = std::move(new_snippets);
670 std::make_move_iterator(new_snippets.begin()),
671 std::make_move_iterator(new_snippets.end()));
672 } 746 }
673 747
674 std::set<std::string> NTPSnippetsService::GetSnippetHostsFromPrefs() const { 748 std::set<std::string> NTPSnippetsService::GetSnippetHostsFromPrefs() const {
675 std::set<std::string> hosts; 749 std::set<std::string> hosts;
676 const base::ListValue* list = pref_service_->GetList(prefs::kSnippetHosts); 750 const base::ListValue* list = pref_service_->GetList(prefs::kSnippetHosts);
677 for (const auto& value : *list) { 751 for (const auto& value : *list) {
678 std::string str; 752 std::string str;
679 bool success = value->GetAsString(&str); 753 bool success = value->GetAsString(&str);
680 DCHECK(success) << "Failed to parse snippet host from prefs"; 754 DCHECK(success) << "Failed to parse snippet host from prefs";
681 hosts.insert(std::move(str)); 755 hosts.insert(std::move(str));
682 } 756 }
683 return hosts; 757 return hosts;
684 } 758 }
685 759
686 void NTPSnippetsService::StoreSnippetHostsToPrefs( 760 void NTPSnippetsService::StoreSnippetHostsToPrefs(
687 const std::set<std::string>& hosts) { 761 const std::set<std::string>& hosts) {
688 base::ListValue list; 762 base::ListValue list;
689 for (const std::string& host : hosts) 763 for (const std::string& host : hosts)
690 list.AppendString(host); 764 list.AppendString(host);
691 pref_service_->Set(prefs::kSnippetHosts, list); 765 pref_service_->Set(prefs::kSnippetHosts, list);
692 } 766 }
693 767
694 void NTPSnippetsService::ClearExpiredSnippets() { 768 void NTPSnippetsService::ClearExpiredDismissedSnippets() {
695 std::vector<Category> categories_to_erase; 769 std::vector<Category> categories_to_erase;
696 770
697 const base::Time expiry = base::Time::Now(); 771 const base::Time now = base::Time::Now();
698 base::Time next_expiry = base::Time::Max();
699 772
700 for (auto& item : categories_) { 773 for (auto& item : categories_) {
701 Category category = item.first; 774 Category category = item.first;
702 CategoryContent* content = &item.second; 775 CategoryContent* content = &item.second;
703 776
704 // Move expired snippets over into |to_delete|.
705 NTPSnippet::PtrVector to_delete; 777 NTPSnippet::PtrVector to_delete;
706 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) { 778 // Move expired dismissed snippets over into |to_delete|.
707 if (snippet->expiry_date() <= expiry)
708 to_delete.emplace_back(std::move(snippet));
709 }
710 Compact(&content->snippets);
711
712 // Move expired dismissed snippets over into |to_delete| as well.
713 for (std::unique_ptr<NTPSnippet>& snippet : content->dismissed) { 779 for (std::unique_ptr<NTPSnippet>& snippet : content->dismissed) {
714 if (snippet->expiry_date() <= expiry) 780 if (snippet->expiry_date() <= now)
715 to_delete.emplace_back(std::move(snippet)); 781 to_delete.emplace_back(std::move(snippet));
716 } 782 }
717 Compact(&content->dismissed); 783 Compact(&content->dismissed);
718 784
719 // Finally, actually delete the removed snippets from the DB. 785 // Delete the removed article suggestions from the DB.
720 if (category == articles_category_) 786 if (category == articles_category_) {
787 // The image got already deleted when the suggestion was dismissed.
721 database_->DeleteSnippets(to_delete); 788 database_->DeleteSnippets(to_delete);
722
723 if (content->snippets.empty() && content->dismissed.empty()) {
724 if ((category != articles_category_) && !content->provided_by_server)
725 categories_to_erase.push_back(category);
726 continue;
727 } 789 }
728 790
729 for (const auto& snippet : content->snippets) { 791 if (content->snippets.empty() && content->dismissed.empty() &&
730 if (snippet->expiry_date() < next_expiry) 792 category != articles_category_ && !content->provided_by_server) {
731 next_expiry = snippet->expiry_date(); 793 categories_to_erase.push_back(category);
732 }
733 for (const auto& snippet : content->dismissed) {
734 if (snippet->expiry_date() < next_expiry)
735 next_expiry = snippet->expiry_date();
736 } 794 }
737 } 795 }
738 796
739 for (Category category : categories_to_erase) { 797 for (Category category : categories_to_erase) {
740 UpdateCategoryStatus(category, CategoryStatus::NOT_PROVIDED); 798 UpdateCategoryStatus(category, CategoryStatus::NOT_PROVIDED);
741 categories_.erase(category); 799 categories_.erase(category);
742 } 800 }
801 }
743 802
744 // Unless there are no snippets left, schedule a timer for the next expiry. 803 void NTPSnippetsService::ClearOrphanedImages() {
745 DCHECK_GT(next_expiry, expiry); 804 // TODO(jkrcal): Implement. crbug.com/649009
746 if (next_expiry < base::Time::Max()) {
747 expiry_timer_.Start(FROM_HERE, next_expiry - expiry,
748 base::Bind(&NTPSnippetsService::ClearExpiredSnippets,
749 base::Unretained(this)));
750 }
751 } 805 }
752 806
753 void NTPSnippetsService::NukeAllSnippets() { 807 void NTPSnippetsService::NukeAllSnippets() {
754 std::vector<Category> categories_to_erase; 808 std::vector<Category> categories_to_erase;
755 809
756 // Empty the ARTICLES category and remove all others, since they may or may 810 // Empty the ARTICLES category and remove all others, since they may or may
757 // not be personalized. 811 // not be personalized.
758 for (const auto& item : categories_) { 812 for (const auto& item : categories_) {
759 Category category = item.first; 813 Category category = item.first;
760 814
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
815 869
816 FetchSnippetImageFromNetwork(suggestion_id, callback); 870 FetchSnippetImageFromNetwork(suggestion_id, callback);
817 } 871 }
818 872
819 void NTPSnippetsService::FetchSnippetImageFromNetwork( 873 void NTPSnippetsService::FetchSnippetImageFromNetwork(
820 const std::string& suggestion_id, 874 const std::string& suggestion_id,
821 const ImageFetchedCallback& callback) { 875 const ImageFetchedCallback& callback) {
822 Category category = GetCategoryFromUniqueID(suggestion_id); 876 Category category = GetCategoryFromUniqueID(suggestion_id);
823 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id); 877 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id);
824 878
825 auto category_it = categories_.find(category); 879 if (categories_.find(category) == categories_.end()) {
826 if (category_it == categories_.end()) {
827 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image()); 880 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image());
828 return; 881 return;
829 } 882 }
830 883
831 const CategoryContent& content = category_it->second; 884 GURL image_url = FindSnippetImageUrl(category, snippet_id);
832 auto it =
833 std::find_if(content.snippets.begin(), content.snippets.end(),
834 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
835 return snippet->id() == snippet_id;
836 });
837 885
838 if (it == content.snippets.end() || 886 if (image_url.is_empty() ||
839 !thumbnail_requests_throttler_.DemandQuotaForRequest( 887 !thumbnail_requests_throttler_.DemandQuotaForRequest(
840 /*interactive_request=*/true)) { 888 /*interactive_request=*/true)) {
841 // Return an empty image. Directly, this is never synchronous with the 889 // Return an empty image. Directly, this is never synchronous with the
842 // original FetchSuggestionImage() call - an asynchronous database query has 890 // original FetchSuggestionImage() call - an asynchronous database query has
843 // happened in the meantime. 891 // happened in the meantime.
844 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image()); 892 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image());
845 return; 893 return;
846 } 894 }
847 895
848 const NTPSnippet& snippet = *it->get();
849
850 // TODO(jkrcal): We probably should rename OnImageDataFetched() to
851 // CacheImageData(). This would document that this is actually independent
852 // from the individual fetch-flow.
853 // The image fetcher calls OnImageDataFetched() with the raw data (this object
854 // is an ImageFetcherDelegate) and then also
855 // OnSnippetImageDecodedFromNetwork() after the raw data gets decoded.
856 image_fetcher_->StartOrQueueNetworkRequest( 896 image_fetcher_->StartOrQueueNetworkRequest(
857 suggestion_id, snippet.salient_image_url(), 897 suggestion_id, image_url,
858 base::Bind(&NTPSnippetsService::OnSnippetImageDecodedFromNetwork, 898 base::Bind(&NTPSnippetsService::OnSnippetImageDecodedFromNetwork,
859 base::Unretained(this), callback)); 899 base::Unretained(this), callback));
860 } 900 }
861 901
862 void NTPSnippetsService::OnSnippetImageDecodedFromNetwork( 902 void NTPSnippetsService::OnSnippetImageDecodedFromNetwork(
863 const ImageFetchedCallback& callback, 903 const ImageFetchedCallback& callback,
864 const std::string& suggestion_id, 904 const std::string& suggestion_id,
865 const gfx::Image& image) { 905 const gfx::Image& image) {
866 callback.Run(image); 906 callback.Run(image);
867 } 907 }
(...skipping 15 matching lines...) Expand all
883 suggestions_service_subscription_ = 923 suggestions_service_subscription_ =
884 suggestions_service_->AddCallback(base::Bind( 924 suggestions_service_->AddCallback(base::Bind(
885 &NTPSnippetsService::OnSuggestionsChanged, base::Unretained(this))); 925 &NTPSnippetsService::OnSuggestionsChanged, base::Unretained(this)));
886 } 926 }
887 927
888 RescheduleFetching(); 928 RescheduleFetching();
889 } 929 }
890 930
891 void NTPSnippetsService::EnterStateDisabled() { 931 void NTPSnippetsService::EnterStateDisabled() {
892 NukeAllSnippets(); 932 NukeAllSnippets();
893 expiry_timer_.Stop();
894 suggestions_service_subscription_.reset(); 933 suggestions_service_subscription_.reset();
895 RescheduleFetching(); 934 RescheduleFetching();
896 } 935 }
897 936
898 void NTPSnippetsService::EnterStateError() { 937 void NTPSnippetsService::EnterStateError() {
899 expiry_timer_.Stop();
900 suggestions_service_subscription_.reset(); 938 suggestions_service_subscription_.reset();
901 RescheduleFetching(); 939 RescheduleFetching();
902 snippets_status_service_.reset(); 940 snippets_status_service_.reset();
903 } 941 }
904 942
905 void NTPSnippetsService::FinishInitialization() { 943 void NTPSnippetsService::FinishInitialization() {
906 if (nuke_after_load_) { 944 if (nuke_after_load_) {
907 NukeAllSnippets(); 945 NukeAllSnippets();
908 nuke_after_load_ = false; 946 nuke_after_load_ = false;
909 } 947 }
(...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after
1044 } 1082 }
1045 1083
1046 NTPSnippetsService::CategoryContent::CategoryContent() = default; 1084 NTPSnippetsService::CategoryContent::CategoryContent() = default;
1047 NTPSnippetsService::CategoryContent::CategoryContent(CategoryContent&&) = 1085 NTPSnippetsService::CategoryContent::CategoryContent(CategoryContent&&) =
1048 default; 1086 default;
1049 NTPSnippetsService::CategoryContent::~CategoryContent() = default; 1087 NTPSnippetsService::CategoryContent::~CategoryContent() = default;
1050 NTPSnippetsService::CategoryContent& NTPSnippetsService::CategoryContent:: 1088 NTPSnippetsService::CategoryContent& NTPSnippetsService::CategoryContent::
1051 operator=(CategoryContent&&) = default; 1089 operator=(CategoryContent&&) = default;
1052 1090
1053 } // namespace ntp_snippets 1091 } // namespace ntp_snippets
OLDNEW
« no previous file with comments | « components/ntp_snippets/ntp_snippets_service.h ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698