Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(533)

Side by Side Diff: components/ntp_snippets/ntp_snippets_service.cc

Issue 2355393002: New snippets now replace old snippets and do not merge (Closed)
Patch Set: Marc's comments + further changes to make unittests happy Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2015 The Chromium Authors. All rights reserved. 1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "components/ntp_snippets/ntp_snippets_service.h" 5 #include "components/ntp_snippets/ntp_snippets_service.h"
6 6
7 #include <algorithm> 7 #include <algorithm>
8 #include <iterator> 8 #include <iterator>
9 #include <utility> 9 #include <utility>
10 10
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after
43 using suggestions::SuggestionsService; 43 using suggestions::SuggestionsService;
44 44
45 namespace ntp_snippets { 45 namespace ntp_snippets {
46 46
47 namespace { 47 namespace {
48 48
49 // Number of snippets requested to the server. Consider replacing sparse UMA 49 // Number of snippets requested to the server. Consider replacing sparse UMA
50 // histograms with COUNTS() if this number increases beyond 50. 50 // histograms with COUNTS() if this number increases beyond 50.
51 const int kMaxSnippetCount = 10; 51 const int kMaxSnippetCount = 10;
52 52
53 // Number of archived snippets we keep around in memory.
54 const int kMaxArchivedSnippetCount = 200;
55
53 // Default values for snippets fetching intervals - once per day only. 56 // Default values for snippets fetching intervals - once per day only.
54 const int kDefaultFetchingIntervalWifiSeconds = 0; 57 const int kDefaultFetchingIntervalWifiSeconds = 0;
55 const int kDefaultFetchingIntervalFallbackSeconds = 24 * 60 * 60; 58 const int kDefaultFetchingIntervalFallbackSeconds = 24 * 60 * 60;
56 59
57 // Variation parameters than can override the default fetching intervals. 60 // Variation parameters than can override the default fetching intervals.
58 const char kFetchingIntervalWifiParamName[] = 61 const char kFetchingIntervalWifiParamName[] =
59 "fetching_interval_wifi_seconds"; 62 "fetching_interval_wifi_seconds";
60 const char kFetchingIntervalFallbackParamName[] = 63 const char kFetchingIntervalFallbackParamName[] =
61 "fetching_interval_fallback_seconds"; 64 "fetching_interval_fallback_seconds";
62 65
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
111 std::set<std::string> hosts; 114 std::set<std::string> hosts;
112 for (int i = 0; i < suggestions.suggestions_size(); ++i) { 115 for (int i = 0; i < suggestions.suggestions_size(); ++i) {
113 const ChromeSuggestion& suggestion = suggestions.suggestions(i); 116 const ChromeSuggestion& suggestion = suggestions.suggestions(i);
114 GURL url(suggestion.url()); 117 GURL url(suggestion.url());
115 if (url.is_valid()) 118 if (url.is_valid())
116 hosts.insert(url.host()); 119 hosts.insert(url.host());
117 } 120 }
118 return hosts; 121 return hosts;
119 } 122 }
120 123
121 void InsertAllIDs(const NTPSnippet::PtrVector& snippets, 124 std::set<std::string> GetAllIDs(const NTPSnippet::PtrVector& snippets) {
122 std::set<std::string>* ids) { 125 std::set<std::string> ids;
123 for (const std::unique_ptr<NTPSnippet>& snippet : snippets) { 126 for (const std::unique_ptr<NTPSnippet>& snippet : snippets) {
124 ids->insert(snippet->id()); 127 ids.insert(snippet->id());
125 for (const SnippetSource& source : snippet->sources()) 128 for (const SnippetSource& source : snippet->sources())
126 ids->insert(source.url.spec()); 129 ids.insert(source.url.spec());
127 } 130 }
131 return ids;
132 }
133
134 std::set<std::string> GetMainIDs(const NTPSnippet::PtrVector& snippets) {
135 std::set<std::string> ids;
136 for (const std::unique_ptr<NTPSnippet>& snippet : snippets)
137 ids.insert(snippet->id());
138 return ids;
139 }
140
141 bool IsSnippetInSet(const std::unique_ptr<NTPSnippet>& snippet,
142 const std::set<std::string>& ids,
143 bool match_all_ids) {
144 if (ids.count(snippet->id()))
145 return true;
146 if (!match_all_ids)
147 return false;
148 for (const SnippetSource& source : snippet->sources()) {
149 if (ids.count(source.url.spec()))
150 return true;
151 }
152 return false;
153 }
154
155 void EraseMatchingSnippets(NTPSnippet::PtrVector* snippets,
156 const std::set<std::string>& matching_ids,
157 bool match_all_ids) {
158 snippets->erase(
159 std::remove_if(snippets->begin(), snippets->end(),
160 [&matching_ids, match_all_ids](
161 const std::unique_ptr<NTPSnippet>& snippet) {
162 return IsSnippetInSet(snippet, matching_ids,
163 match_all_ids);
164 }),
165 snippets->end());
128 } 166 }
129 167
130 void Compact(NTPSnippet::PtrVector* snippets) { 168 void Compact(NTPSnippet::PtrVector* snippets) {
131 snippets->erase( 169 snippets->erase(
132 std::remove_if( 170 std::remove_if(
133 snippets->begin(), snippets->end(), 171 snippets->begin(), snippets->end(),
134 [](const std::unique_ptr<NTPSnippet>& snippet) { return !snippet; }), 172 [](const std::unique_ptr<NTPSnippet>& snippet) { return !snippet; }),
135 snippets->end()); 173 snippets->end());
136 } 174 }
137 175
(...skipping 178 matching lines...) Expand 10 before | Expand all | Expand 10 after
316 void NTPSnippetsService::ClearCachedSuggestions(Category category) { 354 void NTPSnippetsService::ClearCachedSuggestions(Category category) {
317 if (!initialized()) 355 if (!initialized())
318 return; 356 return;
319 357
320 if (categories_.find(category) == categories_.end()) 358 if (categories_.find(category) == categories_.end())
321 return; 359 return;
322 CategoryContent* content = &categories_[category]; 360 CategoryContent* content = &categories_[category];
323 if (content->snippets.empty()) 361 if (content->snippets.empty())
324 return; 362 return;
325 363
326 if (category == articles_category_) 364 if (category == articles_category_) {
327 database_->DeleteSnippets(content->snippets); 365 database_->DeleteSnippets(content->snippets);
366 database_->DeleteImages(content->snippets);
367 }
328 content->snippets.clear(); 368 content->snippets.clear();
329 369
330 NotifyNewSuggestions(); 370 NotifyNewSuggestions();
331 } 371 }
332 372
333 void NTPSnippetsService::GetDismissedSuggestionsForDebugging( 373 void NTPSnippetsService::GetDismissedSuggestionsForDebugging(
334 Category category, 374 Category category,
335 const DismissedSuggestionsCallback& callback) { 375 const DismissedSuggestionsCallback& callback) {
336 DCHECK(categories_.find(category) != categories_.end()); 376 DCHECK(categories_.find(category) != categories_.end());
337 377
(...skipping 20 matching lines...) Expand all
358 Category category) { 398 Category category) {
359 DCHECK(categories_.find(category) != categories_.end()); 399 DCHECK(categories_.find(category) != categories_.end());
360 400
361 if (!initialized()) 401 if (!initialized())
362 return; 402 return;
363 403
364 CategoryContent* content = &categories_[category]; 404 CategoryContent* content = &categories_[category];
365 if (content->dismissed.empty()) 405 if (content->dismissed.empty())
366 return; 406 return;
367 407
368 if (category == articles_category_) 408 if (category == articles_category_) {
409 // The image got already deleted when the suggestion was dismissed.
369 database_->DeleteSnippets(content->dismissed); 410 database_->DeleteSnippets(content->dismissed);
411 }
370 content->dismissed.clear(); 412 content->dismissed.clear();
371 } 413 }
372 414
373 std::set<std::string> NTPSnippetsService::GetSuggestionsHosts() const { 415 std::set<std::string> NTPSnippetsService::GetSuggestionsHosts() const {
374 // |suggestions_service_| can be null in tests. 416 // |suggestions_service_| can be null in tests.
375 if (!suggestions_service_) 417 if (!suggestions_service_)
376 return std::set<std::string>(); 418 return std::set<std::string>();
377 419
378 // TODO(treib): This should just call GetSnippetHostsFromPrefs. 420 // TODO(treib): This should just call GetSnippetHostsFromPrefs.
379 return GetSuggestionsHostsImpl( 421 return GetSuggestionsHostsImpl(
380 suggestions_service_->GetSuggestionsDataFromCache()); 422 suggestions_service_->GetSuggestionsDataFromCache());
381 } 423 }
382 424
383 // static 425 // static
384 int NTPSnippetsService::GetMaxSnippetCountForTesting() { 426 int NTPSnippetsService::GetMaxSnippetCountForTesting() {
385 return kMaxSnippetCount; 427 return kMaxSnippetCount;
386 } 428 }
387 429
388 //////////////////////////////////////////////////////////////////////////////// 430 ////////////////////////////////////////////////////////////////////////////////
389 // Private methods 431 // Private methods
390 432
433 GURL NTPSnippetsService::FindSnippetImageUrl(
434 Category category,
435 const std::string& snippet_id) const {
436 DCHECK(categories_.find(category) != categories_.end());
437
438 const CategoryContent& content = categories_.at(category);
439 // Search for the snippet in current and archived snippets.
440 auto it =
441 std::find_if(content.snippets.begin(), content.snippets.end(),
442 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
443 return snippet->id() == snippet_id;
444 });
445 if (it != content.snippets.end())
446 return (*it)->salient_image_url();
447
448 it = std::find_if(content.archived.begin(), content.archived.end(),
449 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
450 return snippet->id() == snippet_id;
451 });
452 if (it != content.archived.end())
453 return (*it)->salient_image_url();
454
455 return GURL();
456 }
457
391 // image_fetcher::ImageFetcherDelegate implementation. 458 // image_fetcher::ImageFetcherDelegate implementation.
392 void NTPSnippetsService::OnImageDataFetched(const std::string& suggestion_id, 459 void NTPSnippetsService::OnImageDataFetched(const std::string& suggestion_id,
393 const std::string& image_data) { 460 const std::string& image_data) {
394 if (image_data.empty()) 461 if (image_data.empty())
395 return; 462 return;
396 463
397 Category category = GetCategoryFromUniqueID(suggestion_id); 464 Category category = GetCategoryFromUniqueID(suggestion_id);
398 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id); 465 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id);
399 466
400 auto category_it = categories_.find(category); 467 if (categories_.find(category) == categories_.end())
401 if (category_it == categories_.end())
402 return; 468 return;
403 469
404 const CategoryContent& content = category_it->second;
405
406 // Only save the image if the corresponding snippet still exists. 470 // Only save the image if the corresponding snippet still exists.
407 auto it = 471 if (FindSnippetImageUrl(category, snippet_id).is_empty())
408 std::find_if(content.snippets.begin(), content.snippets.end(),
409 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
410 return snippet->id() == snippet_id;
411 });
412 if (it == content.snippets.end())
413 return; 472 return;
414 473
474 // Only cache the data in the DB, the actual serving is done in the callback
475 // provided to |image_fetcher_| (OnSnippetImageDecodedFromNetwork()).
415 database_->SaveImage(snippet_id, image_data); 476 database_->SaveImage(snippet_id, image_data);
416 } 477 }
417 478
418 void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) { 479 void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) {
419 if (state_ == State::ERROR_OCCURRED) 480 if (state_ == State::ERROR_OCCURRED)
420 return; 481 return;
421 DCHECK(state_ == State::NOT_INITED); 482 DCHECK(state_ == State::NOT_INITED);
422 DCHECK(categories_.size() == 1); // Only articles category, so far. 483 DCHECK_EQ(1u, categories_.size()); // Only articles category, so far.
423 DCHECK(categories_.find(articles_category_) != categories_.end()); 484 DCHECK(categories_.find(articles_category_) != categories_.end());
424 485
425 // TODO(sfiera): support non-article categories in database. 486 // TODO(sfiera): support non-article categories in database.
426 CategoryContent* content = &categories_[articles_category_]; 487 CategoryContent* content = &categories_[articles_category_];
427 for (std::unique_ptr<NTPSnippet>& snippet : snippets) { 488 for (std::unique_ptr<NTPSnippet>& snippet : snippets) {
428 if (snippet->is_dismissed()) 489 if (snippet->is_dismissed())
429 content->dismissed.emplace_back(std::move(snippet)); 490 content->dismissed.emplace_back(std::move(snippet));
430 else 491 else
431 content->snippets.emplace_back(std::move(snippet)); 492 content->snippets.emplace_back(std::move(snippet));
432 } 493 }
433 494
434 std::sort(content->snippets.begin(), content->snippets.end(), 495 std::sort(content->snippets.begin(), content->snippets.end(),
435 [](const std::unique_ptr<NTPSnippet>& lhs, 496 [](const std::unique_ptr<NTPSnippet>& lhs,
436 const std::unique_ptr<NTPSnippet>& rhs) { 497 const std::unique_ptr<NTPSnippet>& rhs) {
437 return lhs->score() > rhs->score(); 498 return lhs->score() > rhs->score();
438 }); 499 });
439 500
440 ClearExpiredSnippets(); 501 ClearExpiredDismissedSnippets();
502 ClearOrphanedImages();
441 FinishInitialization(); 503 FinishInitialization();
442 } 504 }
443 505
444 void NTPSnippetsService::OnDatabaseError() { 506 void NTPSnippetsService::OnDatabaseError() {
445 EnterState(State::ERROR_OCCURRED); 507 EnterState(State::ERROR_OCCURRED);
446 UpdateAllCategoryStatus(CategoryStatus::LOADING_ERROR); 508 UpdateAllCategoryStatus(CategoryStatus::LOADING_ERROR);
447 } 509 }
448 510
449 // TODO(dgn): name clash between content suggestions and suggestions hosts. 511 // TODO(dgn): name clash between content suggestions and suggestions hosts.
450 // method name should be changed. 512 // method name should be changed.
(...skipping 14 matching lines...) Expand all
465 // and apply the same logic to them here. Maybe never? 527 // and apply the same logic to them here. Maybe never?
466 // 528 //
467 // First, move them over into |to_delete|. 529 // First, move them over into |to_delete|.
468 CategoryContent* content = &categories_[articles_category_]; 530 CategoryContent* content = &categories_[articles_category_];
469 NTPSnippet::PtrVector to_delete; 531 NTPSnippet::PtrVector to_delete;
470 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) { 532 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) {
471 if (!hosts.count(snippet->best_source().url.host())) 533 if (!hosts.count(snippet->best_source().url.host()))
472 to_delete.emplace_back(std::move(snippet)); 534 to_delete.emplace_back(std::move(snippet));
473 } 535 }
474 Compact(&content->snippets); 536 Compact(&content->snippets);
475 // Then delete the removed snippets from the database. 537 ArchiveSnippets(articles_category_, &to_delete);
476 database_->DeleteSnippets(to_delete);
477 538
478 StoreSnippetHostsToPrefs(hosts); 539 StoreSnippetHostsToPrefs(hosts);
479 540
480 // We removed some suggestions, so we want to let the client know about that. 541 // We removed some suggestions, so we want to let the client know about that.
481 // The fetch might take a long time or not complete so we don't want to wait 542 // The fetch might take a long time or not complete so we don't want to wait
482 // for its callback. 543 // for its callback.
483 NotifyNewSuggestions(); 544 NotifyNewSuggestions();
484 545
485 FetchSnippetsFromHosts(hosts, /*force_request=*/false); 546 FetchSnippetsFromHosts(hosts, /*force_request=*/false);
486 } 547 }
487 548
488 void NTPSnippetsService::OnFetchFinished( 549 void NTPSnippetsService::OnFetchFinished(
489 NTPSnippetsFetcher::OptionalSnippets snippets) { 550 NTPSnippetsFetcher::OptionalSnippets snippets) {
490 if (!ready()) 551 if (!ready())
491 return; 552 return;
492 553
493 for (auto& item : categories_) { 554 for (auto& item : categories_) {
494 CategoryContent* content = &item.second; 555 CategoryContent* content = &item.second;
495 content->provided_by_server = false; 556 content->provided_by_server = false;
496 } 557 }
497 558
559 // Clear up expired dismissed snippets before we use them to filter new ones.
560 ClearExpiredDismissedSnippets();
561
498 // If snippets were fetched successfully, update our |categories_| from each 562 // If snippets were fetched successfully, update our |categories_| from each
499 // category provided by the server. 563 // category provided by the server.
500 if (snippets) { 564 if (snippets) {
565 // TODO(jkrcal): A bit hard to understand with so many variables called
566 // "*categor*". Isn't here some room for simplification?
501 for (NTPSnippetsFetcher::FetchedCategory& fetched_category : *snippets) { 567 for (NTPSnippetsFetcher::FetchedCategory& fetched_category : *snippets) {
502 Category category = fetched_category.category; 568 Category category = fetched_category.category;
503 569
504 // TODO(sfiera): Avoid hard-coding articles category checks in so many 570 // TODO(sfiera): Avoid hard-coding articles category checks in so many
505 // places. 571 // places.
506 if (category != articles_category_) { 572 if (category != articles_category_) {
507 // Only update titles from server-side provided categories. 573 // Only update titles from server-side provided categories.
508 categories_[category].localized_title = 574 categories_[category].localized_title =
509 fetched_category.localized_title; 575 fetched_category.localized_title;
510 } 576 }
577 categories_[category].provided_by_server = true;
511 578
512 DCHECK_LE(snippets->size(), static_cast<size_t>(kMaxSnippetCount)); 579 DCHECK_LE(snippets->size(), static_cast<size_t>(kMaxSnippetCount));
513 // TODO(sfiera): histograms for server categories. 580 // TODO(sfiera): histograms for server categories.
514 // Sparse histogram used because the number of snippets is small (bound by 581 // Sparse histogram used because the number of snippets is small (bound by
515 // kMaxSnippetCount). 582 // kMaxSnippetCount).
516 if (category == articles_category_) { 583 if (category == articles_category_) {
517 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticlesFetched", 584 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticlesFetched",
518 fetched_category.snippets.size()); 585 fetched_category.snippets.size());
519 } 586 }
520 587
521 MergeSnippets(category, std::move(fetched_category.snippets)); 588 ReplaceSnippets(category, std::move(fetched_category.snippets));
522
523 // If there are more snippets than we want to show, delete the extra ones.
524 CategoryContent* content = &categories_[category];
525 content->provided_by_server = true;
526 if (content->snippets.size() > kMaxSnippetCount) {
527 NTPSnippet::PtrVector to_delete(
528 std::make_move_iterator(content->snippets.begin() +
529 kMaxSnippetCount),
530 std::make_move_iterator(content->snippets.end()));
531 content->snippets.resize(kMaxSnippetCount);
532 if (category == articles_category_)
533 database_->DeleteSnippets(to_delete);
534 }
535 } 589 }
536 } 590 }
537 591
538 // Trigger expiration. This probably won't expire any current snippets (old
539 // ones should have already been expired by the timer, and new ones shouldn't
540 // have expired yet), but it will update the timer for the next run.
541 ClearExpiredSnippets();
542
543 for (const auto& item : categories_) { 592 for (const auto& item : categories_) {
544 Category category = item.first; 593 Category category = item.first;
545 UpdateCategoryStatus(category, CategoryStatus::AVAILABLE); 594 UpdateCategoryStatus(category, CategoryStatus::AVAILABLE);
546 } 595 }
547 596
548 // TODO(sfiera): equivalent metrics for non-articles. 597 // TODO(sfiera): equivalent metrics for non-articles.
549 const CategoryContent& content = categories_[articles_category_]; 598 const CategoryContent& content = categories_[articles_category_];
550 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles", 599 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles",
551 content.snippets.size()); 600 content.snippets.size());
552 if (content.snippets.empty() && !content.dismissed.empty()) { 601 if (content.snippets.empty() && !content.dismissed.empty()) {
553 UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded", 602 UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded",
554 content.dismissed.size()); 603 content.dismissed.size());
555 } 604 }
556 605
557 // TODO(sfiera): notify only when a category changed above. 606 // TODO(sfiera): notify only when a category changed above.
558 NotifyNewSuggestions(); 607 NotifyNewSuggestions();
559 608
560 // Reschedule after a successful fetch. This resets all currently scheduled 609 // Reschedule after a successful fetch. This resets all currently scheduled
561 // fetches, to make sure the fallback interval triggers only if no wifi fetch 610 // fetches, to make sure the fallback interval triggers only if no wifi fetch
562 // succeeded, and also that we don't do a background fetch immediately after 611 // succeeded, and also that we don't do a background fetch immediately after
563 // a user-initiated one. 612 // a user-initiated one.
564 if (snippets) 613 if (snippets)
565 RescheduleFetching(); 614 RescheduleFetching();
566 } 615 }
567 616
568 void NTPSnippetsService::MergeSnippets(Category category, 617 void NTPSnippetsService::ArchiveSnippets(Category category,
569 NTPSnippet::PtrVector new_snippets) { 618 NTPSnippet::PtrVector* to_archive) {
619 CategoryContent* content = &categories_[category];
620
621 // TODO(sfiera): handle DB for non-articles too.
622 if (category == articles_category_) {
623 database_->DeleteSnippets(*to_archive);
624 // Do not delete the thumbnail images as they are still handy on open NTPs.
625 }
626
627 // Archive previous snippets - move them at the beginning of the list.
628 content->archived.insert(content->archived.begin(),
629 std::make_move_iterator(to_archive->begin()),
630 std::make_move_iterator(to_archive->end()));
631 Compact(to_archive);
632
633 // If there are more archived snippets than we want to keep, delete the
634 // oldest ones by their fetch time (which are always in the back).
635 if (content->archived.size() > kMaxArchivedSnippetCount) {
636 NTPSnippet::PtrVector to_delete(
637 std::make_move_iterator(content->archived.begin() +
638 kMaxArchivedSnippetCount),
639 std::make_move_iterator(content->archived.end()));
640 content->archived.resize(kMaxArchivedSnippetCount);
641 if (category == articles_category_)
642 database_->DeleteImages(to_delete);
643 }
644 }
645
646 void NTPSnippetsService::ReplaceSnippets(Category category,
647 NTPSnippet::PtrVector new_snippets) {
570 DCHECK(ready()); 648 DCHECK(ready());
571 CategoryContent* content = &categories_[category]; 649 CategoryContent* content = &categories_[category];
572 650
573 // Remove new snippets that we already have, or that have been dismissed. 651 // Remove new snippets that have been dismissed.
574 std::set<std::string> old_snippet_ids; 652 EraseMatchingSnippets(&new_snippets, GetAllIDs(content->dismissed),
575 InsertAllIDs(content->dismissed, &old_snippet_ids); 653 /*match_all_ids=*/true);
576 InsertAllIDs(content->snippets, &old_snippet_ids);
577 new_snippets.erase(
578 std::remove_if(
579 new_snippets.begin(), new_snippets.end(),
580 [&old_snippet_ids](const std::unique_ptr<NTPSnippet>& snippet) {
581 if (old_snippet_ids.count(snippet->id()))
582 return true;
583 for (const SnippetSource& source : snippet->sources()) {
584 if (old_snippet_ids.count(source.url.spec()))
585 return true;
586 }
587 return false;
588 }),
589 new_snippets.end());
590 654
591 // Fill in default publish/expiry dates where required. 655 // Fill in default publish/expiry dates where required.
592 for (std::unique_ptr<NTPSnippet>& snippet : new_snippets) { 656 for (std::unique_ptr<NTPSnippet>& snippet : new_snippets) {
593 if (snippet->publish_date().is_null()) 657 if (snippet->publish_date().is_null())
594 snippet->set_publish_date(base::Time::Now()); 658 snippet->set_publish_date(base::Time::Now());
595 if (snippet->expiry_date().is_null()) { 659 if (snippet->expiry_date().is_null()) {
596 snippet->set_expiry_date( 660 snippet->set_expiry_date(
597 snippet->publish_date() + 661 snippet->publish_date() +
598 base::TimeDelta::FromMinutes(kDefaultExpiryTimeMins)); 662 base::TimeDelta::FromMinutes(kDefaultExpiryTimeMins));
599 } 663 }
(...skipping 14 matching lines...) Expand all
614 new_snippets.end()); 678 new_snippets.end());
615 int num_snippets_dismissed = num_new_snippets - new_snippets.size(); 679 int num_snippets_dismissed = num_new_snippets - new_snippets.size();
616 UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch", 680 UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch",
617 num_snippets_dismissed > 0); 681 num_snippets_dismissed > 0);
618 if (num_snippets_dismissed > 0) { 682 if (num_snippets_dismissed > 0) {
619 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets", 683 UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets",
620 num_snippets_dismissed); 684 num_snippets_dismissed);
621 } 685 }
622 } 686 }
623 687
624 // Save new articles to the DB. 688 // Do not touch the current set of snippets if the newly fetched one is empty.
625 // TODO(sfiera): save non-articles to DB too. 689 if (new_snippets.empty())
626 if (category == articles_category_) 690 return;
691
692 // Remove current snippets that have been fetched again. We do not need to
693 // archive those as they will be in the new current set.
694 EraseMatchingSnippets(&content->snippets, GetMainIDs(new_snippets),
695 /*match_all_ids=*/false);
696
697 ArchiveSnippets(category, &content->snippets);
698
699 // TODO(sfiera): handle DB for non-articles too.
700 if (category == articles_category_) {
701 // Save new articles to the DB.
627 database_->SaveSnippets(new_snippets); 702 database_->SaveSnippets(new_snippets);
703 }
628 704
629 // Insert the new snippets at the front. 705 content->snippets = std::move(new_snippets);
630 content->snippets.insert(content->snippets.begin(),
631 std::make_move_iterator(new_snippets.begin()),
632 std::make_move_iterator(new_snippets.end()));
633 } 706 }
634 707
635 std::set<std::string> NTPSnippetsService::GetSnippetHostsFromPrefs() const { 708 std::set<std::string> NTPSnippetsService::GetSnippetHostsFromPrefs() const {
636 std::set<std::string> hosts; 709 std::set<std::string> hosts;
637 const base::ListValue* list = pref_service_->GetList(prefs::kSnippetHosts); 710 const base::ListValue* list = pref_service_->GetList(prefs::kSnippetHosts);
638 for (const auto& value : *list) { 711 for (const auto& value : *list) {
639 std::string str; 712 std::string str;
640 bool success = value->GetAsString(&str); 713 bool success = value->GetAsString(&str);
641 DCHECK(success) << "Failed to parse snippet host from prefs"; 714 DCHECK(success) << "Failed to parse snippet host from prefs";
642 hosts.insert(std::move(str)); 715 hosts.insert(std::move(str));
643 } 716 }
644 return hosts; 717 return hosts;
645 } 718 }
646 719
647 void NTPSnippetsService::StoreSnippetHostsToPrefs( 720 void NTPSnippetsService::StoreSnippetHostsToPrefs(
648 const std::set<std::string>& hosts) { 721 const std::set<std::string>& hosts) {
649 base::ListValue list; 722 base::ListValue list;
650 for (const std::string& host : hosts) 723 for (const std::string& host : hosts)
651 list.AppendString(host); 724 list.AppendString(host);
652 pref_service_->Set(prefs::kSnippetHosts, list); 725 pref_service_->Set(prefs::kSnippetHosts, list);
653 } 726 }
654 727
655 void NTPSnippetsService::ClearExpiredSnippets() { 728 void NTPSnippetsService::ClearExpiredDismissedSnippets() {
656 std::vector<Category> categories_to_erase; 729 std::vector<Category> categories_to_erase;
657 730
658 const base::Time expiry = base::Time::Now(); 731 const base::Time now = base::Time::Now();
659 base::Time next_expiry = base::Time::Max();
660 732
661 for (auto& item : categories_) { 733 for (auto& item : categories_) {
662 Category category = item.first; 734 Category category = item.first;
663 CategoryContent* content = &item.second; 735 CategoryContent* content = &item.second;
664 736
665 // Move expired snippets over into |to_delete|.
666 NTPSnippet::PtrVector to_delete; 737 NTPSnippet::PtrVector to_delete;
667 for (std::unique_ptr<NTPSnippet>& snippet : content->snippets) { 738 // Move expired dismissed snippets over into |to_delete|.
668 if (snippet->expiry_date() <= expiry)
669 to_delete.emplace_back(std::move(snippet));
670 }
671 Compact(&content->snippets);
672
673 // Move expired dismissed snippets over into |to_delete| as well.
674 for (std::unique_ptr<NTPSnippet>& snippet : content->dismissed) { 739 for (std::unique_ptr<NTPSnippet>& snippet : content->dismissed) {
675 if (snippet->expiry_date() <= expiry) 740 if (snippet->expiry_date() <= now)
676 to_delete.emplace_back(std::move(snippet)); 741 to_delete.emplace_back(std::move(snippet));
677 } 742 }
678 Compact(&content->dismissed); 743 Compact(&content->dismissed);
679 744
680 // Finally, actually delete the removed snippets from the DB. 745 // Delete the removed article suggestions from the DB.
681 if (category == articles_category_) 746 if (category == articles_category_) {
747 // The image got already deleted when the suggestion was dismissed.
682 database_->DeleteSnippets(to_delete); 748 database_->DeleteSnippets(to_delete);
683
684 if (content->snippets.empty() && content->dismissed.empty()) {
685 if ((category != articles_category_) && !content->provided_by_server)
686 categories_to_erase.push_back(category);
687 continue;
688 } 749 }
689 750
690 for (const auto& snippet : content->snippets) { 751 if (content->snippets.empty() && content->dismissed.empty() &&
691 if (snippet->expiry_date() < next_expiry) 752 category != articles_category_ && !content->provided_by_server) {
692 next_expiry = snippet->expiry_date(); 753 categories_to_erase.push_back(category);
693 }
694 for (const auto& snippet : content->dismissed) {
695 if (snippet->expiry_date() < next_expiry)
696 next_expiry = snippet->expiry_date();
697 } 754 }
698 } 755 }
699 756
700 for (Category category : categories_to_erase) { 757 for (Category category : categories_to_erase) {
701 UpdateCategoryStatus(category, CategoryStatus::NOT_PROVIDED); 758 UpdateCategoryStatus(category, CategoryStatus::NOT_PROVIDED);
702 categories_.erase(category); 759 categories_.erase(category);
703 } 760 }
761 }
704 762
705 // Unless there are no snippets left, schedule a timer for the next expiry. 763 void NTPSnippetsService::ClearOrphanedImages() {
706 DCHECK_GT(next_expiry, expiry); 764 // TODO(jkrcal): Implement. crbug.com/649009
707 if (next_expiry < base::Time::Max()) {
708 expiry_timer_.Start(FROM_HERE, next_expiry - expiry,
709 base::Bind(&NTPSnippetsService::ClearExpiredSnippets,
710 base::Unretained(this)));
711 }
712 } 765 }
713 766
714 void NTPSnippetsService::NukeAllSnippets() { 767 void NTPSnippetsService::NukeAllSnippets() {
715 std::vector<Category> categories_to_erase; 768 std::vector<Category> categories_to_erase;
716 769
717 // Empty the ARTICLES category and remove all others, since they may or may 770 // Empty the ARTICLES category and remove all others, since they may or may
718 // not be personalized. 771 // not be personalized.
719 for (const auto& item : categories_) { 772 for (const auto& item : categories_) {
720 Category category = item.first; 773 Category category = item.first;
721 774
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
776 829
777 FetchSnippetImageFromNetwork(suggestion_id, callback); 830 FetchSnippetImageFromNetwork(suggestion_id, callback);
778 } 831 }
779 832
780 void NTPSnippetsService::FetchSnippetImageFromNetwork( 833 void NTPSnippetsService::FetchSnippetImageFromNetwork(
781 const std::string& suggestion_id, 834 const std::string& suggestion_id,
782 const ImageFetchedCallback& callback) { 835 const ImageFetchedCallback& callback) {
783 Category category = GetCategoryFromUniqueID(suggestion_id); 836 Category category = GetCategoryFromUniqueID(suggestion_id);
784 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id); 837 std::string snippet_id = GetWithinCategoryIDFromUniqueID(suggestion_id);
785 838
786 auto category_it = categories_.find(category); 839 if (categories_.find(category) == categories_.end()) {
787 if (category_it == categories_.end()) {
788 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image()); 840 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image());
789 return; 841 return;
790 } 842 }
791 843
792 const CategoryContent& content = category_it->second; 844 GURL image_url = FindSnippetImageUrl(category, snippet_id);
793 auto it =
794 std::find_if(content.snippets.begin(), content.snippets.end(),
795 [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
796 return snippet->id() == snippet_id;
797 });
798 845
799 if (it == content.snippets.end() || 846 if (image_url.is_empty() ||
800 !thumbnail_requests_throttler_.DemandQuotaForRequest( 847 !thumbnail_requests_throttler_.DemandQuotaForRequest(
801 /*interactive_request=*/true)) { 848 /*interactive_request=*/true)) {
802 // Return an empty image. Directly, this is never synchronous with the 849 // Return an empty image. Directly, this is never synchronous with the
803 // original FetchSuggestionImage() call - an asynchronous database query has 850 // original FetchSuggestionImage() call - an asynchronous database query has
804 // happened in the meantime. 851 // happened in the meantime.
805 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image()); 852 OnSnippetImageDecodedFromNetwork(callback, suggestion_id, gfx::Image());
806 return; 853 return;
807 } 854 }
808 855
809 const NTPSnippet& snippet = *it->get();
810
811 // TODO(jkrcal): We probably should rename OnImageDataFetched() to
812 // CacheImageData(). This would document that this is actually independent
813 // from the individual fetch-flow.
814 // The image fetcher calls OnImageDataFetched() with the raw data (this object
815 // is an ImageFetcherDelegate) and then also
816 // OnSnippetImageDecodedFromNetwork() after the raw data gets decoded.
817 image_fetcher_->StartOrQueueNetworkRequest( 856 image_fetcher_->StartOrQueueNetworkRequest(
818 suggestion_id, snippet.salient_image_url(), 857 suggestion_id, image_url,
819 base::Bind(&NTPSnippetsService::OnSnippetImageDecodedFromNetwork, 858 base::Bind(&NTPSnippetsService::OnSnippetImageDecodedFromNetwork,
820 base::Unretained(this), callback)); 859 base::Unretained(this), callback));
821 } 860 }
822 861
823 void NTPSnippetsService::OnSnippetImageDecodedFromNetwork( 862 void NTPSnippetsService::OnSnippetImageDecodedFromNetwork(
824 const ImageFetchedCallback& callback, 863 const ImageFetchedCallback& callback,
825 const std::string& suggestion_id, 864 const std::string& suggestion_id,
826 const gfx::Image& image) { 865 const gfx::Image& image) {
827 callback.Run(image); 866 callback.Run(image);
828 } 867 }
(...skipping 26 matching lines...) Expand all
855 // |suggestions_service_| can be null in tests. 894 // |suggestions_service_| can be null in tests.
856 if (snippets_fetcher_->UsesHostRestrictions() && suggestions_service_) { 895 if (snippets_fetcher_->UsesHostRestrictions() && suggestions_service_) {
857 suggestions_service_subscription_ = 896 suggestions_service_subscription_ =
858 suggestions_service_->AddCallback(base::Bind( 897 suggestions_service_->AddCallback(base::Bind(
859 &NTPSnippetsService::OnSuggestionsChanged, base::Unretained(this))); 898 &NTPSnippetsService::OnSuggestionsChanged, base::Unretained(this)));
860 } 899 }
861 } 900 }
862 901
863 void NTPSnippetsService::EnterStateDisabled() { 902 void NTPSnippetsService::EnterStateDisabled() {
864 NukeAllSnippets(); 903 NukeAllSnippets();
865 expiry_timer_.Stop();
866 suggestions_service_subscription_.reset(); 904 suggestions_service_subscription_.reset();
867 } 905 }
868 906
869 void NTPSnippetsService::EnterStateError() { 907 void NTPSnippetsService::EnterStateError() {
870 expiry_timer_.Stop();
871 suggestions_service_subscription_.reset(); 908 suggestions_service_subscription_.reset();
872 snippets_status_service_.reset(); 909 snippets_status_service_.reset();
873 } 910 }
874 911
875 void NTPSnippetsService::FinishInitialization() { 912 void NTPSnippetsService::FinishInitialization() {
876 if (nuke_when_initialized_) { 913 if (nuke_when_initialized_) {
877 // We nuke here in addition to EnterStateReady, so that it happens even if 914 // We nuke here in addition to EnterStateReady, so that it happens even if
878 // we enter the DISABLED state below. 915 // we enter the DISABLED state below.
879 NukeAllSnippets(); 916 NukeAllSnippets();
880 nuke_when_initialized_ = false; 917 nuke_when_initialized_ = false;
(...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after
1012 } 1049 }
1013 1050
1014 NTPSnippetsService::CategoryContent::CategoryContent() = default; 1051 NTPSnippetsService::CategoryContent::CategoryContent() = default;
1015 NTPSnippetsService::CategoryContent::CategoryContent(CategoryContent&&) = 1052 NTPSnippetsService::CategoryContent::CategoryContent(CategoryContent&&) =
1016 default; 1053 default;
1017 NTPSnippetsService::CategoryContent::~CategoryContent() = default; 1054 NTPSnippetsService::CategoryContent::~CategoryContent() = default;
1018 NTPSnippetsService::CategoryContent& NTPSnippetsService::CategoryContent:: 1055 NTPSnippetsService::CategoryContent& NTPSnippetsService::CategoryContent::
1019 operator=(CategoryContent&&) = default; 1056 operator=(CategoryContent&&) = default;
1020 1057
1021 } // namespace ntp_snippets 1058 } // namespace ntp_snippets
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698