[NTP Snippets] Cache images in a LevelDB

components/ntp_snippets/ntp_snippets_service.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/ntp_snippets/ntp_snippets_service.h"
 
 #include <algorithm>
 #include <iterator>
 #include <utility>
 
 #include "base/command_line.h"
 #include "base/files/file_path.h"
 #include "base/files/file_util.h"
 #include "base/location.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/metrics/sparse_histogram.h"
 #include "base/path_service.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/task_runner_util.h"
 #include "base/time/time.h"
 #include "base/values.h"
+#include "components/image_fetcher/image_decoder.h"
 #include "components/image_fetcher/image_fetcher.h"
 #include "components/ntp_snippets/ntp_snippets_constants.h"
 #include "components/ntp_snippets/ntp_snippets_database.h"
 #include "components/ntp_snippets/pref_names.h"
 #include "components/ntp_snippets/switches.h"
 #include "components/prefs/pref_registry_simple.h"
 #include "components/prefs/pref_service.h"
 #include "components/suggestions/proto/suggestions.pb.h"
 #include "components/sync_driver/sync_service.h"
 #include "components/variations/variations_associated_data.h"
 #include "ui/gfx/image/image.h"
 
+using image_fetcher::ImageDecoder;
 using image_fetcher::ImageFetcher;
 using suggestions::ChromeSuggestion;
 using suggestions::SuggestionsProfile;
 using suggestions::SuggestionsService;
 
 namespace ntp_snippets {
 
 namespace {
 
 // Number of snippets requested to the server. Consider replacing sparse UMA
@@ skipping 135 matching lines @@
 
 NTPSnippetsService::NTPSnippetsService(
     bool enabled,
     PrefService* pref_service,
     sync_driver::SyncService* sync_service,
     SuggestionsService* suggestions_service,
     const std::string& application_language_code,
     NTPSnippetsScheduler* scheduler,
     std::unique_ptr<NTPSnippetsFetcher> snippets_fetcher,
     std::unique_ptr<ImageFetcher> image_fetcher,
+    std::unique_ptr<ImageDecoder> image_decoder,
     std::unique_ptr<NTPSnippetsDatabase> database)
     : state_(State::NOT_INITED),
       explicitly_disabled_(!enabled),
       pref_service_(pref_service),
       sync_service_(sync_service),
       sync_service_observer_(this),
       suggestions_service_(suggestions_service),
       application_language_code_(application_language_code),
       scheduler_(scheduler),
       snippets_fetcher_(std::move(snippets_fetcher)),
       image_fetcher_(std::move(image_fetcher)),
+      image_decoder_(std::move(image_decoder)),
       database_(std::move(database)),
       fetch_after_load_(false) {
   // TODO(dgn) should be removed after branch point (https:://crbug.com/617585).
   ClearDeprecatedPrefs();
 
   if (explicitly_disabled_) {
     EnterState(State::DISABLED);
     return;
   }
 
   // We transition to other states while finalizing the initialization, when the
   // database is done loading.
-  database_->Load(base::Bind(&NTPSnippetsService::OnDatabaseLoaded,
-                             base::Unretained(this)));
+  database_->LoadSnippets(base::Bind(&NTPSnippetsService::OnDatabaseLoaded,
+                                     base::Unretained(this)));
 }
 
 NTPSnippetsService::~NTPSnippetsService() {
   DCHECK(state_ == State::SHUT_DOWN);
 }
 
 // static
 void NTPSnippetsService::RegisterProfilePrefs(PrefRegistrySimple* registry) {
   registry->RegisterListPref(prefs::kDeprecatedSnippets);
   registry->RegisterListPref(prefs::kDeprecatedDiscardedSnippets);
@@ skipping 31 matching lines @@
         GetFetchingIntervalWifiCharging(), GetFetchingIntervalWifi(now),
         GetFetchingIntervalFallback(), GetRescheduleTime(now));
   } else {
     scheduler_->Unschedule();
   }
 }
 
 void NTPSnippetsService::FetchSnippetImage(
     const std::string& snippet_id,
     const ImageFetchedCallback& callback) {
-  auto it =
-      std::find_if(snippets_.begin(), snippets_.end(),
-                   [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
-                     return snippet->id() == snippet_id;
-                   });
-  if (it == snippets_.end()) {
-    gfx::Image empty_image;
-    callback.Run(snippet_id, empty_image);
-    return;
-  }
-
-  const NTPSnippet& snippet = *it->get();
-  image_fetcher_->StartOrQueueNetworkRequest(
-      snippet.id(), snippet.salient_image_url(), callback);
-  // TODO(treib): Cache/persist the snippet image.
+  database_->LoadImage(
+      snippet_id,
+      base::Bind(&NTPSnippetsService::OnSnippetImageFetchedFromDatabase,
+                 base::Unretained(this), snippet_id, callback));
 }
 
 void NTPSnippetsService::ClearSnippets() {
   if (!initialized())
     return;
 
   if (snippets_.empty())
     return;
 
-  database_->Delete(snippets_);
+  database_->DeleteSnippets(snippets_);
   snippets_.clear();
 
   FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_,
                     NTPSnippetsServiceLoaded());
 }
 
 std::set<std::string> NTPSnippetsService::GetSuggestionsHosts() const {
   // |suggestions_service_| can be null in tests.
   if (!suggestions_service_)
     return std::set<std::string>();
@@ skipping 10 matching lines @@
   auto it =
       std::find_if(snippets_.begin(), snippets_.end(),
                    [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
                      return snippet->id() == snippet_id;
                    });
   if (it == snippets_.end())
     return false;
 
   (*it)->set_discarded(true);
 
-  database_->Save(**it);
+  database_->SaveSnippet(**it);
+  database_->DeleteImage((*it)->id());
 
   discarded_snippets_.push_back(std::move(*it));
   snippets_.erase(it);
 
   FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_,
                     NTPSnippetsServiceLoaded());
   return true;
 }
 
 void NTPSnippetsService::ClearDiscardedSnippets() {
   if (!initialized())
     return;
 
-  database_->Delete(discarded_snippets_);
+  database_->DeleteSnippets(discarded_snippets_);
   discarded_snippets_.clear();
 }
 
 void NTPSnippetsService::AddObserver(NTPSnippetsServiceObserver* observer) {
   observers_.AddObserver(observer);
 }
 
 void NTPSnippetsService::RemoveObserver(NTPSnippetsServiceObserver* observer) {
   observers_.RemoveObserver(observer);
 }
@@ skipping 35 matching lines @@
 // Private methods
 
 void NTPSnippetsService::OnStateChanged() {
   if (state_ == State::SHUT_DOWN)
     return;
 
   DVLOG(1) << "[OnStateChanged]";
   EnterState(GetStateForDependenciesStatus());
 }
 
+// image_fetcher::ImageFetcherDelegate implementation.
+void NTPSnippetsService::OnImageDataFetched(const std::string& snippet_id,
+                                            const std::string& image_data) {
+  if (image_data.empty())
+    return;
+
+  // Only save the image if the corresponding snippet still exists.
+  auto it =
+      std::find_if(snippets_.begin(), snippets_.end(),
+                   [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
+                     return snippet->id() == snippet_id;
+                   });
+  if (it == snippets_.end())
+    return;
+
+  database_->SaveImage(snippet_id, image_data);
+}
+
 void NTPSnippetsService::OnDatabaseLoaded(NTPSnippet::PtrVector snippets) {
   DCHECK(state_ == State::NOT_INITED || state_ == State::SHUT_DOWN);
   if (state_ == State::SHUT_DOWN)
     return;
 
   DCHECK(snippets_.empty());
   DCHECK(discarded_snippets_.empty());
   for (std::unique_ptr<NTPSnippet>& snippet : snippets) {
     if (snippet->is_discarded())
       discarded_snippets_.emplace_back(std::move(snippet));
@@ skipping 22 matching lines @@
   // TODO(treib,maybelle): If there is another source with an allowed host,
   // then we should fall back to that.
   // First, move them over into |to_delete|.
   NTPSnippet::PtrVector to_delete;
   for (std::unique_ptr<NTPSnippet>& snippet : snippets_) {
     if (!hosts.count(snippet->best_source().url.host()))
       to_delete.emplace_back(std::move(snippet));
   }
   Compact(&snippets_);
   // Then delete the removed snippets from the database.
-  database_->Delete(to_delete);
+  database_->DeleteSnippets(to_delete);
 
   StoreSnippetHostsToPrefs(hosts);
 
   FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_,
                     NTPSnippetsServiceLoaded());
 
   FetchSnippetsFromHosts(hosts);
 }
 
 void NTPSnippetsService::OnFetchFinished(
     NTPSnippetsFetcher::OptionalSnippets snippets) {
   if (!ready())
     return;
 
   if (snippets) {
     // Sparse histogram used because the number of snippets is small (bound by
     // kMaxSnippetCount).
     DCHECK_LE(snippets->size(), static_cast<size_t>(kMaxSnippetCount));
     UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticlesFetched",
                                 snippets->size());
     MergeSnippets(std::move(*snippets));
   }
 
   ClearExpiredSnippets();
 
-  // If there are still more snippets than we want to show, move the extra ones
-  // over into |to_delete|.
-  NTPSnippet::PtrVector to_delete;
+  // If there are more snippets than we want to show, delete the extra ones.
   if (snippets_.size() > kMaxSnippetCount) {
-    to_delete.insert(
-        to_delete.end(),
+    NTPSnippet::PtrVector to_delete(
         std::make_move_iterator(snippets_.begin() + kMaxSnippetCount),
-                     std::make_move_iterator(snippets_.end()));
+        std::make_move_iterator(snippets_.end()));
     snippets_.resize(kMaxSnippetCount);
+    database_->DeleteSnippets(to_delete);
   }
-  database_->Delete(to_delete);
 
   UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumArticles",
                               snippets_.size());
   if (snippets_.empty() && !discarded_snippets_.empty()) {
     UMA_HISTOGRAM_COUNTS("NewTabPage.Snippets.NumArticlesZeroDueToDiscarded",
                          discarded_snippets_.size());
   }
 
   FOR_EACH_OBSERVER(NTPSnippetsServiceObserver, observers_,
                     NTPSnippetsServiceLoaded());
@@ skipping 47 matching lines @@
     int num_snippets_discarded = num_new_snippets - new_snippets.size();
     UMA_HISTOGRAM_BOOLEAN("NewTabPage.Snippets.IncompleteSnippetsAfterFetch",
                           num_snippets_discarded > 0);
     if (num_snippets_discarded > 0) {
       UMA_HISTOGRAM_SPARSE_SLOWLY("NewTabPage.Snippets.NumIncompleteSnippets",
                                   num_snippets_discarded);
     }
   }
 
   // Save the new snippets to the DB.
-  database_->Save(new_snippets);
+  database_->SaveSnippets(new_snippets);
 
   // Insert the new snippets at the front.
   snippets_.insert(snippets_.begin(),
                    std::make_move_iterator(new_snippets.begin()),
                    std::make_move_iterator(new_snippets.end()));
 }
 
 std::set<std::string> NTPSnippetsService::GetSnippetHostsFromPrefs() const {
   std::set<std::string> hosts;
   const base::ListValue* list = pref_service_->GetList(prefs::kSnippetHosts);
@@ skipping 26 matching lines @@
   Compact(&snippets_);
 
   // Move expired discarded snippets over into |to_delete| as well.
   for (std::unique_ptr<NTPSnippet>& snippet : discarded_snippets_) {
     if (snippet->expiry_date() <= expiry)
       to_delete.emplace_back(std::move(snippet));
   }
   Compact(&discarded_snippets_);
 
   // Finally, actually delete the removed snippets from the DB.
-  database_->Delete(to_delete);
+  database_->DeleteSnippets(to_delete);
 
   // If there are any snippets left, schedule a timer for the next expiry.
   if (snippets_.empty() && discarded_snippets_.empty())
     return;
 
   base::Time next_expiry = base::Time::Max();
   for (const auto& snippet : snippets_) {
     if (snippet->expiry_date() < next_expiry)
       next_expiry = snippet->expiry_date();
   }
   for (const auto& snippet : discarded_snippets_) {
     if (snippet->expiry_date() < next_expiry)
       next_expiry = snippet->expiry_date();
   }
   DCHECK_GT(next_expiry, expiry);
   expiry_timer_.Start(FROM_HERE, next_expiry - expiry,
                       base::Bind(&NTPSnippetsService::ClearExpiredSnippets,
                                  base::Unretained(this)));
 }
 
+void NTPSnippetsService::OnSnippetImageFetchedFromDatabase(
+    const std::string& snippet_id,
+    const ImageFetchedCallback& callback,
+    std::string data) {
+  // |image_decoder_| is null on iOS and in tests.
+  if (image_decoder_ && !data.empty()) {
+    image_decoder_->DecodeImage(
+        std::move(data),
+        base::Bind(&NTPSnippetsService::OnSnippetImageDecoded,
+                   base::Unretained(this), snippet_id, callback));
+    return;
+  }
+
+  // Fetching from the DB failed; start a network fetch.
+  FetchSnippetImageFromNetwork(snippet_id, callback);
+}
+
+void NTPSnippetsService::OnSnippetImageDecoded(
+    const std::string& snippet_id,
+    const ImageFetchedCallback& callback,
+    const gfx::Image& image) {
+  if (!image.IsEmpty()) {
+    callback.Run(snippet_id, image);
+    return;
+  }
+
+  // If decoding the image failed, delete the DB entry.
+  database_->DeleteImage(snippet_id);
+
+  FetchSnippetImageFromNetwork(snippet_id, callback);
+}
+
+void NTPSnippetsService::FetchSnippetImageFromNetwork(
+    const std::string& snippet_id,
+    const ImageFetchedCallback& callback) {
+  auto it =
+      std::find_if(snippets_.begin(), snippets_.end(),
+                   [&snippet_id](const std::unique_ptr<NTPSnippet>& snippet) {
+                     return snippet->id() == snippet_id;
+                   });
+  if (it == snippets_.end()) {
+    callback.Run(snippet_id, gfx::Image());
+    return;
+  }
+
+  const NTPSnippet& snippet = *it->get();
+  image_fetcher_->StartOrQueueNetworkRequest(
+      snippet.id(), snippet.salient_image_url(), callback);
+}
+
 void NTPSnippetsService::EnterStateEnabled(bool fetch_snippets) {
   if (fetch_snippets)
     FetchSnippets();
 
   // If host restrictions are enabled, register for host list updates.
   // |suggestions_service_| can be null in tests.
   if (snippets_fetcher_->UsesHostRestrictions() && suggestions_service_) {
     suggestions_service_subscription_ =
         suggestions_service_->AddCallback(base::Bind(
             &NTPSnippetsService::OnSuggestionsChanged, base::Unretained(this)));
@@ skipping 22 matching lines @@
   suggestions_service_subscription_.reset();
 
   if (sync_service_)
     sync_service_observer_.Remove(sync_service_);
 }
 
 void NTPSnippetsService::FinishInitialization() {
   snippets_fetcher_->SetCallback(
       base::Bind(&NTPSnippetsService::OnFetchFinished, base::Unretained(this)));
 
+  // |image_fetcher_| can be null in tests.
+  if (image_fetcher_)
+    image_fetcher_->SetImageFetcherDelegate(this);
+
   // |sync_service_| can be null in tests or if sync is disabled.
   // This is a service we want to keep listening to all the time, independently
   // from the state, since it will allow us to enable or disable the snippets
   // service.
   if (sync_service_)
     sync_service_observer_.Add(sync_service_);
 
   // Change state after we started loading the snippets. During startup, the
   // Sync service might not be completely loaded when we initialize this
   // service, so we might stay in the NOT_INITED state until the sync state is
@@ skipping 72 matching lines @@
       return;
   }
 }
 
 void NTPSnippetsService::ClearDeprecatedPrefs() {
   pref_service_->ClearPref(prefs::kDeprecatedSnippets);
   pref_service_->ClearPref(prefs::kDeprecatedDiscardedSnippets);
 }
 
 }  // namespace ntp_snippets