[Background fetching] Background fetching when opening an NTP.

components/ntp_snippets/remote/scheduling_remote_suggestions_provider.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/ntp_snippets/remote/scheduling_remote_suggestions_provider.h"
 
+#include <random>
 #include <string>
 #include <utility>
 
 #include "base/memory/ptr_util.h"
 #include "components/ntp_snippets/features.h"
 #include "components/ntp_snippets/pref_names.h"
 #include "components/ntp_snippets/remote/persistent_scheduler.h"
 #include "components/ntp_snippets/status.h"
 #include "components/ntp_snippets/user_classifier.h"
 #include "components/prefs/pref_registry_simple.h"
 #include "components/prefs/pref_service.h"
 #include "components/variations/variations_associated_data.h"
 
 namespace ntp_snippets {
 
 namespace {
 
+enum class FetchingInterval {
+  BACKGROUND_FALLBACK,
+  BACKGROUND_WIFI,
+  SOFT_ACTIVE,

[tschumann, 2017/01/03 14:35:04]

hmm... what about: 'SOFT_ON_USAGE_EVENT' or just 'ON_USAGE_EVENT'?
We already have a notion of 'active' (--> active users) so overloading this
might be confusing.

[jkrcal, 2017/01/03 18:36:04]

Done.

+  MAX
+};
+
 // Default values for fetching intervals, fallback and wifi.
-const double kDefaultFetchingIntervalRareNtpUser[] = {48.0, 24.0};
-const double kDefaultFetchingIntervalActiveNtpUser[] = {24.0, 6.0};
-const double kDefaultFetchingIntervalActiveSuggestionsConsumer[] = {24.0, 6.0};
+const double kDefaultFetchingIntervalRareNtpUser[] = {48.0, 24.0, 12.0};
+const double kDefaultFetchingIntervalActiveNtpUser[] = {24.0, 6.0, 2.0};
+const double kDefaultFetchingIntervalActiveSuggestionsConsumer[] = {24.0, 6.0,
+                                                                    2.0};
 
 // Variation parameters than can the default fetching intervals.
 const char* kFetchingIntervalParamNameRareNtpUser[] = {
     "fetching_interval_hours-fallback-rare_ntp_user",
-    "fetching_interval_hours-wifi-rare_ntp_user"};
+    "fetching_interval_hours-wifi-rare_ntp_user",
+    "soft_fetching_interval_hours-active-rare_ntp_user"};
 const char* kFetchingIntervalParamNameActiveNtpUser[] = {
     "fetching_interval_hours-fallback-active_ntp_user",
-    "fetching_interval_hours-wifi-active_ntp_user"};
+    "fetching_interval_hours-wifi-active_ntp_user",
+    "soft_fetching_interval_hours-active-active_ntp_user"};
 const char* kFetchingIntervalParamNameActiveSuggestionsConsumer[] = {
     "fetching_interval_hours-fallback-active_suggestions_consumer",
-    "fetching_interval_hours-wifi-active_suggestions_consumer"};
+    "fetching_interval_hours-wifi-active_suggestions_consumer",
+    "soft_fetching_interval_hours-active-active_suggestions_consumer"};
 
-base::TimeDelta GetDesiredUpdateInterval(
-    bool is_wifi,
+// Default value for the maximum value of the random jitter that is added to the
+// soft fetching interval (the minimum value is 0.0).
+const double kDefaultSoftFetchingJitterInHours = 0.5;
+const char* kSoftFetchingJitterParamName =
+    "soft_fetching_interval_jitter_hours";
+
+base::TimeDelta GetDesiredFetchingInterval(
+    FetchingInterval interval,
     UserClassifier::UserClass user_class) {
   double default_value_hours = 0.0;
 
-  const int index = is_wifi ? 1 : 0;
+  const int index = static_cast<int>(interval);
+  DCHECK(index >=0 && index < static_cast<int>(FetchingInterval::MAX));

[tschumann, 2017/01/03 14:35:04]

hmm... i guess what you really want to check is:
DCHECK(interval != FetchingInterval::MAX);
and
DCHECK(index >=0 && index < arraysize(kDefaultFetchingIntervalRareNtpUser));

(and if you really care, compare against the other arrays as well).

[jkrcal, 2017/01/03 18:36:04]

Done.

+
   const char* param_name = nullptr;
   switch (user_class) {
     case UserClassifier::UserClass::RARE_NTP_USER:
       default_value_hours = kDefaultFetchingIntervalRareNtpUser[index];
       param_name = kFetchingIntervalParamNameRareNtpUser[index];
       break;
     case UserClassifier::UserClass::ACTIVE_NTP_USER:
       default_value_hours = kDefaultFetchingIntervalActiveNtpUser[index];
       param_name = kFetchingIntervalParamNameActiveNtpUser[index];
       break;
     case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER:
       default_value_hours =
           kDefaultFetchingIntervalActiveSuggestionsConsumer[index];
       param_name = kFetchingIntervalParamNameActiveSuggestionsConsumer[index];
       break;
   }
 
   double value_hours = variations::GetVariationParamByFeatureAsDouble(
       ntp_snippets::kArticleSuggestionsFeature, param_name,
       default_value_hours);
 
   return base::TimeDelta::FromSecondsD(value_hours * 3600.0);
 }
 
+base::TimeDelta GetDesiredFetchingJitter() {
+  double value_hours = variations::GetVariationParamByFeatureAsDouble(
+      ntp_snippets::kArticleSuggestionsFeature,
+      kSoftFetchingJitterParamName, kDefaultSoftFetchingJitterInHours);
+
+  return base::TimeDelta::FromSecondsD(value_hours * 3600.0);
+}
+
 }  // namespace
 
 struct SchedulingRemoteSuggestionsProvider::FetchingSchedule {
   base::TimeDelta interval_wifi;
   base::TimeDelta interval_fallback;
+  base::TimeDelta soft_interval_active;
 
   static FetchingSchedule Empty() {
-    return FetchingSchedule{base::TimeDelta(),
+    return FetchingSchedule{base::TimeDelta(), base::TimeDelta(),
                             base::TimeDelta()};
   }
 
   bool operator==(const FetchingSchedule& other) const {
     return interval_wifi == other.interval_wifi &&
-           interval_fallback == other.interval_fallback;
+           interval_fallback == other.interval_fallback &&
+           soft_interval_active == other.soft_interval_active;
   }
 
   bool operator!=(const FetchingSchedule& other) const {
     return !operator==(other);
   }
 
   bool is_empty() const {
-    return interval_wifi.is_zero() && interval_fallback.is_zero();
+    return interval_wifi.is_zero() && interval_fallback.is_zero() &&
+           soft_interval_active.is_zero();
   }
 };
 
 SchedulingRemoteSuggestionsProvider::SchedulingRemoteSuggestionsProvider(
     Observer* observer,
     std::unique_ptr<RemoteSuggestionsProvider> provider,
     PersistentScheduler* persistent_scheduler,
     const UserClassifier* user_classifier,
     PrefService* pref_service)
     : RemoteSuggestionsProvider(observer),
@@ skipping 23 matching lines @@
                               0);
 }
 
 void SchedulingRemoteSuggestionsProvider::RescheduleFetching() {
   // Force the reschedule by stopping and starting it again.
   StopScheduling();
   StartScheduling();
 }
 
 void SchedulingRemoteSuggestionsProvider::OnPersistentSchedulerWakeUp() {
-  provider_->RefetchInTheBackground(
-      base::MakeUnique<RemoteSuggestionsProvider::FetchStatusCallback>(
-          base::Bind(&SchedulingRemoteSuggestionsProvider::OnFetchCompleted,
-                     base::Unretained(this))));
+  RefetchInTheBackground(nullptr);

[tschumann, 2017/01/03 15:16:13]

please add /*callback=*/nullptr
to document the parameter.
Same in other places below.

[jkrcal, 2017/01/03 18:36:04]

Done.

 }
 
 void SchedulingRemoteSuggestionsProvider::OnBrowserStartup() {
   // TODO(jkrcal): Implement.
 }
 
 void SchedulingRemoteSuggestionsProvider::OnNTPOpened() {
-  // TODO(jkrcal): Implement.
+  if (!ShouldRefetchInTheBackgroundNow()) {
+    return;
+  }
+  RefetchInTheBackground(nullptr);
 }
 
 void SchedulingRemoteSuggestionsProvider::SetProviderStatusCallback(
       std::unique_ptr<ProviderStatusCallback> callback) {
   provider_->SetProviderStatusCallback(std::move(callback));
 }
 
 void SchedulingRemoteSuggestionsProvider::RefetchInTheBackground(
     std::unique_ptr<FetchStatusCallback> callback) {
-  provider_->RefetchInTheBackground(std::move(callback));

[jkrcal, 2017/01/03 14:02:09]

This was a previous bug that OnFetchCompleted() (now
RefetchInTheBackgroundFinished()) was not called after this call.

+  if (background_fetch_in_progress_) {
+    if (callback) {
+      callback->Run(
+          Status(StatusCode::TEMPORARY_ERROR, "Background fetch in progress"));
+    }
+    return;
+  }
+
+  RemoteSuggestionsProvider::FetchStatusCallback wrapper_callback = base::Bind(
+          &SchedulingRemoteSuggestionsProvider::RefetchInTheBackgroundFinished,
+          base::Unretained(this), base::Passed(&callback));
+  provider_->RefetchInTheBackground(
+      base::MakeUnique<RemoteSuggestionsProvider::FetchStatusCallback>(
+          std::move(wrapper_callback)));
 }
 
 const NTPSnippetsFetcher* SchedulingRemoteSuggestionsProvider::
     snippets_fetcher_for_testing_and_debugging() const {
   return provider_->snippets_fetcher_for_testing_and_debugging();
 }
 
 CategoryStatus SchedulingRemoteSuggestionsProvider::GetCategoryStatus(
     Category category) {
   return provider_->GetCategoryStatus(category);
@@ skipping 63 matching lines @@
       StartScheduling();
       return;
     case RemoteSuggestionsProvider::ProviderStatus::INACTIVE:
       StopScheduling();
       return;
   }
   NOTREACHED();
 }
 
 void SchedulingRemoteSuggestionsProvider::StartScheduling() {
-  // The scheduler only exists on Android so far, it's null on other platforms.
-  if (!persistent_scheduler_) {
-    return;
-  }
-
   FetchingSchedule last_schedule = GetLastFetchingSchedule();
   FetchingSchedule schedule = GetDesiredFetchingSchedule();
 
   // Reset the schedule only if the parameters have changed.
   if (last_schedule != schedule) {
-    ApplyFetchingSchedule(schedule);
+    ApplyFetchingSchedule(schedule, /*also_apply_persistent_schedule=*/true);
   }
 }
 
 void SchedulingRemoteSuggestionsProvider::StopScheduling() {
-  // The scheduler only exists on Android so far, it's null on other platforms.
-  if (!persistent_scheduler_) {
-    return;
-  }
-
   // Do not unschedule if already switched off
   FetchingSchedule last_schedule = GetLastFetchingSchedule();
   if (last_schedule.is_empty()) {
     return;
   }
 
-  persistent_scheduler_->Unschedule();
+  pref_service_->ClearPref(prefs::kSnippetNextActiveSoftFetch);
+
+  // The scheduler only exists on Android so far, it's null on other platforms.
+  if (persistent_scheduler_) {
+    persistent_scheduler_->Unschedule();
+  }
 
   StoreLastFetchingSchedule(FetchingSchedule::Empty());
 }
 
 void SchedulingRemoteSuggestionsProvider::ApplyFetchingSchedule(
-    const FetchingSchedule& schedule) {
-  persistent_scheduler_->Schedule(schedule.interval_wifi,
-                                  schedule.interval_fallback);
+    const FetchingSchedule& schedule,
+    bool also_apply_persistent_schedule) {
+
+  std::default_random_engine generator;
+  std::uniform_real_distribution<double> distribution(0.0,1.0);
+  // generates a random double in the range 0.0 .. 1.0
+  double random_factor = distribution(generator);
+
+  base::Time next_active_soft_fetch =
+      base::Time::Now() + schedule.soft_interval_active +
+      (GetDesiredFetchingJitter() * random_factor);
+  pref_service_->SetInt64(prefs::kSnippetNextActiveSoftFetch,
+                          next_active_soft_fetch.ToInternalValue());
+
+  // The scheduler only exists on Android so far, it's null on other platforms.
+  if (persistent_scheduler_ && also_apply_persistent_schedule) {
+    persistent_scheduler_->Schedule(schedule.interval_wifi,
+                                    schedule.interval_fallback);
+  }
 
   StoreLastFetchingSchedule(schedule);
 }
 
 SchedulingRemoteSuggestionsProvider::FetchingSchedule
 SchedulingRemoteSuggestionsProvider::GetDesiredFetchingSchedule() const {
   UserClassifier::UserClass user_class = user_classifier_->GetUserClass();
 
   FetchingSchedule schedule;
   schedule.interval_wifi =
-      GetDesiredUpdateInterval(/*is_wifi=*/true, user_class);
-  schedule.interval_fallback =
-      GetDesiredUpdateInterval(/*is_wifi=*/false, user_class);
+      GetDesiredFetchingInterval(FetchingInterval::BACKGROUND_WIFI, user_class);
+  schedule.interval_fallback = GetDesiredFetchingInterval(
+      FetchingInterval::BACKGROUND_FALLBACK, user_class);
+  schedule.soft_interval_active =
+      GetDesiredFetchingInterval(FetchingInterval::SOFT_ACTIVE, user_class);
+
   return schedule;
 }
 
 SchedulingRemoteSuggestionsProvider::FetchingSchedule
 SchedulingRemoteSuggestionsProvider::GetLastFetchingSchedule() const {
   FetchingSchedule schedule;
   schedule.interval_wifi = base::TimeDelta::FromInternalValue(
       pref_service_->GetInt64(prefs::kSnippetBackgroundFetchingIntervalWifi));
   schedule.interval_fallback =
       base::TimeDelta::FromInternalValue(pref_service_->GetInt64(
           prefs::kSnippetBackgroundFetchingIntervalFallback));
+  schedule.soft_interval_active = base::TimeDelta::FromInternalValue(
+      pref_service_->GetInt64(prefs::kSnippetSoftFetchingIntervalActive));
+
   return schedule;
 }
 
 void SchedulingRemoteSuggestionsProvider::StoreLastFetchingSchedule(
     const FetchingSchedule& schedule) {
   pref_service_->SetInt64(
       prefs::kSnippetBackgroundFetchingIntervalWifi,
       schedule.interval_wifi.ToInternalValue());
   pref_service_->SetInt64(
       prefs::kSnippetBackgroundFetchingIntervalFallback,
       schedule.interval_fallback.ToInternalValue());
+  pref_service_->SetInt64(
+      prefs::kSnippetSoftFetchingIntervalActive,
+      schedule.soft_interval_active.ToInternalValue());
+}
+
+bool SchedulingRemoteSuggestionsProvider::ShouldRefetchInTheBackgroundNow() {
+  if (!pref_service_->HasPrefPath(prefs::kSnippetNextActiveSoftFetch)) {

[tschumann, 2017/01/03 15:16:13]

i guess this checks whether the path has been registered. Do we do this
somewhere?

[jkrcal, 2017/01/03 18:36:04]

No, a path should always be registered (I've forgotten to do so, thanks for
pointing out). Here, I check that something has been stored there. I use it as a
signal that background fetching is switched on / off.

[tschumann, 2017/01/03 18:54:24]

Ah, got it -- that's what ClearPref() is doing :-) It might be worth adding a
comment to line 299 explaining that this is not only cleaning up but also
signalling that soft fetches are stopped.

[jkrcal, 2017/01/04 10:06:30]

I fixed it, in the end.

+    // Soft background fetches are switched off.
+    return false;
+  }
+
+  base::Time next_active_soft_fetch = base::Time::FromInternalValue(
+      pref_service_->GetInt64(prefs::kSnippetNextActiveSoftFetch));
+  return next_active_soft_fetch <= base::Time::Now();
 }
 
 void SchedulingRemoteSuggestionsProvider::FetchFinished(
     const FetchDoneCallback& callback,
-    Status status_code,
+    Status fetch_status,
     std::vector<ContentSuggestion> suggestions) {
-  OnFetchCompleted(status_code);
-  callback.Run(status_code, std::move(suggestions));
+  // Reschedule after a fetch. The persistent schedule is applied only after a
+  // successful fetch. After a failed fetch, we want to keep the previous
+  // persistent schedule intact so that we eventually get a persistent fallback
+  // fetch (if the wifi persistent fetches keep failing).
+  ApplyFetchingSchedule(GetLastFetchingSchedule(),
+                        /*also_apply_persistent_schedule=*/fetch_status.code ==
+                            StatusCode::SUCCESS);
+
+  callback.Run(fetch_status, std::move(suggestions));
 }
 
-void SchedulingRemoteSuggestionsProvider::OnFetchCompleted(
+void SchedulingRemoteSuggestionsProvider::RefetchInTheBackgroundFinished(
+    std::unique_ptr<FetchStatusCallback> callback,
     Status fetch_status) {
-  // The scheduler only exists on Android so far, it's null on other platforms.
-  if (!persistent_scheduler_) {
-    return;
+  background_fetch_in_progress_ = false;
+
+  // Reschedule after a fetch. The persistent schedule is applied only after a
+  // successful fetch. After a failed fetch, we want to keep the previous
+  // persistent schedule intact so that we eventually get a persistent fallback
+  // fetch (if the wifi persistent fetches keep failing).
+  ApplyFetchingSchedule(GetLastFetchingSchedule(),
+                        /*also_apply_persistent_schedule=*/fetch_status.code ==
+                            StatusCode::SUCCESS);
+
+  if (callback) {
+    callback->Run(fetch_status);
   }
-
-  if (fetch_status.code != StatusCode::SUCCESS) {
-    return;
-  }
-
-  // Reschedule after a successful fetch. This resets all currently scheduled
-  // fetches, to make sure the fallback interval triggers only if no wifi fetch
-  // succeeded, and also that we don't do a background fetch immediately after
-  // a user-initiated one.
-  ApplyFetchingSchedule(GetLastFetchingSchedule());
 }
 
 }  // namespace ntp_snippets