Add a new app API to enable watchdog behavior restarts in kiosk apps

extensions/browser/api/runtime/runtime_api.cc

Index: extensions/browser/api/runtime/runtime_api.cc
diff --git a/extensions/browser/api/runtime/runtime_api.cc b/extensions/browser/api/runtime/runtime_api.cc
index 79e7c97e48c2a502739687c5e90c07e38c54e166..6203f7fd89ee683c50c85ff39525fc8eefb4bcb2 100644
--- a/extensions/browser/api/runtime/runtime_api.cc
+++ b/extensions/browser/api/runtime/runtime_api.cc
@@ -13,9 +13,12 @@
 #include "base/memory/ptr_util.h"
 #include "base/metrics/histogram.h"
 #include "base/single_thread_task_runner.h"
+#include "base/strings/string_number_conversions.h"
 #include "base/threading/thread_task_runner_handle.h"
 #include "base/values.h"
 #include "base/version.h"
+#include "components/prefs/pref_registry_simple.h"
+#include "components/prefs/pref_service.h"
 #include "content/public/browser/browser_context.h"
 #include "content/public/browser/child_process_security_policy.h"
 #include "content/public/browser/notification_service.h"
@@ -80,6 +83,27 @@ const char kPrefPreviousVersion[] = "previous_version";
 // with the equivalent Pepper API.
 const char kPackageDirectoryPath[] = "crxfs";
 
+// Preference key for storing the last successful restart on the watchdog timer
+// timing out.
+constexpr char kPrefLastRestartAfterDelayTime[] =
+    "last_restart_on_watchdog_time";
+
+// Error and status messages strings for the restartAfterDelay() API.
+constexpr char kErrorInvalidArgument[] = "Invalid argument: *.";
+constexpr char kErrorOnlyKioskModeAllowed[] =
+    "API available only for ChromeOS kiosk mode.";
+constexpr char kErrorOnlyFirstExtensionAllowed[] =
+    "Not the first extension to call this API.";
+constexpr char kErrorInvalidStatus[] = "Invalid restart request status.";
+constexpr char kErrorRequestedTooSoon[] =
+    "Restart was requested too soon. It was throttled instead.";
+
+constexpr int kMinDurationBetweenSuccessiveRestartsHours = 3;
+
+// This is used for browsertests, so that we can test the restartAfterDelay
+// API without a kiost app.
+bool allow_non_kiost_apps_restart_api_for_test = false;
+
 void DispatchOnStartupEventImpl(BrowserContext* browser_context,
                                 const std::string& extension_id,
                                 bool first_call,
@@ -151,6 +175,11 @@ BrowserContextKeyedAPIFactory<RuntimeAPI>* RuntimeAPI::GetFactoryInstance() {
   return g_factory.Pointer();
 }
 
+// static
+void RuntimeAPI::RegisterPrefs(PrefRegistrySimple* registry) {
+  registry->RegisterDoublePref(kPrefLastRestartAfterDelayTime, 0.0);
+}
+
 template <>
 void BrowserContextKeyedAPIFactory<RuntimeAPI>::DeclareFactoryDependencies() {
   DependsOn(ProcessManagerFactory::GetInstance());
@@ -158,9 +187,11 @@ void BrowserContextKeyedAPIFactory<RuntimeAPI>::DeclareFactoryDependencies() {
 
 RuntimeAPI::RuntimeAPI(content::BrowserContext* context)
     : browser_context_(context),
-      dispatch_chrome_updated_event_(false),
       extension_registry_observer_(this),
-      process_manager_observer_(this) {
+      process_manager_observer_(this),
+      minimum_duration_between_restarts_(base::TimeDelta::FromHours(
+          kMinDurationBetweenSuccessiveRestartsHours)),
+      weak_ptr_factory_(this) {
   // RuntimeAPI is redirected in incognito, so |browser_context_| is never
   // incognito.
   DCHECK(!browser_context_->IsOffTheRecord());
@@ -174,6 +205,9 @@ RuntimeAPI::RuntimeAPI(content::BrowserContext* context)
   delegate_ = ExtensionsBrowserClient::Get()->CreateRuntimeAPIDelegate(
       browser_context_);
 
+  delegate_->SetOnDeviceShutdownCallback(base::Bind(
+      &RuntimeAPI::OnDeviceShutdown, weak_ptr_factory_.GetWeakPtr()));
+
   // Check if registered events are up-to-date. We can only do this once
   // per browser context, since it updates internal state when called.
   dispatch_chrome_updated_event_ =
@@ -319,13 +353,124 @@ bool RuntimeAPI::GetPlatformInfo(runtime::PlatformInfo* info) {
 }
 
 bool RuntimeAPI::RestartDevice(std::string* error_message) {
-  return delegate_->RestartDevice(error_message);
+  expecting_non_watchdog_restart_ = delegate_->RestartDevice(error_message);
+  return expecting_non_watchdog_restart_;
+}
+
+RuntimeAPI::RestartOnWatchdogStatus RuntimeAPI::RestartDeviceOnWatchdogTimeout(
+    const std::string& extension_id,
+    int seconds_from_now) {
+  // To achieve as much accuracy as possible, record the time of the call as
+  // |now| here.
+  const base::Time now = base::Time::NowFromSystemTime();
+
+  if (schedule_restart_first_extension_id_.empty()) {
+    schedule_restart_first_extension_id_ = extension_id;
+  } else if (extension_id != schedule_restart_first_extension_id_) {
+    // We only allow the first extension to call this API to call it repeatedly.
+    // Any other extension will fail.
+    return RestartOnWatchdogStatus::FAILED_NOT_FIRST_EXTENSION;
+  }
+
+  MaybeCancelRunningWatchdogTimer();
+
+  if (seconds_from_now == -1) {
+    // We already stopped the running timer (if any).
+    return RestartOnWatchdogStatus::SUCCESS_RESTART_CANCELED;
+  }
+
+  // Try to read any previous successful restart attempt time resulting from
+  // this API.
+  PrefService* pref_service =
+      ExtensionsBrowserClient::Get()->GetPrefServiceForContext(
+          browser_context_);
+  DCHECK(pref_service);
+  double last_restart_time =
+      pref_service->GetDouble(kPrefLastRestartAfterDelayTime);
+
+  return ScheduleDelayedRestart(now, seconds_from_now, last_restart_time);
 }
 
 bool RuntimeAPI::OpenOptionsPage(const Extension* extension) {
   return delegate_->OpenOptionsPage(extension);
 }
 
+void RuntimeAPI::MaybeCancelRunningWatchdogTimer() {
+  if (watchdog_timer_.IsRunning())
+    watchdog_timer_.Stop();
+}
+
+RuntimeAPI::RestartOnWatchdogStatus RuntimeAPI::ScheduleDelayedRestart(
+    const base::Time& now,
+    int seconds_from_now,
+    double stored_last_restart) {
+  base::TimeDelta delay_till_restart =
+      base::TimeDelta::FromSeconds(seconds_from_now);
+
+  // Throttle restart requests that are received too soon successively.
+  bool was_throttled = false;
+  base::Time last_restart_time = base::Time::FromDoubleT(stored_last_restart);
+  base::Time future_restart_time = now + delay_till_restart;
+  base::TimeDelta future_time_since_last_restart =
+      future_restart_time > last_restart_time
+          ? future_restart_time - last_restart_time
+          : base::TimeDelta();
+  if (future_time_since_last_restart < minimum_duration_between_restarts_) {
+    // Schedule the restart after |minimum_duration_between_restarts_| has
+    // passed.
+    delay_till_restart =
+        minimum_duration_between_restarts_ - (now - last_restart_time);
+    was_throttled = true;
+  }
+
+  watchdog_timer_.Start(FROM_HERE, delay_till_restart,
+                        base::Bind(&RuntimeAPI::OnRestartWatchdogTimeout,
+                                   weak_ptr_factory_.GetWeakPtr()));
+
+  return was_throttled ? RestartOnWatchdogStatus::FAILED_THROTTLED
+                       : RestartOnWatchdogStatus::SUCCESS_RESTART_SCHEDULED;
+}
+
+void RuntimeAPI::OnRestartWatchdogTimeout() {
+  // We can persist "now" as the last successful restart time, assuming that the
+  // following restart request will succeed, since it can only fail if requested
+  // by non kiosk apps, and we prevent that from the beginning (unless in
+  // browsertests).
+  // This assumption is important, since once restart is requested, we might not
+  // have enough time to persist the data to disk.
+  expecting_watchdog_restart_ = true;
+  std::string error_message;
+  const bool success = delegate_->RestartDevice(&error_message);
+
+  if (!success && !allow_non_kiost_apps_restart_api_for_test) {
+    // This is breaking our above assumption and should never be reached.
+    NOTREACHED();
+  }
+}
+
+void RuntimeAPI::OnDeviceShutdown() {

[Devlin, 2016/06/10 00:34:45]

This worries me for a couple of reasons:
- We typically want to steer clear of NOTIFICATIONs.
- This means we're blocking shutdown on persisting something to disk, which is
something we really like to avoid.

I wonder if there's a way we can get around this.  What if we did something
like:
In the delayed restart, if we succeed in scheduling it, assume it will succeed,
and persist both the last restart time and a bool that says "restarted due to
delayed restart".
In the ctor (or, maybe better yet, even lazily based on the first restart on
delay request) check the "restarted due to delayed" bit, and, if it's present,
we were restarted due to the API and we enforce the throttle.  Then clear the
bit, and any future calls (including across non-delayed restarts) will not have
the bit, and we won't throttle.

WDYT?

[xiyuan, 2016/06/10 16:33:48]

This would not work because the restartAfterDelay is supposed to be called
periodically by the app to implement a heart beat. Clearing the flag on first
call essentially makes the throttling ineffective.

I think the throttling is overly complicated to implement with the API. We
should probably take a step back. The purpose of the throttling is to
avoid/break reboot loop for bad apps. Instead of doing this in API, how about we
take another approach that we keep track of auto launch times and stop launching
the app if we do it too fast (say 5 times in last 15 minutes). This way, we
break the potential reboot loop. And we do it on device boot instead of during
shutdown. The API would be much simpler without the throttling logic.

[Devlin, 2016/06/10 18:06:17]

Sorry, I was unclear.  I meant we should have a member
did_restart_due_to_delayed_restart_ on the RuntimeAPI, which is set if the
preference bit is set (and then clear the preference bit, but the member stays
set for the lifetime of the RuntimeAPI, which is also the lifetime of the
profile).  I believe that would be sufficient, though, as I said, setting in the
RuntimeAPI ctor is also an option.
I'm fine with this, too, though I'm not as familiar with the code so don't know
how complex it will become.  I just don't want to slow down shutdown if we can
avoid it.  Anything that avoids that is fine with me.

[afakhry, 2016/06/13 14:46:00]

I didn't get this comment until I read your explanation on IM. Thanks for the
clarification over there.
Done.

+  if (expecting_non_watchdog_restart_) {
+    // Non-watchdog restarts should never clear the watchdog throttle.
+    return;
+  }
+
+  // Persist "now" as the current last watchdog restart time if we are expecting
+  // one, clear it otherwise.
+  double restart_time = 0.0;
+  if (expecting_watchdog_restart_)
+    restart_time = base::Time::NowFromSystemTime().ToDoubleT();
+
+  PrefService* pref_service =
+      ExtensionsBrowserClient::Get()->GetPrefServiceForContext(
+          browser_context_);
+  DCHECK(pref_service);
+  pref_service->SetDouble(kPrefLastRestartAfterDelayTime, restart_time);
+}
+
+void RuntimeAPI::AllowNonKiostAppsInRestartOnWatchdogForTesting() {
+  allow_non_kiost_apps_restart_api_for_test = true;
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 
 // static
@@ -553,6 +698,41 @@ ExtensionFunction::ResponseAction RuntimeRestartFunction::Run() {
   return RespondNow(NoArguments());
 }
 
+ExtensionFunction::ResponseAction RuntimeRestartAfterDelayFunction::Run() {
+  if (!allow_non_kiost_apps_restart_api_for_test &&
+      !ExtensionsBrowserClient::Get()->IsRunningInForcedAppMode()) {
+    return RespondNow(Error(kErrorOnlyKioskModeAllowed));
+  }
+
+  std::unique_ptr<api::runtime::RestartAfterDelay::Params> params(
+      api::runtime::RestartAfterDelay::Params::Create(*args_));
+  EXTENSION_FUNCTION_VALIDATE(params.get());
+  int seconds = params->seconds;
+
+  if (seconds < -1)
+    return RespondNow(Error(kErrorInvalidArgument, base::IntToString(seconds)));
+
+  RuntimeAPI::RestartOnWatchdogStatus request_status =
+      RuntimeAPI::GetFactoryInstance()
+          ->Get(browser_context())
+          ->RestartDeviceOnWatchdogTimeout(extension()->id(), seconds);
+
+  switch (request_status) {
+    case RuntimeAPI::RestartOnWatchdogStatus::FAILED_NOT_FIRST_EXTENSION:
+      return RespondNow(Error(kErrorOnlyFirstExtensionAllowed));
+
+    case RuntimeAPI::RestartOnWatchdogStatus::FAILED_THROTTLED:
+      return RespondNow(Error(kErrorRequestedTooSoon));
+
+    case RuntimeAPI::RestartOnWatchdogStatus::SUCCESS_RESTART_CANCELED:
+    case RuntimeAPI::RestartOnWatchdogStatus::SUCCESS_RESTART_SCHEDULED:
+      return RespondNow(NoArguments());
+  }
+
+  NOTREACHED();
+  return RespondNow(Error(kErrorInvalidStatus));
+}
+
 ExtensionFunction::ResponseAction RuntimeGetPlatformInfoFunction::Run() {
   runtime::PlatformInfo info;
   if (!RuntimeAPI::GetFactoryInstance()