Suspend (rather than shut down) the host upon policy errors.

remoting/host/remoting_me2me_host.cc

Index: remoting/host/remoting_me2me_host.cc
diff --git a/remoting/host/remoting_me2me_host.cc b/remoting/host/remoting_me2me_host.cc
index b66f8376f791bea52a9ff8cc57ce6339324331d2..8cf6ccee505728f99c5e5587acba15fe03f03f5e 100644
--- a/remoting/host/remoting_me2me_host.cc
+++ b/remoting/host/remoting_me2me_host.cc
@@ -147,6 +147,12 @@ const int kShutdownTimeoutSeconds = 15;
 // before continuing normal process shutdown.
 const int kHostOfflineReasonTimeoutSeconds = 10;
 
+// Host offline reasons not associated with shutting down the host process
+// and therefore not expressible through HostExitCodes enum.
+const char kHostOfflineReasonPolicyReadError[] = "POLICY_READ_ERROR";
+const char kHostOfflineReasonPolicyChangeRequiresRestart[] =
+    "POLICY_CHANGE_REQUIRES_RESTART";
+
 }  // namespace
 
 namespace remoting {
@@ -182,40 +188,46 @@ class HostProcess : public ConfigWatcher::Delegate,
       IPC::PlatformFileForTransit unprivileged_key);
 
  private:
+  // See SetState method for a list of allowed state transitions.
   enum HostState {
-    // Host process has just been started. Waiting for config and policies to be
-    // read from the disk.
-    HOST_INITIALIZING,
+    // Waiting for valid config and policies to be read from the disk.
+    // Either the host process has just been started, or it is trying to start
+    // again after temporarily going offline due to policy change or error.
+    HOST_STARTING,
 
     // Host is started and running.
     HOST_STARTED,
 
-    // Host is being stopped and will need to be started again.
-    HOST_STOPPING_TO_RESTART,
+    // Host is sending offline reason, before trying to restart.
+    HOST_GOING_OFFLINE_TO_RESTART,
 
-    // Host is being stopped.
-    HOST_STOPPING,
+    // Host is sending offline reason, before shutting down.
+    HOST_GOING_OFFLINE_TO_STOP,
 
-    // Host has been stopped.
+    // Host has been stopped (host process will end soon).
     HOST_STOPPED,
+  };
 
-    // Allowed state transitions:
-    //   INITIALIZING->STARTED
-    //   INITIALIZING->STOPPED
-    //   STARTED->STOPPING_TO_RESTART
-    //   STARTED->STOPPING
-    //   STOPPING_TO_RESTART->STARTED
-    //   STOPPING_TO_RESTART->STOPPING
-    //   STOPPING->STOPPED
-    //   STOPPED->STARTED
-    //
-    // |host_| must be nullptr in INITIALIZING and STOPPED states and not
-    // nullptr in all other states.
+  enum PolicyState {
+    // Cannot start the host, because a valid policy has not been read yet.
+    POLICY_INITIALIZING,
+
+    // Policy was loaded successfully.
+    POLICY_LOADED,
+
+    // Policy error was detected, and we haven't yet sent out a
+    // host-offline-reason (i.e. because we haven't yet read the config).
+    POLICY_ERROR_REPORT_PENDING,
+
+    // Policy error was detected, and we have sent out a host-offline-reason.
+    POLICY_ERROR_REPORTED,
   };
 
   friend class base::RefCountedThreadSafe<HostProcess>;
   ~HostProcess() override;
 
+  void SetState(HostState target_state);
+
   void StartOnNetworkThread();
 
 #if defined(OS_POSIX)
@@ -245,6 +257,7 @@ class HostProcess : public ConfigWatcher::Delegate,
   // Handles policy updates, by calling On*PolicyUpdate methods.
   void OnPolicyUpdate(scoped_ptr<base::DictionaryValue> policies);
   void OnPolicyError();
+  void ReportPolicyErrorAndRestartHost();
   void ApplyHostDomainPolicy();
   void ApplyUsernamePolicy();
   bool OnHostDomainPolicyUpdate(base::DictionaryValue* policies);
@@ -268,16 +281,12 @@ class HostProcess : public ConfigWatcher::Delegate,
   void OnUnknownHostIdError() override;
   void OnAuthFailed() override;
 
-  void RestartHost();
-
-  // Stops the host and shuts down the process with the specified |exit_code|.
+  void RestartHost(const std::string& host_offline_reason);
   void ShutdownHost(HostExitCodes exit_code);
 
-  // Private helper used by ShutdownHost method to initiate sending of
-  // host-offline-reason before continuing shutdown.
-  void SendOfflineReasonAndShutdownOnNetworkThread(HostExitCodes exit_code);
-
-  void ShutdownOnNetworkThread();
+  // Helper methods doing the work needed by RestartHost and ShutdownHost.
+  void GoOffline(const std::string& host_offline_reason);
+  void OnHostOfflineReasonAck(bool success);
 
 #if defined(OS_WIN)
   // Initializes the pairing registry on Windows. This should be invoked on the
@@ -325,7 +334,7 @@ class HostProcess : public ConfigWatcher::Delegate,
   int64_t frame_recorder_buffer_size_;
 
   scoped_ptr<PolicyWatcher> policy_watcher_;
-  bool policies_loaded_;
+  PolicyState policy_state_;
   std::string host_domain_;
   bool host_username_match_required_;
   bool allow_nat_traversal_;
@@ -377,11 +386,11 @@ HostProcess::HostProcess(scoped_ptr<ChromotingHostContext> context,
                          int* exit_code_out,
                          ShutdownWatchdog* shutdown_watchdog)
     : context_(context.Pass()),
-      state_(HOST_INITIALIZING),
+      state_(HOST_STARTING),
       use_service_account_(false),
       enable_vp9_(false),
       frame_recorder_buffer_size_(0),
-      policies_loaded_(false),
+      policy_state_(POLICY_INITIALIZING),
       host_username_match_required_(false),
       allow_nat_traversal_(true),
       allow_relay_(true),
@@ -551,12 +560,11 @@ void HostProcess::OnConfigUpdated(
     return;
   }
 
-  if (state_ == HOST_INITIALIZING) {
+  if (state_ == HOST_STARTING) {
     StartHostIfReady();
   } else if (state_ == HOST_STARTED) {
-    DCHECK(policies_loaded_);
-
     // Reapply policies that could be affected by a new config.
+    DCHECK_EQ(policy_state_, POLICY_LOADED);
     ApplyHostDomainPolicy();
     ApplyUsernamePolicy();
 
@@ -572,6 +580,51 @@ void HostProcess::OnConfigWatcherError() {
   ShutdownHost(kInvalidHostConfigurationExitCode);
 }
 
+// Allowed state transitions (enforced via DCHECKs in SetState method):
+//   STARTING->STARTED (once we have valid config + policy)
+//   STARTING->GOING_OFFLINE_TO_STOP
+//   STARTING->GOING_OFFLINE_TO_RESTART
+//   STARTED->GOING_OFFLINE_TO_STOP
+//   STARTED->GOING_OFFLINE_TO_RESTART
+//   GOING_OFFLINE_TO_RESTART->GOING_OFFLINE_TO_STOP
+//   GOING_OFFLINE_TO_RESTART->STARTING (after OnHostOfflineReasonAck)
+//   GOING_OFFLINE_TO_STOP->STOPPED (after OnHostOfflineReasonAck)
+//
+// |host_| must be not-null in STARTED state and nullptr in all other states
+// (although this invariant can be temporarily violated when doing
+// synchronous processing on the networking thread).
+void HostProcess::SetState(HostState target_state) {
+  DCHECK(context_->network_task_runner()->BelongsToCurrentThread());
+
+  // DCHECKs below enforce state allowed transitions listed in HostState.
+  switch (state_) {
+    case HOST_STARTING:
+      DCHECK((target_state == HOST_STARTED) ||
+             (target_state == HOST_GOING_OFFLINE_TO_STOP) ||
+             (target_state == HOST_GOING_OFFLINE_TO_RESTART))
+          << state_ << " -> " << target_state;
+      break;
+    case HOST_STARTED:
+      DCHECK((target_state == HOST_GOING_OFFLINE_TO_STOP) ||
+             (target_state == HOST_GOING_OFFLINE_TO_RESTART))
+          << state_ << " -> " << target_state;
+      break;
+    case HOST_GOING_OFFLINE_TO_RESTART:
+      DCHECK((target_state == HOST_GOING_OFFLINE_TO_STOP) ||
+             (target_state == HOST_STARTING))
+          << state_ << " -> " << target_state;
+      break;
+    case HOST_GOING_OFFLINE_TO_STOP:
+      DCHECK_EQ(target_state, HOST_STOPPED);
+      break;
+    case HOST_STOPPED:  // HOST_STOPPED is a terminal state.
+    default:
+      NOTREACHED() << state_ << " -> " << target_state;
+      break;
+  }
+  state_ = target_state;
+}
+
 void HostProcess::StartOnNetworkThread() {
   DCHECK(context_->network_task_runner()->BelongsToCurrentThread());
 
@@ -786,7 +839,6 @@ void HostProcess::ShutdownOnUiThread() {
   // Tear down resources that need to be torn down on the UI thread.
   daemon_channel_.reset();
   desktop_environment_factory_.reset();
-
   policy_watcher_.reset();
 
   // It is now safe for the HostProcess to be deleted.
@@ -981,13 +1033,13 @@ void HostProcess::OnPolicyUpdate(scoped_ptr<base::DictionaryValue> policies) {
   restart_required |= OnPairingPolicyUpdate(policies.get());
   restart_required |= OnGnubbyAuthPolicyUpdate(policies.get());
 
-  policies_loaded_ = true;
+  policy_state_ = POLICY_LOADED;
 
-  if (state_ == HOST_INITIALIZING) {
+  if (state_ == HOST_STARTING) {
     StartHostIfReady();
   } else if (state_ == HOST_STARTED) {
     if (restart_required)
-      RestartHost();
+      RestartHost(kHostOfflineReasonPolicyChangeRequiresRestart);
   }
 }
 
@@ -998,7 +1050,24 @@ void HostProcess::OnPolicyError() {
     return;
   }
 
-  ShutdownHost(kInvalidHostConfigurationExitCode);
+  if (policy_state_ != POLICY_ERROR_REPORTED) {
+    policy_state_ = POLICY_ERROR_REPORT_PENDING;
+    if ((state_ == HOST_STARTED) ||
+        (state_ == HOST_STARTING && !serialized_config_.empty())) {
+      ReportPolicyErrorAndRestartHost();
+    }
+  }
+}
+
+void HostProcess::ReportPolicyErrorAndRestartHost() {
+  DCHECK(context_->network_task_runner()->BelongsToCurrentThread());
+  DCHECK(!serialized_config_.empty());
+
+  DCHECK_EQ(policy_state_, POLICY_ERROR_REPORT_PENDING);
+  policy_state_ = POLICY_ERROR_REPORTED;
+
+  LOG(INFO) << "Restarting the host due to policy errors.";
+  RestartHost(kHostOfflineReasonPolicyReadError);
 }
 
 void HostProcess::ApplyHostDomainPolicy() {
@@ -1300,20 +1369,24 @@ scoped_ptr<HostSignalingManager> HostProcess::CreateHostSignalingManager() {
                                              oauth_refresh_token_,
                                              use_service_account_));
 
-  return HostSignalingManager::Create(this, context_->network_task_runner(),
-                                      context_->url_request_context_getter(),
-                                      xmpp_server_config_, talkgadget_prefix_,
-                                      host_id_, key_pair_, directory_bot_jid_,
-                                      oauth_credentials.Pass());
+  return HostSignalingManager::Create(
+      this, context_->url_request_context_getter(), xmpp_server_config_,
+      talkgadget_prefix_, host_id_, key_pair_, directory_bot_jid_,
+      oauth_credentials.Pass());
 }
 
 void HostProcess::StartHostIfReady() {
   DCHECK(context_->network_task_runner()->BelongsToCurrentThread());
-  DCHECK_EQ(state_, HOST_INITIALIZING);
+  DCHECK_EQ(state_, HOST_STARTING);
 
   // Start the host if both the config and the policies are loaded.
-  if (!serialized_config_.empty() && policies_loaded_)
-    StartHost();
+  if (!serialized_config_.empty()) {
+    if (policy_state_ == POLICY_LOADED) {
+      StartHost();
+    } else if (policy_state_ == POLICY_ERROR_REPORT_PENDING) {
+      ReportPolicyErrorAndRestartHost();
+    }
+  }
 }
 
 void HostProcess::StartHost() {
@@ -1321,10 +1394,7 @@ void HostProcess::StartHost() {
   DCHECK(!host_);
   DCHECK(!host_signaling_manager_);
 
-  DCHECK(state_ == HOST_INITIALIZING || state_ == HOST_STOPPING_TO_RESTART ||
-         state_ == HOST_STOPPED)
-      << "state_ = " << state_;
-  state_ = HOST_STARTED;
+  SetState(HOST_STARTED);
 
   host_signaling_manager_ = CreateHostSignalingManager();
 
@@ -1409,22 +1479,12 @@ void HostProcess::OnAuthFailed() {
   ShutdownHost(kInvalidOauthCredentialsExitCode);
 }
 
-void HostProcess::RestartHost() {
+void HostProcess::RestartHost(const std::string& host_offline_reason) {
   DCHECK(context_->network_task_runner()->BelongsToCurrentThread());
-  DCHECK_EQ(state_, HOST_STARTED);
+  DCHECK(!host_offline_reason.empty());
 
-  state_ = HOST_STOPPING_TO_RESTART;
-  ShutdownOnNetworkThread();
-}
-
-void HostProcess::SendOfflineReasonAndShutdownOnNetworkThread(
-    HostExitCodes exit_code) {
-  DCHECK(context_->network_task_runner()->BelongsToCurrentThread());
-  DCHECK(host_signaling_manager_);
-  host_signaling_manager_.release()->SendHostOfflineReasonAndDelete(
-      ExitCodeToString(exit_code),
-      base::TimeDelta::FromSeconds(kHostOfflineReasonTimeoutSeconds));
-  ShutdownOnNetworkThread();
+  SetState(HOST_GOING_OFFLINE_TO_RESTART);
+  GoOffline(host_offline_reason);
 }
 
 void HostProcess::ShutdownHost(HostExitCodes exit_code) {
@@ -1433,42 +1493,67 @@ void HostProcess::ShutdownHost(HostExitCodes exit_code) {
   *exit_code_out_ = exit_code;
 
   switch (state_) {
-    case HOST_INITIALIZING:
-      state_ = HOST_STOPPING;
-      DCHECK(!host_signaling_manager_);
-      host_signaling_manager_ = CreateHostSignalingManager();
-      SendOfflineReasonAndShutdownOnNetworkThread(exit_code);
-      break;
-
+    case HOST_STARTING:
     case HOST_STARTED:
-      state_ = HOST_STOPPING;
-      SendOfflineReasonAndShutdownOnNetworkThread(exit_code);
+      SetState(HOST_GOING_OFFLINE_TO_STOP);
+      GoOffline(ExitCodeToString(exit_code));
       break;
 
-    case HOST_STOPPING_TO_RESTART:
-      state_ = HOST_STOPPING;
+    case HOST_GOING_OFFLINE_TO_RESTART:
+      SetState(HOST_GOING_OFFLINE_TO_STOP);
       break;
 
-    case HOST_STOPPING:
+    case HOST_GOING_OFFLINE_TO_STOP:
     case HOST_STOPPED:
       // Host is already stopped or being stopped. No action is required.
       break;
   }
 }
 
-void HostProcess::ShutdownOnNetworkThread() {
+void HostProcess::GoOffline(const std::string& host_offline_reason) {
   DCHECK(context_->network_task_runner()->BelongsToCurrentThread());
+  DCHECK(!host_offline_reason.empty());
+  DCHECK((state_ == HOST_GOING_OFFLINE_TO_STOP) ||
+         (state_ == HOST_GOING_OFFLINE_TO_RESTART));
 
+  // Shut down everything except the HostSignalingManager.
   host_.reset();
   host_event_logger_.reset();
   host_status_logger_.reset();
-  host_signaling_manager_.reset();
   host_change_notification_listener_.reset();
 
-  if (state_ == HOST_STOPPING_TO_RESTART) {
-    StartHost();
-  } else if (state_ == HOST_STOPPING) {
-    state_ = HOST_STOPPED;
+  // Before shutting down HostSignalingManager, send the |host_offline_reason|
+  // if possible (i.e. if we have the config).
+  if (!serialized_config_.empty()) {
+    if (!host_signaling_manager_) {
+      host_signaling_manager_ = CreateHostSignalingManager();
+    }
+
+    host_signaling_manager_->SendHostOfflineReason(
+        host_offline_reason,
+        base::TimeDelta::FromSeconds(kHostOfflineReasonTimeoutSeconds),
+        base::Bind(&HostProcess::OnHostOfflineReasonAck, this));
+    return;  // Shutdown will resume after OnHostOfflineReasonAck.
+  }
+
+  // Continue the shutdown without sending the host offline reason.
+  HOST_LOG << "Can't send offline reason (" << host_offline_reason << ") "
+           << "without a valid host config.";
+  OnHostOfflineReasonAck(false);
+}
+
+void HostProcess::OnHostOfflineReasonAck(bool success) {
+  DCHECK(context_->network_task_runner()->BelongsToCurrentThread());
+  DCHECK(!host_);  // Assert that the host is really offline at this point.
+
+  HOST_LOG << "SendHostOfflineReason " << (success ? "succeeded." : "failed.");
+  host_signaling_manager_.reset();
+
+  if (state_ == HOST_GOING_OFFLINE_TO_RESTART) {
+    SetState(HOST_STARTING);
+    StartHostIfReady();
+  } else if (state_ == HOST_GOING_OFFLINE_TO_STOP) {
+    SetState(HOST_STOPPED);
 
     shutdown_watchdog_->SetExitCode(*exit_code_out_);
     shutdown_watchdog_->Arm();
@@ -1479,7 +1564,6 @@ void HostProcess::ShutdownOnNetworkThread() {
     context_->ui_task_runner()->PostTask(
         FROM_HERE, base::Bind(&HostProcess::ShutdownOnUiThread, this));
   } else {
-    // This method is only called in STOPPING_TO_RESTART and STOPPING states.
     NOTREACHED();
   }
 }