Add heartbeat timeout.

remoting/host/heartbeat_sender.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "remoting/host/heartbeat_sender.h"
 
 #include <math.h>
 #include <stdint.h>
 
 #include "base/bind.h"
@@ skipping 30 matching lines @@
 const char kHostOperatingSystemNameTag[] = "host-os-name";
 const char kHostOperatingSystemVersionTag[] = "host-os-version";
 
 const char kErrorTag[] = "error";
 const char kNotFoundTag[] = "item-not-found";
 
 const char kHeartbeatResultTag[] = "heartbeat-result";
 const char kSetIntervalTag[] = "set-interval";
 const char kExpectedSequenceIdTag[] = "expected-sequence-id";
 
-const int64_t kDefaultHeartbeatIntervalMs = 5 * 60 * 1000;  // 5 minutes.
-const int64_t kResendDelayMs = 10 * 1000;                   // 10 seconds.
-const int64_t kResendDelayOnHostNotFoundMs = 10 * 1000;     // 10 seconds.
+constexpr base::TimeDelta kDefaultHeartbeatInterval =
+    base::TimeDelta::FromMinutes(5);
+constexpr base::TimeDelta kHeartbeatResponseTimeout =
+    base::TimeDelta::FromSeconds(10);

[Jamie, 2017/06/02 00:53:52]

This seems a bit on the low side. IIRC, TCP timeout are typically closer to 30s;
any reason not to make this the same?

[Sergey Ulanov, 2017/06/02 18:37:01]

Done.

+constexpr base::TimeDelta kResendDelay = base::TimeDelta::FromSeconds(10);
+constexpr base::TimeDelta kResendDelayOnHostNotFound =
+    base::TimeDelta::FromSeconds(10);
+
 const int kMaxResendOnHostNotFoundCount = 12;  // 2 minutes (12 x 10 seconds).
+const int kMaxHeartbeatTimeouts = 2;
 
 }  // namespace
 
 HeartbeatSender::HeartbeatSender(
     const base::Closure& on_heartbeat_successful_callback,
     const base::Closure& on_unknown_host_id_error,
     const std::string& host_id,
     SignalStrategy* signal_strategy,
     const scoped_refptr<const RsaKeyPair>& host_key_pair,
     const std::string& directory_bot_jid)
     : on_heartbeat_successful_callback_(on_heartbeat_successful_callback),
       on_unknown_host_id_error_(on_unknown_host_id_error),
       host_id_(host_id),
       signal_strategy_(signal_strategy),
       host_key_pair_(host_key_pair),
       directory_bot_jid_(directory_bot_jid),
-      interval_ms_(kDefaultHeartbeatIntervalMs),
-      sequence_id_(0),
-      sequence_id_was_set_(false),
-      sequence_id_recent_set_num_(0),
-      heartbeat_succeeded_(false),
-      failed_startup_heartbeat_count_(0) {
+      interval_(kDefaultHeartbeatInterval) {
   DCHECK(signal_strategy_);
   DCHECK(host_key_pair_.get());
   DCHECK(thread_checker_.CalledOnValidThread());
 
   signal_strategy_->AddListener(this);
 
   // Start heartbeats if the |signal_strategy_| is already connected.
   OnSignalStrategyStateChange(signal_strategy_->GetState());
 }
 
 HeartbeatSender::~HeartbeatSender() {
   DCHECK(thread_checker_.CalledOnValidThread());
   signal_strategy_->RemoveListener(this);
 }
 
 void HeartbeatSender::OnSignalStrategyStateChange(SignalStrategy::State state) {
   DCHECK(thread_checker_.CalledOnValidThread());
   if (state == SignalStrategy::CONNECTED) {
-    iq_sender_.reset(new IqSender(signal_strategy_));
+    iq_sender_ = base::MakeUnique<IqSender>(signal_strategy_);
     SendStanza();
-    timer_.Start(FROM_HERE, base::TimeDelta::FromMilliseconds(interval_ms_),
-                 this, &HeartbeatSender::SendStanza);
+    timer_.Start(FROM_HERE, interval_, this, &HeartbeatSender::SendStanza);
   } else if (state == SignalStrategy::DISCONNECTED) {
     request_.reset();
     iq_sender_.reset();
     timer_.Stop();
     timer_resend_.Stop();
   }
 }
 
 bool HeartbeatSender::OnSignalStrategyIncomingStanza(
     const buzz::XmlElement* stanza) {
@@ skipping 53 matching lines @@
   DoSendStanza();
 }
 
 void HeartbeatSender::DoSendStanza() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(signal_strategy_->GetState() == SignalStrategy::CONNECTED);
   VLOG(1) << "Sending heartbeat stanza to " << directory_bot_jid_;
 
   request_ = iq_sender_->SendIq(
       buzz::STR_SET, directory_bot_jid_, CreateHeartbeatMessage(),
-      base::Bind(&HeartbeatSender::ProcessResponse,
-                 base::Unretained(this),
+      base::Bind(&HeartbeatSender::ProcessResponse, base::Unretained(this),
                  !host_offline_reason_.empty()));
+  request_->SetTimeout(kHeartbeatResponseTimeout);
   ++sequence_id_;
 }
 
-void HeartbeatSender::ProcessResponse(
-    bool is_offline_heartbeat_response,
-    IqRequest* request,
-    const XmlElement* response) {
+void HeartbeatSender::ProcessResponse(bool is_offline_heartbeat_response,
+                                      IqRequest* request,
+                                      const XmlElement* response) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
+  if (!response) {
+    timed_out_heartbeats_count_++;
+    if (timed_out_heartbeats_count_ >= kMaxHeartbeatTimeouts) {
+      LOG(ERROR) << "Heartbeat timed out. Reconnecting XMPP.";
+      timed_out_heartbeats_count_ = 0;
+      // SignalingConnector will reconnect SignalStrategy.
+      signal_strategy_->Disconnect();
+    } else {
+      LOG(ERROR) << "Heartbeat timed out.";
+    }
+    return;
+  } else {
+    timed_out_heartbeats_count_ = 0;
+  }
+
   std::string type = response->Attr(buzz::QN_TYPE);
   if (type == buzz::STR_ERROR) {
     const XmlElement* error_element =
         response->FirstNamed(QName(buzz::NS_CLIENT, kErrorTag));
     if (error_element) {
       if (error_element->FirstNamed(QName(buzz::NS_STANZA, kNotFoundTag))) {
         LOG(ERROR) << "Received error: Host ID not found";
         // If the host was registered immediately before it sends a heartbeat,
         // then server-side latency may prevent the server recognizing the
         // host ID in the heartbeat. So even if all of the first few heartbeats
         // get a "host ID not found" error, that's not a good enough reason to
         // exit.
         failed_startup_heartbeat_count_++;
         if (!heartbeat_succeeded_ && (failed_startup_heartbeat_count_ <=
-                kMaxResendOnHostNotFoundCount)) {
-          timer_resend_.Start(FROM_HERE,
-                              base::TimeDelta::FromMilliseconds(
-                                  kResendDelayOnHostNotFoundMs),
-                              this,
+                                      kMaxResendOnHostNotFoundCount)) {
+          timer_resend_.Start(FROM_HERE, kResendDelayOnHostNotFound, this,
                               &HeartbeatSender::ResendStanza);
           return;
         }
         on_unknown_host_id_error_.Run();
         return;
       }
     }
 
     LOG(ERROR) << "Received error in response to heartbeat: "
                << response->Str();
     return;
   }
 
   // This method must only be called for error or result stanzas.
   DCHECK_EQ(std::string(buzz::STR_RESULT), type);
 
   const XmlElement* result_element =
       response->FirstNamed(QName(kChromotingXmlNamespace, kHeartbeatResultTag));
   if (result_element) {
     const XmlElement* set_interval_element =
         result_element->FirstNamed(QName(kChromotingXmlNamespace,
                                          kSetIntervalTag));
     if (set_interval_element) {
       const std::string& interval_str = set_interval_element->BodyText();
-      int interval;
-      if (!base::StringToInt(interval_str, &interval) || interval <= 0) {
+      int interval_seconds;
+      if (!base::StringToInt(interval_str, &interval_seconds) ||
+          interval_seconds <= 0) {
         LOG(ERROR) << "Received invalid set-interval: "
                    << set_interval_element->Str();
       } else {
-        SetInterval(interval * base::Time::kMillisecondsPerSecond);
+        SetInterval(base::TimeDelta::FromSeconds(interval_seconds));
       }
     }
 
     bool did_set_sequence_id = false;
     const XmlElement* expected_sequence_id_element =
         result_element->FirstNamed(QName(kChromotingXmlNamespace,
                                          kExpectedSequenceIdTag));
     if (expected_sequence_id_element) {
       // The sequence ID sent in the previous heartbeat was not what the server
       // expected, so send another heartbeat with the expected sequence ID.
@@ skipping 20 matching lines @@
       heartbeat_succeeded_ = true;
 
       // Notify caller of SetHostOfflineReason that we got an ack.
       if (is_offline_heartbeat_response) {
         OnHostOfflineReasonAck();
       }
     }
   }
 }
 
-void HeartbeatSender::SetInterval(int interval) {
-  if (interval != interval_ms_) {
-    interval_ms_ = interval;
+void HeartbeatSender::SetInterval(base::TimeDelta interval) {
+  if (interval != interval_) {
+    interval_ = interval;
 
     // Restart the timer with the new interval.
     if (timer_.IsRunning()) {
       timer_.Stop();
-      timer_.Start(FROM_HERE, base::TimeDelta::FromMilliseconds(interval_ms_),
-                   this, &HeartbeatSender::SendStanza);
+      timer_.Start(FROM_HERE, interval_, this, &HeartbeatSender::SendStanza);
     }
   }
 }
 
 void HeartbeatSender::SetSequenceId(int sequence_id) {
   sequence_id_ = sequence_id;
   // Setting the sequence ID may be a symptom of a temporary server-side
   // problem, which would affect many hosts, so don't send a new heartbeat
   // immediately, as many hosts doing so may overload the server.
   // But the server will usually set the sequence ID when it receives the first
   // heartbeat from a host. In that case, we can send a new heartbeat
   // immediately, as that only happens once per host instance.
   if (!sequence_id_was_set_) {
     ResendStanza();
   } else {
     HOST_LOG << "The heartbeat sequence ID has been set more than once: "
               << "the new value is " << sequence_id;
-    double delay = pow(2.0, sequence_id_recent_set_num_) *
-        (1 + base::RandDouble()) * kResendDelayMs;
-    if (delay <= interval_ms_) {
-      timer_resend_.Start(FROM_HERE, base::TimeDelta::FromMilliseconds(delay),
-                          this, &HeartbeatSender::ResendStanza);
+    base::TimeDelta delay = pow(2.0, sequence_id_recent_set_num_) *
+                            (1 + base::RandDouble()) * kResendDelay;
+    if (delay <= interval_) {
+      timer_resend_.Start(FROM_HERE, delay, this,
+                          &HeartbeatSender::ResendStanza);
     }
   }
   sequence_id_was_set_ = true;
 }
 
 std::unique_ptr<XmlElement> HeartbeatSender::CreateHeartbeatMessage() {
   // Create heartbeat stanza.
   std::unique_ptr<XmlElement> heartbeat(
       new XmlElement(QName(kChromotingXmlNamespace, kHeartbeatQueryTag)));
   heartbeat->AddAttr(QName(kChromotingXmlNamespace, kHostIdAttr), host_id_);
@@ skipping 38 matching lines @@
 
   std::string message = signal_strategy_->GetLocalAddress().jid() + ' ' +
                         base::IntToString(sequence_id_);
   std::string signature(host_key_pair_->SignMessage(message));
   signature_tag->AddText(signature);
 
   return signature_tag;
 }
 
 }  // namespace remoting