QUIC - disable QUIC under recent pathological connection errors.

net/quic/quic_stream_factory.cc

Index: net/quic/quic_stream_factory.cc
diff --git a/net/quic/quic_stream_factory.cc b/net/quic/quic_stream_factory.cc
index 02b0b986be890fff786eafe859914e1d963c3fa4..1f459f06a348f8d2559edd93bf99ad4ef3631ebb 100644
--- a/net/quic/quic_stream_factory.cc
+++ b/net/quic/quic_stream_factory.cc
@@ -565,6 +565,9 @@ QuicStreamFactory::QuicStreamFactory(
     bool prefer_aes,
     int max_number_of_lossy_connections,
     float packet_loss_threshold,
+    int max_disabled_reasons,
+    int threshold_public_resets_post_handshake,
+    int threshold_timeouts_with_open_streams,
     int socket_receive_buffer_size,
     const QuicTagVector& connection_options)
     : require_confirmation_(true),
@@ -591,6 +594,15 @@ QuicStreamFactory::QuicStreamFactory(
       prefer_aes_(prefer_aes),
       max_number_of_lossy_connections_(max_number_of_lossy_connections),
       packet_loss_threshold_(packet_loss_threshold),
+      max_disabled_reasons_(max_disabled_reasons),
+      num_public_resets_post_handshake_(0),
+      num_timeouts_with_open_streams_(0),
+      max_public_resets_post_handshake_(0),
+      max_timeouts_with_open_streams_(0),
+      threshold_timeouts_with_open_streams_(
+          threshold_timeouts_with_open_streams),
+      threshold_public_resets_post_handshake_(
+          threshold_public_resets_post_handshake),
       socket_receive_buffer_size_(socket_receive_buffer_size),
       port_seed_(random_generator_->RandUint64()),
       check_persisted_supports_quic_(true),
@@ -821,9 +833,47 @@ scoped_ptr<QuicHttpStream> QuicStreamFactory::CreateFromSession(
   return scoped_ptr<QuicHttpStream>(new QuicHttpStream(session->GetWeakPtr()));
 }
 
+QuicDisabledReason QuicStreamFactory::QuicDisabledReason(uint16 port) {
+  if (max_number_of_lossy_connections_ > 0 &&
+      number_of_lossy_connections_[port] >= max_number_of_lossy_connections_) {
+    return QUIC_DISABLED_BAD_PACKET_LOSS_RATE;
+  }
+  if (threshold_public_resets_post_handshake_ > 0 &&
+      num_public_resets_post_handshake_ >=
+          threshold_public_resets_post_handshake_) {
+    return QUIC_DISABLED_PUBLIC_RESET_POST_HANDSHAKE;
+  }
+  if (threshold_timeouts_with_open_streams_ > 0 &&
+      num_timeouts_with_open_streams_ >=
+          threshold_timeouts_with_open_streams_) {
+    return QUIC_DISABLED_TIMEOUT_WITH_OPEN_STREAMS;
+  }
+  return QUIC_DISABLED_NOT;
+}
+
+std::string QuicStreamFactory::QuicDisabledReasonString() const {
+  // TODO(ckrasic) - better solution for lossy connections?
+  uint16 port = 443;
+  if (max_number_of_lossy_connections_ > 0 &&
+      number_of_lossy_connections_.at(port) >=
+          max_number_of_lossy_connections_) {
+    return "bad packet loss rate";
+  }
+  if (threshold_public_resets_post_handshake_ > 0 &&
+      num_public_resets_post_handshake_ >=
+          threshold_public_resets_post_handshake_) {
+    return "public resets after successful handshakes";
+  }
+  if (threshold_timeouts_with_open_streams_ > 0 &&
+      num_timeouts_with_open_streams_ >=
+          threshold_timeouts_with_open_streams_) {
+    return "connection timeouts with streams open";
+  }

[Ryan Hamilton, 2015/07/06 18:25:41]

nit: It looks like this is just duplicating the logic in the previous method,
but returning strings instead of enums. So how about:

QuicDisabledReason reason = QuicDisabledReason(443);
switch (reason) {
  case QUIC_DISABLED_BAD_PACKET_LOSS_RATE:
    return "bad packet loss rate"
  ...
}

[Buck, 2015/07/06 21:39:32]

Done.

+  return "not applicable";
+}
+
 bool QuicStreamFactory::IsQuicDisabled(uint16 port) {
-  return max_number_of_lossy_connections_ > 0 &&
-         number_of_lossy_connections_[port] >= max_number_of_lossy_connections_;
+  return QuicDisabledReason(port) != QUIC_DISABLED_NOT;
 }
 
 bool QuicStreamFactory::OnHandshakeConfirmed(QuicClientSession* session,
@@ -854,17 +904,25 @@ bool QuicStreamFactory::OnHandshakeConfirmed(QuicClientSession* session,
                  max_number_of_lossy_connections_));
   }
 
-  bool is_quic_disabled = IsQuicDisabled(port);
-  if (is_quic_disabled) {
+  enum QuicDisabledReason disabled_reason = QuicDisabledReason(port);
+  if (disabled_reason == QUIC_DISABLED_NOT)
+    return false;
+
+  if (disabled_reason == QUIC_DISABLED_PUBLIC_RESET_POST_HANDSHAKE) {
     // Close QUIC connection if Quic is disabled for this port.
     session->CloseSessionOnErrorAndNotifyFactoryLater(
+        ERR_ABORTED, QUIC_PUBLIC_RESETS_POST_HANDSHAKE);
+  } else if (disabled_reason == QUIC_DISABLED_TIMEOUT_WITH_OPEN_STREAMS) {
+    session->CloseSessionOnErrorAndNotifyFactoryLater(
+        ERR_ABORTED, QUIC_TIMEOUTS_WITH_OPEN_STREAMS);
+  } else if (!was_quic_disabled &&
+             disabled_reason == QUIC_DISABLED_BAD_PACKET_LOSS_RATE) {
+    session->CloseSessionOnErrorAndNotifyFactoryLater(
         ERR_ABORTED, QUIC_BAD_PACKET_LOSS_RATE);
-
     // If this bad packet loss rate disabled the QUIC, then record it.
-    if (!was_quic_disabled)
-      UMA_HISTOGRAM_SPARSE_SLOWLY("Net.QuicStreamFactory.QuicIsDisabled", port);
+    UMA_HISTOGRAM_SPARSE_SLOWLY("Net.QuicStreamFactory.QuicIsDisabled", port);
   }
-  return is_quic_disabled;
+  return true;
 }
 
 void QuicStreamFactory::OnIdleSession(QuicClientSession* session) {
@@ -897,8 +955,53 @@ void QuicStreamFactory::OnSessionGoingAway(QuicClientSession* session) {
   session_aliases_.erase(session);
 }
 
+void QuicStreamFactory::MaybeDisableQuic(QuicClientSession* session) {
+  DCHECK(session);
+  uint16 port = session->server_id().port();
+  if (IsQuicDisabled(port))
+    return;
+
+  // Expire the oldest disabled_reason if appropriate.  This enforces that we
+  // only consider the max_disabled_reasons_ most recent sessions.
+  enum QuicDisabledReason disabled_reason;
+  if (static_cast<int>(disabled_reasons_.size()) == max_disabled_reasons_) {
+    disabled_reason = disabled_reasons_.front();
+    disabled_reasons_.pop_front();
+    if (disabled_reason == QUIC_DISABLED_PUBLIC_RESET_POST_HANDSHAKE) {
+      num_public_resets_post_handshake_--;
+    } else if (disabled_reason == QUIC_DISABLED_TIMEOUT_WITH_OPEN_STREAMS) {
+      num_timeouts_with_open_streams_--;
+    }
+  }
+  disabled_reason = session->disabled_reason();
+  disabled_reasons_.push_back(disabled_reason);
+  if (disabled_reason == QUIC_DISABLED_PUBLIC_RESET_POST_HANDSHAKE) {
+    num_public_resets_post_handshake_++;
+  } else if (disabled_reason == QUIC_DISABLED_TIMEOUT_WITH_OPEN_STREAMS) {
+    num_timeouts_with_open_streams_++;
+  }
+  if (num_timeouts_with_open_streams_ > max_timeouts_with_open_streams_) {
+    max_timeouts_with_open_streams_ = num_timeouts_with_open_streams_;
+    UMA_HISTOGRAM_CUSTOM_COUNTS("Net.QuicStreamFactory.TimeoutsWithOpenStreams",
+                                num_timeouts_with_open_streams_, 0, 20, 10);
+  }
+
+  if (num_public_resets_post_handshake_ > max_public_resets_post_handshake_) {
+    max_public_resets_post_handshake_ = num_public_resets_post_handshake_;
+    UMA_HISTOGRAM_CUSTOM_COUNTS(
+        "Net.QuicStreamFactory.PublicResetsPostHandshake",
+        num_public_resets_post_handshake_, 0, 20, 10);
+  }
+
+  if (IsQuicDisabled(port)) {
+    UMA_HISTOGRAM_ENUMERATION("Net.QuicStreamFactory.DisabledReasons",
+                              disabled_reason, QUIC_DISABLED_MAX);
+  }
+}
+
 void QuicStreamFactory::OnSessionClosed(QuicClientSession* session) {
   DCHECK_EQ(0u, session->GetNumOpenStreams());
+  MaybeDisableQuic(session);
   OnSessionGoingAway(session);
   delete session;
   all_sessions_.erase(session);