QUIC - disable QUIC under recent pathological connection errors.

net/quic/quic_stream_factory.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/quic/quic_stream_factory.h"
 
 #include <algorithm>
 #include <set>
 
 #include "base/cpu.h"
@@ skipping 547 matching lines @@
     bool enable_port_selection,
     bool always_require_handshake_confirmation,
     bool disable_connection_pooling,
     float load_server_info_timeout_srtt_multiplier,
     bool enable_connection_racing,
     bool enable_non_blocking_io,
     bool disable_disk_cache,
     bool prefer_aes,
     int max_number_of_lossy_connections,
     float packet_loss_threshold,
+    int max_disabled_reasons,
+    int threshold_public_resets_post_handshake,
+    int threshold_timeouts_with_open_streams,
     int socket_receive_buffer_size,
     const QuicTagVector& connection_options)
     : require_confirmation_(true),
       host_resolver_(host_resolver),
       client_socket_factory_(client_socket_factory),
       http_server_properties_(http_server_properties),
       transport_security_state_(transport_security_state),
       quic_server_info_factory_(nullptr),
       quic_crypto_client_stream_factory_(quic_crypto_client_stream_factory),
       random_generator_(random_generator),
       clock_(clock),
       max_packet_length_(max_packet_length),
       config_(InitializeQuicConfig(connection_options)),
       supported_versions_(supported_versions),
       enable_port_selection_(enable_port_selection),
       always_require_handshake_confirmation_(
           always_require_handshake_confirmation),
       disable_connection_pooling_(disable_connection_pooling),
       load_server_info_timeout_srtt_multiplier_(
           load_server_info_timeout_srtt_multiplier),
       enable_connection_racing_(enable_connection_racing),
       enable_non_blocking_io_(enable_non_blocking_io),
       disable_disk_cache_(disable_disk_cache),
       prefer_aes_(prefer_aes),
       max_number_of_lossy_connections_(max_number_of_lossy_connections),
       packet_loss_threshold_(packet_loss_threshold),
+      max_disabled_reasons_(max_disabled_reasons),
+      num_public_resets_post_handshake_(0),
+      num_timeouts_with_open_streams_(0),
+      max_public_resets_post_handshake_(0),
+      max_timeouts_with_open_streams_(0),
+      threshold_timeouts_with_open_streams_(
+          threshold_timeouts_with_open_streams),
+      threshold_public_resets_post_handshake_(
+          threshold_public_resets_post_handshake),
       socket_receive_buffer_size_(socket_receive_buffer_size),
       port_seed_(random_generator_->RandUint64()),
       check_persisted_supports_quic_(true),
       task_runner_(nullptr),
       weak_factory_(this) {
   DCHECK(transport_security_state_);
   crypto_config_.set_user_agent_id(user_agent_id);
   crypto_config_.AddCanonicalSuffix(".c.youtube.com");
   crypto_config_.AddCanonicalSuffix(".googlevideo.com");
   crypto_config_.AddCanonicalSuffix(".googleusercontent.com");
@@ skipping 210 matching lines @@
   STLDeleteElements(&(active_jobs_[server_id]));
   active_jobs_.erase(server_id);
   job_requests_map_.erase(server_id);
 }
 
 scoped_ptr<QuicHttpStream> QuicStreamFactory::CreateFromSession(
     QuicClientSession* session) {
   return scoped_ptr<QuicHttpStream>(new QuicHttpStream(session->GetWeakPtr()));
 }
 
+QuicDisabledReason QuicStreamFactory::QuicDisabledReason(uint16 port) {
+  if (max_number_of_lossy_connections_ > 0 &&
+      number_of_lossy_connections_[port] >= max_number_of_lossy_connections_) {
+    return QUIC_DISABLED_BAD_PACKET_LOSS_RATE;
+  }
+  if (threshold_public_resets_post_handshake_ > 0 &&
+      num_public_resets_post_handshake_ >=
+          threshold_public_resets_post_handshake_) {
+    return QUIC_DISABLED_PUBLIC_RESET_POST_HANDSHAKE;
+  }
+  if (threshold_timeouts_with_open_streams_ > 0 &&
+      num_timeouts_with_open_streams_ >=
+          threshold_timeouts_with_open_streams_) {
+    return QUIC_DISABLED_TIMEOUT_WITH_OPEN_STREAMS;
+  }
+  return QUIC_DISABLED_NOT;
+}
+
+std::string QuicStreamFactory::QuicDisabledReasonString() const {
+  // TODO(ckrasic) - better solution for lossy connections?
+  uint16 port = 443;
+  if (max_number_of_lossy_connections_ > 0 &&
+      number_of_lossy_connections_.at(port) >=
+          max_number_of_lossy_connections_) {
+    return "bad packet loss rate";
+  }
+  if (threshold_public_resets_post_handshake_ > 0 &&
+      num_public_resets_post_handshake_ >=
+          threshold_public_resets_post_handshake_) {
+    return "public resets after successful handshakes";
+  }
+  if (threshold_timeouts_with_open_streams_ > 0 &&
+      num_timeouts_with_open_streams_ >=
+          threshold_timeouts_with_open_streams_) {
+    return "connection timeouts with streams open";
+  }

[Ryan Hamilton, 2015/07/06 18:25:41]

nit: It looks like this is just duplicating the logic in the previous method,
but returning strings instead of enums. So how about:

QuicDisabledReason reason = QuicDisabledReason(443);
switch (reason) {
  case QUIC_DISABLED_BAD_PACKET_LOSS_RATE:
    return "bad packet loss rate"
  ...
}

[Buck, 2015/07/06 21:39:32]

Done.

+  return "not applicable";
+}
+
 bool QuicStreamFactory::IsQuicDisabled(uint16 port) {
-  return max_number_of_lossy_connections_ > 0 &&
-         number_of_lossy_connections_[port] >= max_number_of_lossy_connections_;
+  return QuicDisabledReason(port) != QUIC_DISABLED_NOT;
 }
 
 bool QuicStreamFactory::OnHandshakeConfirmed(QuicClientSession* session,
                                              float packet_loss_rate) {
   DCHECK(session);
   uint16 port = session->server_id().port();
   if (packet_loss_rate < packet_loss_threshold_) {
     number_of_lossy_connections_[port] = 0;
     return false;
   }
@@ skipping 10 matching lines @@
 
   // Collect data for port 443 for packet loss events.
   if (port == 443 && max_number_of_lossy_connections_ > 0) {
     UMA_HISTOGRAM_SPARSE_SLOWLY(
         base::StringPrintf("Net.QuicStreamFactory.BadPacketLossEvents%d",
                            max_number_of_lossy_connections_),
         std::min(number_of_lossy_connections_[port],
                  max_number_of_lossy_connections_));
   }
 
-  bool is_quic_disabled = IsQuicDisabled(port);
-  if (is_quic_disabled) {
+  enum QuicDisabledReason disabled_reason = QuicDisabledReason(port);
+  if (disabled_reason == QUIC_DISABLED_NOT)
+    return false;
+
+  if (disabled_reason == QUIC_DISABLED_PUBLIC_RESET_POST_HANDSHAKE) {
     // Close QUIC connection if Quic is disabled for this port.
     session->CloseSessionOnErrorAndNotifyFactoryLater(
+        ERR_ABORTED, QUIC_PUBLIC_RESETS_POST_HANDSHAKE);
+  } else if (disabled_reason == QUIC_DISABLED_TIMEOUT_WITH_OPEN_STREAMS) {
+    session->CloseSessionOnErrorAndNotifyFactoryLater(
+        ERR_ABORTED, QUIC_TIMEOUTS_WITH_OPEN_STREAMS);
+  } else if (!was_quic_disabled &&
+             disabled_reason == QUIC_DISABLED_BAD_PACKET_LOSS_RATE) {
+    session->CloseSessionOnErrorAndNotifyFactoryLater(
         ERR_ABORTED, QUIC_BAD_PACKET_LOSS_RATE);
-
     // If this bad packet loss rate disabled the QUIC, then record it.
-    if (!was_quic_disabled)
-      UMA_HISTOGRAM_SPARSE_SLOWLY("Net.QuicStreamFactory.QuicIsDisabled", port);
+    UMA_HISTOGRAM_SPARSE_SLOWLY("Net.QuicStreamFactory.QuicIsDisabled", port);
   }
-  return is_quic_disabled;
+  return true;
 }
 
 void QuicStreamFactory::OnIdleSession(QuicClientSession* session) {
 }
 
 void QuicStreamFactory::OnSessionGoingAway(QuicClientSession* session) {
   const AliasSet& aliases = session_aliases_[session];
   for (AliasSet::const_iterator it = aliases.begin(); it != aliases.end();
        ++it) {
     DCHECK(active_sessions_.count(*it));
@@ skipping 12 matching lines @@
     const IpAliasKey ip_alias_key(session->connection()->peer_address(),
                                   aliases.begin()->is_https());
     ip_aliases_[ip_alias_key].erase(session);
     if (ip_aliases_[ip_alias_key].empty()) {
       ip_aliases_.erase(ip_alias_key);
     }
   }
   session_aliases_.erase(session);
 }
 
+void QuicStreamFactory::MaybeDisableQuic(QuicClientSession* session) {
+  DCHECK(session);
+  uint16 port = session->server_id().port();
+  if (IsQuicDisabled(port))
+    return;
+
+  // Expire the oldest disabled_reason if appropriate.  This enforces that we
+  // only consider the max_disabled_reasons_ most recent sessions.
+  enum QuicDisabledReason disabled_reason;
+  if (static_cast<int>(disabled_reasons_.size()) == max_disabled_reasons_) {
+    disabled_reason = disabled_reasons_.front();
+    disabled_reasons_.pop_front();
+    if (disabled_reason == QUIC_DISABLED_PUBLIC_RESET_POST_HANDSHAKE) {
+      num_public_resets_post_handshake_--;
+    } else if (disabled_reason == QUIC_DISABLED_TIMEOUT_WITH_OPEN_STREAMS) {
+      num_timeouts_with_open_streams_--;
+    }
+  }
+  disabled_reason = session->disabled_reason();
+  disabled_reasons_.push_back(disabled_reason);
+  if (disabled_reason == QUIC_DISABLED_PUBLIC_RESET_POST_HANDSHAKE) {
+    num_public_resets_post_handshake_++;
+  } else if (disabled_reason == QUIC_DISABLED_TIMEOUT_WITH_OPEN_STREAMS) {
+    num_timeouts_with_open_streams_++;
+  }
+  if (num_timeouts_with_open_streams_ > max_timeouts_with_open_streams_) {
+    max_timeouts_with_open_streams_ = num_timeouts_with_open_streams_;
+    UMA_HISTOGRAM_CUSTOM_COUNTS("Net.QuicStreamFactory.TimeoutsWithOpenStreams",
+                                num_timeouts_with_open_streams_, 0, 20, 10);
+  }
+
+  if (num_public_resets_post_handshake_ > max_public_resets_post_handshake_) {
+    max_public_resets_post_handshake_ = num_public_resets_post_handshake_;
+    UMA_HISTOGRAM_CUSTOM_COUNTS(
+        "Net.QuicStreamFactory.PublicResetsPostHandshake",
+        num_public_resets_post_handshake_, 0, 20, 10);
+  }
+
+  if (IsQuicDisabled(port)) {
+    UMA_HISTOGRAM_ENUMERATION("Net.QuicStreamFactory.DisabledReasons",
+                              disabled_reason, QUIC_DISABLED_MAX);
+  }
+}
+
 void QuicStreamFactory::OnSessionClosed(QuicClientSession* session) {
   DCHECK_EQ(0u, session->GetNumOpenStreams());
+  MaybeDisableQuic(session);
   OnSessionGoingAway(session);
   delete session;
   all_sessions_.erase(session);
 }
 
 void QuicStreamFactory::OnSessionConnectTimeout(
     QuicClientSession* session) {
   const AliasSet& aliases = session_aliases_[session];
   for (AliasSet::const_iterator it = aliases.begin(); it != aliases.end();
        ++it) {
@@ skipping 359 matching lines @@
   // Since the session was active, there's no longer an
   // HttpStreamFactoryImpl::Job running which can mark it broken, unless the TCP
   // job also fails. So to avoid not using QUIC when we otherwise could, we mark
   // it as recently broken, which means that 0-RTT will be disabled but we'll
   // still race.
   http_server_properties_->MarkAlternativeServiceRecentlyBroken(
       alternative_service);
 }
 
 }  // namespace net