Adds TCP FastOpen blackhole recovery code to avoid getting persistently blackholed.

net/socket/tcp_socket_libevent.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "net/socket/tcp_socket.h"
 
 #include <errno.h>
 #include <netinet/tcp.h>
 #include <sys/socket.h>
 
@@ skipping 21 matching lines @@
 
 namespace net {
 
 namespace {
 
 // True if OS supports TCP FastOpen.
 bool g_tcp_fastopen_supported = false;
 // True if TCP FastOpen is user-enabled for all connections.
 // TODO(jri): Change global variable to param in HttpNetworkSession::Params.
 bool g_tcp_fastopen_user_enabled = false;
+// True if TCP FastOpen connect-with-write has failed at least once.
+bool g_tcp_fastopen_has_failed = false;
+
 
 // SetTCPNoDelay turns on/off buffering in the kernel. By default, TCP sockets
 // will wait up to 200ms for more data to complete a packet before transmitting.
 // After calling this function, the kernel will not wait. See TCP_NODELAY in
 // `man 7 tcp`.
 bool SetTCPNoDelay(int fd, bool no_delay) {
   int on = no_delay ? 1 : 0;
   int error = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on));
   return error == 0;
 }
@@ skipping 70 matching lines @@
       base::WorkerPool::GetTaskRunner(/*task_is_slow=*/false).get(),
       FROM_HERE,
       base::Bind(SystemSupportsTCPFastOpen),
       base::Bind(RegisterTCPFastOpenIntentAndSupport, user_enabled));
 #endif
 }
 
 TCPSocketLibevent::TCPSocketLibevent(NetLog* net_log,
                                      const NetLog::Source& source)
     : use_tcp_fastopen_(false),
+      tcp_fastopen_write_attempted_(false),
       tcp_fastopen_connected_(false),
-      fast_open_status_(FAST_OPEN_STATUS_UNKNOWN),
+      tcp_fastopen_status_(FAST_OPEN_STATUS_UNKNOWN),
       logging_multiple_connect_attempts_(false),
       net_log_(BoundNetLog::Make(net_log, NetLog::SOURCE_SOCKET)) {
   net_log_.BeginEvent(NetLog::TYPE_SOCKET_ALIVE,
                       source.ToEventParametersCallback());
 }
 
 TCPSocketLibevent::~TCPSocketLibevent() {
   net_log_.EndEvent(NetLog::TYPE_SOCKET_ALIVE);
-  if (tcp_fastopen_connected_) {
+  if (tcp_fastopen_write_attempted_) {
     UMA_HISTOGRAM_ENUMERATION("Net.TcpFastOpenSocketConnection",
-                              fast_open_status_, FAST_OPEN_MAX_VALUE);
+                              tcp_fastopen_status_, FAST_OPEN_MAX_VALUE);
   }
 }
 
 int TCPSocketLibevent::Open(AddressFamily family) {
   DCHECK(!socket_);
   socket_.reset(new SocketLibevent);
   int rv = socket_->Open(ConvertAddressFamily(family));
   if (rv != OK)
     socket_.reset();
   return rv;
@@ skipping 60 matching lines @@
 
   net_log_.BeginEvent(NetLog::TYPE_TCP_CONNECT_ATTEMPT,
                       CreateNetLogIPEndPointCallback(&address));
 
   SockaddrStorage storage;
   if (!address.ToSockAddr(storage.addr, &storage.addr_len))
     return ERR_ADDRESS_INVALID;
 
   if (use_tcp_fastopen_) {
     // With TCP FastOpen, we pretend that the socket is connected.
-    DCHECK(!tcp_fastopen_connected_);
+    DCHECK(!tcp_fastopen_write_attempted_);
     socket_->SetPeerAddress(storage);
     return OK;
   }
 
   int rv = socket_->Connect(storage,
                             base::Bind(&TCPSocketLibevent::ConnectCompleted,
                                        base::Unretained(this), callback));
   if (rv != ERR_IO_PENDING)
     rv = HandleConnectCompleted(rv);
   return rv;
 }
 
 bool TCPSocketLibevent::IsConnected() const {
   if (!socket_)
     return false;
 
-  if (use_tcp_fastopen_ && !tcp_fastopen_connected_ &&
+  if (use_tcp_fastopen_ && !tcp_fastopen_write_attempted_ &&
       socket_->HasPeerAddress()) {
     // With TCP FastOpen, we pretend that the socket is connected.
     // This allows GetPeerAddress() to return peer_address_.
     return true;
   }
 
   return socket_->IsConnected();
 }
 
 bool TCPSocketLibevent::IsConnectedAndIdle() const {
   // TODO(wtc): should we also handle the TCP FastOpen case here,
   // as we do in IsConnected()?
   return socket_ && socket_->IsConnectedAndIdle();
 }
 
 int TCPSocketLibevent::Read(IOBuffer* buf,
                             int buf_len,
                             const CompletionCallback& callback) {
   DCHECK(socket_);
   DCHECK(!callback.is_null());
 
   int rv = socket_->Read(
       buf, buf_len,
       base::Bind(&TCPSocketLibevent::ReadCompleted,
                  // Grab a reference to |buf| so that ReadCompleted() can still
                  // use it when Read() completes, as otherwise, this transfers
                  // ownership of buf to socket.
                  base::Unretained(this), make_scoped_refptr(buf), callback));
   if (rv >= 0)
-    RecordFastOpenStatus();
+    RecordTCPFastOpenStatus();
   if (rv != ERR_IO_PENDING)
     rv = HandleReadCompleted(buf, rv);
   return rv;
 }
 
 int TCPSocketLibevent::Write(IOBuffer* buf,
                              int buf_len,
                              const CompletionCallback& callback) {
   DCHECK(socket_);
   DCHECK(!callback.is_null());
 
   CompletionCallback write_callback =
       base::Bind(&TCPSocketLibevent::WriteCompleted,
                  // Grab a reference to |buf| so that WriteCompleted() can still
                  // use it when Write() completes, as otherwise, this transfers
                  // ownership of buf to socket.
                  base::Unretained(this), make_scoped_refptr(buf), callback);
   int rv;
-  if (use_tcp_fastopen_ && !tcp_fastopen_connected_) {
+
+  if (use_tcp_fastopen_ && !tcp_fastopen_write_attempted_) {
     rv = TcpFastOpenWrite(buf, buf_len, write_callback);
   } else {
     rv = socket_->Write(buf, buf_len, write_callback);
   }
 
   if (rv != ERR_IO_PENDING)
     rv = HandleWriteCompleted(buf, rv);
   return rv;
 }
 
@@ skipping 106 matching lines @@
 }
 
 bool TCPSocketLibevent::SetNoDelay(bool no_delay) {
   DCHECK(socket_);
   return SetTCPNoDelay(socket_->socket_fd(), no_delay);
 }
 
 void TCPSocketLibevent::Close() {
   socket_.reset();
   tcp_fastopen_connected_ = false;
-  fast_open_status_ = FAST_OPEN_STATUS_UNKNOWN;
+  tcp_fastopen_status_ = FAST_OPEN_STATUS_UNKNOWN;

[Jana, 2014/09/19 02:18:16]

Why do we need to reset tcp_fastopen_status_ here?

[Randy Smith (Not in Mondays), 2014/09/22 16:05:13]

So I don't remember clearly why I did this, but looking at the code, I suspect I
was worried about the object being re-used; I don't see anything in the API
definition of the implementation that disallows that?

[Jana, 2014/09/22 17:04:56]

Maybe Matt can answer this question, but as I read it, I don't think that
happens... client_socket_factory creates TCPClientSockets, and a new TCPSocket
(which maps to TCPSocketLibevent for *ix) is created for each TCPClientSocket.
That said, I agree that I also don't see anything in the API definition of
TCPSocketLibevent that explicitly prevents an object from being reused...

If it is possible that a TCPSocketLibevent object is reused, then we'll have to
reset use_tcp_fastopen_ as well, and I'd want to double-check the code to make
sure that state is correctly reset.

[Randy Smith (Not in Mondays), 2014/09/22 17:42:47]

So I'm a fairly big believer in not making assumptions beyond what's codified in
the header file about how a class will be used.  If you wanted to stick a note
in the header file about not reusing the class and dump a DCHECK someplace in
the state transition to make sure it's not being re-used, I'm comfortable with
not resetting the state.

[Jana, 2014/09/22 23:16:04]

Yup, that sounds reasonable. I will try a DCHECK and check in with Matt as well.
If not, we'll have to reset more TCP FastOpen state as well.

 }
 
 bool TCPSocketLibevent::UsingTCPFastOpen() const {
   return use_tcp_fastopen_;
 }
 
 void TCPSocketLibevent::EnableTCPFastOpenIfSupported() {
-  if (IsTCPFastOpenSupported())
+  // Do not enable TCP FastOpen if it had previously failed.
+  // This check conservatively avoids middleboxes that may blackhole
+  // TCP FastOpen SYN+Data packets; on such a failure, subsequent sockets
+  // should not use TCP FastOpen.
+  if (IsTCPFastOpenSupported() && !g_tcp_fastopen_has_failed)
     use_tcp_fastopen_ = true;
 }
 
 bool TCPSocketLibevent::IsValid() const {
   return socket_ != NULL && socket_->socket_fd() != kInvalidSocket;
 }
 
 void TCPSocketLibevent::StartLoggingMultipleConnectAttempts(
     const AddressList& addresses) {
   if (!logging_multiple_connect_attempts_) {
@@ skipping 112 matching lines @@
                                                       storage.addr_len));
 }
 
 void TCPSocketLibevent::ReadCompleted(const scoped_refptr<IOBuffer>& buf,
                                       const CompletionCallback& callback,
                                       int rv) {
   DCHECK_NE(ERR_IO_PENDING, rv);
   // Records TCP FastOpen status regardless of error in asynchronous case.
   // TODO(rdsmith,jri): Change histogram name to indicate it could be called on
   // error.
-  RecordFastOpenStatus();
+  RecordTCPFastOpenStatus();
   callback.Run(HandleReadCompleted(buf.get(), rv));
 }
 
 int TCPSocketLibevent::HandleReadCompleted(IOBuffer* buf, int rv) {
   if (rv < 0) {
     net_log_.AddEvent(NetLog::TYPE_SOCKET_READ_ERROR,
                       CreateNetLogSocketErrorCallback(rv, errno));
     return rv;
   }
 
   base::StatsCounter read_bytes("tcp.read_bytes");
   read_bytes.Add(rv);
   net_log_.AddByteTransferEvent(NetLog::TYPE_SOCKET_BYTES_RECEIVED, rv,
                                 buf->data());
   return rv;
 }
 
 void TCPSocketLibevent::WriteCompleted(const scoped_refptr<IOBuffer>& buf,
                                        const CompletionCallback& callback,
                                        int rv) const {
   DCHECK_NE(ERR_IO_PENDING, rv);
   callback.Run(HandleWriteCompleted(buf.get(), rv));
 }
 
 int TCPSocketLibevent::HandleWriteCompleted(IOBuffer* buf, int rv) const {
   if (rv < 0) {
     net_log_.AddEvent(NetLog::TYPE_SOCKET_WRITE_ERROR,
                       CreateNetLogSocketErrorCallback(rv, errno));
+    if (tcp_fastopen_write_attempted_ && !tcp_fastopen_connected_) {
+      // TCP FastOpen connect-with-write was attempted, but failed.
+      // Turn off TCP FastOpen for any subsequent connections.
+      g_tcp_fastopen_has_failed = true;
+      RecordTCPFastOpenStatus();
+    }
     return rv;
   }
 
+  tcp_fastopen_connected_ = true;
   base::StatsCounter write_bytes("tcp.write_bytes");
   write_bytes.Add(rv);
   net_log_.AddByteTransferEvent(NetLog::TYPE_SOCKET_BYTES_SENT, rv,
                                 buf->data());
   return rv;
 }
 
 int TCPSocketLibevent::TcpFastOpenWrite(
     IOBuffer* buf,
     int buf_len,
@@ skipping 10 matching lines @@
   // for system support on startup, but users may dynamically disable TCP Fast
   // Open via sysctl.
   flags |= MSG_NOSIGNAL;
 #endif // defined(OS_LINUX)
   rv = HANDLE_EINTR(sendto(socket_->socket_fd(),
                            buf->data(),
                            buf_len,
                            flags,
                            storage.addr,
                            storage.addr_len));
-  tcp_fastopen_connected_ = true;
+  tcp_fastopen_write_attempted_ = true;
 
   if (rv >= 0) {
-    fast_open_status_ = FAST_OPEN_FAST_CONNECT_RETURN;
+    tcp_fastopen_status_ = FAST_OPEN_FAST_CONNECT_RETURN;
     return rv;
   }
 
   DCHECK_NE(EPIPE, errno);
 
   // If errno == EINPROGRESS, that means the kernel didn't have a cookie
   // and would block. The kernel is internally doing a connect() though.
   // Remap EINPROGRESS to EAGAIN so we treat this the same as our other
   // asynchronous cases. Note that the user buffer has not been copied to
   // kernel space.
   if (errno == EINPROGRESS) {
     rv = ERR_IO_PENDING;
   } else {
     rv = MapSystemError(errno);
   }
 
   if (rv != ERR_IO_PENDING) {
-    fast_open_status_ = FAST_OPEN_ERROR;
+    tcp_fastopen_status_ = FAST_OPEN_ERROR;
     return rv;
   }
 
-  fast_open_status_ = FAST_OPEN_SLOW_CONNECT_RETURN;
+  tcp_fastopen_status_ = FAST_OPEN_SLOW_CONNECT_RETURN;
   return socket_->WaitForWrite(buf, buf_len, callback);
 }
 
-void TCPSocketLibevent::RecordFastOpenStatus() {
+void TCPSocketLibevent::RecordTCPFastOpenStatus() {
   if (use_tcp_fastopen_ &&
-      (fast_open_status_ == FAST_OPEN_FAST_CONNECT_RETURN ||
-       fast_open_status_ == FAST_OPEN_SLOW_CONNECT_RETURN)) {
-    DCHECK_NE(FAST_OPEN_STATUS_UNKNOWN, fast_open_status_);
+      (tcp_fastopen_status_ == FAST_OPEN_FAST_CONNECT_RETURN ||
+       tcp_fastopen_status_ == FAST_OPEN_SLOW_CONNECT_RETURN)) {
+    DCHECK_NE(FAST_OPEN_STATUS_UNKNOWN, tcp_fastopen_status_);
+
+    if (g_tcp_fastopen_has_failed && tcp_fastopen_write_attempted_) {
+      // Writing TCP FastOpen SYN+DATA or SYN-option failed.
+      tcp_fastopen_status_ =
+          (tcp_fastopen_status_ == FAST_OPEN_FAST_CONNECT_RETURN ?
+           FAST_OPEN_FAST_CONNECT_FAILED : FAST_OPEN_SLOW_CONNECT_FAILED);
+      return;
+    }
+
     bool getsockopt_success(false);
     bool server_acked_data(false);
 #if defined(TCP_INFO)
     // Probe to see the if the socket used TCP FastOpen.
     tcp_info info;
     socklen_t info_len = sizeof(tcp_info);
     getsockopt_success =
         getsockopt(socket_->socket_fd(), IPPROTO_TCP, TCP_INFO,
                    &info, &info_len) == 0 &&
         info_len == sizeof(tcp_info);
     server_acked_data = getsockopt_success &&
         (info.tcpi_options & TCPI_OPT_SYN_DATA);
 #endif
     if (getsockopt_success) {
-      if (fast_open_status_ == FAST_OPEN_FAST_CONNECT_RETURN) {
-        fast_open_status_ = (server_acked_data ? FAST_OPEN_SYN_DATA_ACK :
+      if (tcp_fastopen_status_ == FAST_OPEN_FAST_CONNECT_RETURN) {
+        tcp_fastopen_status_ = (server_acked_data ? FAST_OPEN_SYN_DATA_ACK :
                              FAST_OPEN_SYN_DATA_NACK);
       } else {
-        fast_open_status_ = (server_acked_data ? FAST_OPEN_NO_SYN_DATA_ACK :
+        tcp_fastopen_status_ = (server_acked_data ? FAST_OPEN_NO_SYN_DATA_ACK :
                              FAST_OPEN_NO_SYN_DATA_NACK);
       }
     } else {
-      fast_open_status_ = (fast_open_status_ == FAST_OPEN_FAST_CONNECT_RETURN ?
-                           FAST_OPEN_SYN_DATA_FAILED :
-                           FAST_OPEN_NO_SYN_DATA_FAILED);
+      tcp_fastopen_status_ =
+          (tcp_fastopen_status_ == FAST_OPEN_FAST_CONNECT_RETURN ?
+           FAST_OPEN_SYN_DATA_FAILED : FAST_OPEN_NO_SYN_DATA_FAILED);
     }
   }
 }
 
 }  // namespace net