Adds TCP FastOpen blackhole recovery code to avoid getting persistently blackholed.

net/socket/tcp_socket_libevent.h

Index: net/socket/tcp_socket_libevent.h
diff --git a/net/socket/tcp_socket_libevent.h b/net/socket/tcp_socket_libevent.h
index 89b6a22bdd08d24b05529b6a9f2031351098d256..b8e0030472e1c466eee5becd1c30271b61ea9a52 100644
--- a/net/socket/tcp_socket_libevent.h
+++ b/net/socket/tcp_socket_libevent.h
@@ -88,52 +88,61 @@ class NET_EXPORT TCPSocketLibevent {
 
  private:
   // States that a fast open socket attempt can result in.
-  enum FastOpenStatus {
-    FAST_OPEN_STATUS_UNKNOWN,
+  enum TCPFastOpenStatus {
+    TCP_FASTOPEN_STATUS_UNKNOWN,
 
     // The initial fast open connect attempted returned synchronously,
     // indicating that we had and sent a cookie along with the initial data.
-    FAST_OPEN_FAST_CONNECT_RETURN,
+    TCP_FASTOPEN_FAST_CONNECT_RETURN,
 
     // The initial fast open connect attempted returned asynchronously,
     // indicating that we did not have a cookie for the server.
-    FAST_OPEN_SLOW_CONNECT_RETURN,
+    TCP_FASTOPEN_SLOW_CONNECT_RETURN,
 
     // Some other error occurred on connection, so we couldn't tell if
     // fast open would have worked.
-    FAST_OPEN_ERROR,
+    TCP_FASTOPEN_ERROR,
 
     // An attempt to do a fast open succeeded immediately
-    // (FAST_OPEN_FAST_CONNECT_RETURN) and we later confirmed that the server
+    // (TCP_FASTOPEN_FAST_CONNECT_RETURN) and we later confirmed that the server
     // had acked the data we sent.
-    FAST_OPEN_SYN_DATA_ACK,
+    TCP_FASTOPEN_SYN_DATA_ACK,
 
     // An attempt to do a fast open succeeded immediately
-    // (FAST_OPEN_FAST_CONNECT_RETURN) and we later confirmed that the server
+    // (TCP_FASTOPEN_FAST_CONNECT_RETURN) and we later confirmed that the server
     // had nacked the data we sent.
-    FAST_OPEN_SYN_DATA_NACK,
+    TCP_FASTOPEN_SYN_DATA_NACK,
 
     // An attempt to do a fast open succeeded immediately
-    // (FAST_OPEN_FAST_CONNECT_RETURN) and our probe to determine if the
+    // (TCP_FASTOPEN_FAST_CONNECT_RETURN) and our probe to determine if the
     // socket was using fast open failed.
-    FAST_OPEN_SYN_DATA_FAILED,
+    TCP_FASTOPEN_SYN_DATA_GETSOCKOPT_FAILED,
 
-    // An attempt to do a fast open failed (FAST_OPEN_SLOW_CONNECT_RETURN)
+    // An attempt to do a fast open failed (TCP_FASTOPEN_SLOW_CONNECT_RETURN)
     // and we later confirmed that the server had acked initial data.  This
     // should never happen (we didn't send data, so it shouldn't have
     // been acked).
-    FAST_OPEN_NO_SYN_DATA_ACK,
+    TCP_FASTOPEN_NO_SYN_DATA_ACK,
 
-    // An attempt to do a fast open failed (FAST_OPEN_SLOW_CONNECT_RETURN)
+    // An attempt to do a fast open failed (TCP_FASTOPEN_SLOW_CONNECT_RETURN)
     // and we later discovered that the server had nacked initial data.  This
-    // is the expected case results for FAST_OPEN_SLOW_CONNECT_RETURN.
-    FAST_OPEN_NO_SYN_DATA_NACK,
+    // is the expected case results for TCP_FASTOPEN_SLOW_CONNECT_RETURN.
+    TCP_FASTOPEN_NO_SYN_DATA_NACK,
 
-    // An attempt to do a fast open failed (FAST_OPEN_SLOW_CONNECT_RETURN)
+    // An attempt to do a fast open failed (TCP_FASTOPEN_SLOW_CONNECT_RETURN)
     // and our later probe for ack/nack state failed.
-    FAST_OPEN_NO_SYN_DATA_FAILED,
+    TCP_FASTOPEN_NO_SYN_DATA_GETSOCKOPT_FAILED,
 
-    FAST_OPEN_MAX_VALUE
+    // The initial fast open connect succeeded immediately
+    // (TCP_FASTOPEN_FAST_CONNECT_RETURN) and a subsequent attempt to read from
+    // the connection resulted in failure.
+    TCP_FASTOPEN_FAST_CONNECT_FAILED,
+
+    // The initial fast open connect failed (TCP_FASTOPEN_SLOW_CONNECT_RETURN)
+    // and a subsequent attempt to read from the connection resulted in failure.
+    TCP_FASTOPEN_SLOW_CONNECT_FAILED,
+
+    TCP_FASTOPEN_MAX_VALUE

[mmenke, 2014/09/24 14:46:11]

One high level question...  How do you intend to look at these histograms?  The
fail and success numbers aren't really comparable, since we disable fastopen on
failure.

Suppose we have two users, FastOpen works for one, but errors out on the other. 
Both use 100 connections that could theoretically support fastopen.

We record 1 failure (Maybe a couple more, if we start multiple connections
before our first failure), and 100 successes.  How do you interpret those
histogram numbers?

[Jana, 2014/09/25 16:10:16]

This is a good point. I thought about this a while... I've added a histogram
element that records "Could have tried TFO but didn't since it had previously
failed." I think between this one and the rest of the success/failure cases,
I'll have all I need. In the case you describe, we will now record 1 failure +
99 previously_failed. I'm failrly certain that this gives us everything we need:
(i) #successes vs. #failures (and potential_failures), and (ii) the distribution
of various kinds of failures.

   };
 
   void AcceptCompleted(scoped_ptr<TCPSocketLibevent>* tcp_socket,
@@ -158,14 +167,14 @@ class NET_EXPORT TCPSocketLibevent {
 
   void WriteCompleted(const scoped_refptr<IOBuffer>& buf,
                       const CompletionCallback& callback,
-                      int rv) const;
-  int HandleWriteCompleted(IOBuffer* buf, int rv) const;
+                      int rv);
+  int HandleWriteCompleted(IOBuffer* buf, int rv);
   int TcpFastOpenWrite(IOBuffer* buf,
                        int buf_len,
                        const CompletionCallback& callback);
 
   // Called when the socket is known to be in a connected state.
-  void RecordFastOpenStatus();
+  void RecordTCPFastOpenStatus();
 
   scoped_ptr<SocketLibevent> socket_;
   scoped_ptr<SocketLibevent> accept_socket_;
@@ -173,9 +182,14 @@ class NET_EXPORT TCPSocketLibevent {
   // Enables experimental TCP FastOpen option.
   bool use_tcp_fastopen_;
 
+  // True when TCP FastOpen is in use and we have attempted the
+  // connect with write.
+  bool tcp_fastopen_write_attempted_;
+
   // True when TCP FastOpen is in use and we have done the connect.
   bool tcp_fastopen_connected_;
-  FastOpenStatus fast_open_status_;
+
+  TCPFastOpenStatus tcp_fastopen_status_;
 
   bool logging_multiple_connect_attempts_;