Optimizations to fill packets better for the edsm server....

net/tools/flip_server/flip_in_mem_edsm_server.cc

 // Copyright (c) 2009 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <dirent.h>
 #include <netinet/tcp.h>  // For TCP_NODELAY
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <sys/file.h>
 #include <sys/stat.h>
@@ skipping 261 matching lines @@
   PrintSslError();
 
   SSL_set_accept_state(ssl);
   PrintSslError();
 
   return ssl;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
-const int kMSS = 1400;  // Linux default
-const int kSSLOverhead = 33;
+const int kMSS = 1460;
+const int kSSLOverhead = 25;
 const int kSpdyOverhead = SpdyFrame::size();
 const int kInitialDataSendersThreshold = (2 * kMSS) - kSpdyOverhead;
 const int kSSLSegmentSize = (1 * kMSS) - kSSLOverhead;
 const int kSpdySegmentSize = kSSLSegmentSize - kSpdyOverhead;
 
 ////////////////////////////////////////////////////////////////////////////////
 
 class DataFrame {
  public:
   const char* data;
@@ skipping 508 matching lines @@
         events_(0),
         registered_in_epoll_server_(false),
         initialized_(false),
         protocol_detected_(false),
         connection_complete_(false),
         connection_pool_(NULL),
         epoll_server_(epoll_server),
         ssl_state_(ssl_state),
         memory_cache_(memory_cache),
         acceptor_(acceptor),
-        read_buffer_(4096*10),
+        read_buffer_(kSpdySegmentSize * 40),
         sm_spdy_interface_(NULL),
         sm_http_interface_(NULL),
         sm_streamer_interface_(NULL),
         sm_interface_(NULL),
         log_prefix_(log_prefix),
         max_bytes_sent_per_dowrite_(4096),
         ssl_(NULL),
         last_read_time_(0)
         {}
 
@@ skipping 143 matching lines @@
 
     epoll_server_->RegisterFD(fd_, this, EPOLLIN | EPOLLOUT | EPOLLET);
 
     if (use_ssl) {
       ssl_ = spdy_new_ssl(ssl_state_->ssl_ctx);
       SSL_set_fd(ssl_, fd_);
       PrintSslError();
     }
   }
 
+  void CorkSocket() {
+    int state = 1;
+    int rv = setsockopt(fd_, IPPROTO_TCP, TCP_CORK, &state, sizeof(state));
+    if (rv < 0)
+      VLOG(1) << "setsockopt(CORK): " << errno;
+  }
+
+  void UncorkSocket() {
+    int state = 0;
+    int rv = setsockopt(fd_, IPPROTO_TCP, TCP_CORK, &state, sizeof(state));
+    if (rv < 0)
+      VLOG(1) << "setsockopt(CORK): " << errno;
+  }
+
   int Send(const char* data, int len, int flags) {
-    ssize_t bytes_written = 0;
+    int rv;
+    CorkSocket();

[fenix, 2011/01/27 20:34:59]

hopefully no ill effects to corking when it is already corked.

     if (ssl_) {
+      ssize_t bytes_written = 0;
       // Write smallish chunks to SSL so that we don't have large
       // multi-packet TLS records to receive before being able to handle
-      // the data.
+      // the data.  We don't have to be too careful here, because our data
+      // frames are already getting chunked appropriately, and those are
+      // the most likely "big" frames.
       while(len > 0) {
-        const int kMaxTLSRecordSize = 1460;
+        const int kMaxTLSRecordSize = 1500;
         const char* ptr = &(data[bytes_written]);
         int chunksize = std::min(len, kMaxTLSRecordSize);
-        int rv = SSL_write(ssl_, ptr, chunksize);
+        rv = SSL_write(ssl_, ptr, chunksize);
+        VLOG(2) << "SSLWrite(" << chunksize << " bytes): " << rv;
         if (rv <= 0) {
           switch(SSL_get_error(ssl_, rv)) {
             case SSL_ERROR_WANT_READ:
             case SSL_ERROR_WANT_WRITE:
             case SSL_ERROR_WANT_ACCEPT:
             case SSL_ERROR_WANT_CONNECT:
               rv = -2;
               break;
             default:
               PrintSslError();
               break;
           }
-          // If we wrote some data, return that count.  Otherwise
-          // return the stall error.
-          return bytes_written > 0 ? bytes_written : rv;
+          break;
         }
         bytes_written += rv;
         len -= rv;
         if (rv != chunksize)
           break;  // If we couldn't write everything, we're implicitly stalled
       }
-      if (!(flags & MSG_MORE)) {
-        int state = 0;
-        setsockopt( fd_, IPPROTO_TCP, TCP_CORK, &state, sizeof( state ) );
-        state = 1;
-        setsockopt( fd_, IPPROTO_TCP, TCP_CORK, &state, sizeof( state ) );
-      }
+      // If we wrote some data, return that count.  Otherwise
+      // return the stall error.
+      if (bytes_written > 0)
+        rv = bytes_written;
     } else {
-      bytes_written = send(fd_, data, len, flags);
+      rv = send(fd_, data, len, flags);
     }
-    return bytes_written;
+    if (!(flags & MSG_MORE))
+      UncorkSocket();
+    return rv;
   }
 
   // the following are from the EpollCallbackInterface
   virtual void OnRegistration(EpollServer* eps, int fd, int event_mask) {
     registered_in_epoll_server_ = true;
   }
   virtual void OnModification(int fd, int event_mask) { }
   virtual void OnEvent(int fd, EpollEvent* event) {
     events_ |= event->in_events;
     HandleEvents();
@@ skipping 149 matching lines @@
           VLOG(2) << log_prefix_ << ACCEPTOR_CLIENT_IDENT
                   << (sm_spdy_interface_ ? "Creating" : "Reusing")
                   << " SPDY interface.";
           if (!sm_spdy_interface_)
             sm_spdy_interface_ = NewSpdySM(this, NULL, epoll_server_,
                                            memory_cache_, acceptor_);
           sm_interface_ = sm_spdy_interface_;
         }
         break;
     }
+
+    CorkSocket();
     if (!sm_interface_->PostAcceptHook())
       return false;
 
     return true;
   }
 
   bool DoRead() {
     VLOG(2) << log_prefix_ << ACCEPTOR_CLIENT_IDENT << "DoRead()";
     while (!read_buffer_.Full()) {
       char* bytes;
@@ skipping 150 matching lines @@
       if (sm_interface_ && output_list_.size() < 2) {
         sm_interface_->GetOutput();
       }
       DataFrame* data_frame = output_list_.front();
       const char*  bytes = data_frame->data;
       int size = data_frame->size;
       bytes += data_frame->index;
       size -= data_frame->index;
       DCHECK_GE(size, 0);
       if (size <= 0) {
-        // Empty data frame. Indicates end of data from client.
-        // Uncork the socket.
-        int state = 0;
-        VLOG(2) << log_prefix_ << "Empty data frame, uncorking socket.";
-        setsockopt( fd_, IPPROTO_TCP, TCP_CORK, &state, sizeof( state ) );
         output_list_.pop_front();
         delete data_frame;
         continue;
       }
 
       flags = MSG_NOSIGNAL | MSG_DONTWAIT;
+      // Look for a queue size > 1 because |this| frame is remains on the list
+      // until it has finished sending.
       if (output_list_.size() > 1) {
         VLOG(2) << log_prefix_ << "Outlist size: " << output_list_.size()
                 << ": Adding MSG_MORE flag";
         flags |= MSG_MORE;
       }
       VLOG(2) << log_prefix_ << "Attempting to send " << size << " bytes.";
       ssize_t bytes_written = Send(bytes, size, flags);
       int stored_errno = errno;
       if (bytes_written == -1) {
         switch (stored_errno) {
@@ skipping 21 matching lines @@
       } else if (bytes_written == -2) {
         // -2 handles SSL_ERROR_WANT_* errors
         events_ &= ~EPOLLOUT;
         goto done;
       }
       VLOG(1) << log_prefix_ << ACCEPTOR_CLIENT_IDENT
               << "0 bytes written with send call.";
       goto error_or_close;
     }
    done:
+    UncorkSocket();
     return true;
 
    error_or_close:
     VLOG(1) << log_prefix_ << ACCEPTOR_CLIENT_IDENT
             << "DoWrite: error_or_close. Returning false "
             << "after cleaning up";
     Cleanup("DoWrite");
+    UncorkSocket();
     return false;
   }
 
   friend ostream& operator<<(ostream& os, const SMConnection& c) {
     os << &c << "\n";
     return os;
   }
 
   void Reset() {
     VLOG(2) << log_prefix_ << ACCEPTOR_CLIENT_IDENT << "Resetting";
@@ skipping 515 matching lines @@
 
   // SMInterface's Cleanup is currently only called by SMConnection after a
   // protocol message as been fully read. Spdy's SMInterface does not need
   // to do any cleanup at this time.
   // TODO (klindsay) This method is probably not being used properly and
   // some logic review and method renaming is probably in order.
   void Cleanup() {}
 
   // Send a settings frame
   int PostAcceptHook() {
-    ssize_t bytes_written;
     spdy::SpdySettings settings;
-    spdy::SettingsFlagsAndId settings_id(0);
-    settings_id.set_id(spdy::SETTINGS_MAX_CONCURRENT_STREAMS);
+    spdy::SettingsFlagsAndId settings_id(spdy::SETTINGS_MAX_CONCURRENT_STREAMS);
     settings.push_back(spdy::SpdySetting(settings_id, 100));
-    scoped_ptr<SpdySettingsControlFrame>
-      settings_frame(spdy_framer_->CreateSettings(settings));
+    SpdySettingsControlFrame* settings_frame =
+        spdy_framer_->CreateSettings(settings);
 
-    char* bytes = settings_frame->data();
-    size_t size = SpdyFrame::size() + settings_frame->length();
     VLOG(1) << ACCEPTOR_CLIENT_IDENT << "Sending Settings Frame";
-    bytes_written = connection_->Send(bytes, size,
-                                      MSG_NOSIGNAL | MSG_DONTWAIT);
-    if (static_cast<size_t>(bytes_written) != size) {
-      LOG(ERROR) << "Trouble sending SETTINGS frame! (" << errno << ")";
-      if (errno == EAGAIN) {
-        return 0;
-      }
-    }
+    SpdyFrameDataFrame* df = new SpdyFrameDataFrame;
+    df->size = settings_frame->length() + SpdyFrame::size();
+    df->data = settings_frame->data();
+    df->frame = settings_frame;
+    df->delete_when_done = true;
+    EnqueueDataFrame(df);
     return 1;
   }
 
   void AddAssociatedContent(FileData* file_data) {
     for (unsigned int i = 0; i < file_data->related_files.size(); ++i) {
       pair<int, string>& related_file = file_data->related_files[i];
       MemCacheIter mci;
       string filename  = "GET_";
       filename += related_file.second;
       if (!memory_cache_->AssignFileData(filename, &mci)) {
@@ skipping 1427 matching lines @@
       }
       break;
     }
     usleep(1000*10);  // 10 ms
   }
 
   unlink(PIDFILE);
   close(pidfile_fd);
   return 0;
 }