THRIFT-2789 TNonblockingServer leaks socket FD's under load

Client: C++
Patch: Sergey <drigh@deviantart.com> and Qiao Mu <qiaomuf@gmail.com>

1 files changed, 10 insertions, 1 deletions

diff --git a/lib/cpp/src/thrift/server/TNonblockingServer.cpp b/lib/cpp/src/thrift/server/TNonblockingServer.cpp
index 1cfdef8fa..9833f10e3 100644
--- a/lib/cpp/src/thrift/server/TNonblockingServer.cpp
+++ b/lib/cpp/src/thrift/server/TNonblockingServer.cpp
@@ -283,6 +283,7 @@ public:
   void forceClose() {
     appState_ = APP_CLOSE_CONNECTION;
     if (!notifyIOThread()) {
+      close();
       throw TException("TConnection::forceClose: failed write on notify pipe");
     }
   }
@@ -342,6 +343,8 @@ public:
 
     // Signal completion back to the libevent thread via a pipe
     if (!connection_->notifyIOThread()) {
+      GlobalOutput.printf("TNonblockingServer: failed to notifyIOThread, closing.");
+      connection_->close();
       throw TException("TNonblockingServer::Task::run: failed write on notify pipe");
     }
   }
@@ -568,6 +571,9 @@ void TNonblockingServer::TConnection::transition() {
         // The ThreadManager is not ready to handle any more tasks (it's probably shutting down).
         GlobalOutput.printf("IllegalStateException: Server::process() %s", ise.what());
         close();
+      } catch (TimedOutException& to) {
+        GlobalOutput.printf("[ERROR] TimedOutException: Server::process() %s", to.what());
+        close();
       }
 
       // Set this connection idle so that libevent doesn't process more
@@ -969,7 +975,10 @@ void TNonblockingServer::handleEvent(THRIFT_SOCKET fd, short which) {
     if (clientConnection->getIOThreadNumber() == 0) {
       clientConnection->transition();
     } else {
-      clientConnection->notifyIOThread();
+      if (!clientConnection->notifyIOThread()) {
+        GlobalOutput.perror("[ERROR] notifyIOThread failed on fresh connection, closing", errno);
+        returnConnection(clientConnection);
+      }
     }
 
     // addrLen is written by the accept() call, so needs to be set before the next call.