Bug #29579 Clients using SSL can hang the server

Added an option to yassl to allow "quiet shutdown" like openssl does. This option causes the SSL libs to NOT perform the close_notify handshake during shutdown. This fixes a hang we experience because we hold a lock during socket shutdown.


mysql-test/t/ssl_big.test:
  BitKeeper file /Users/dkatz/50/mysql-test/t/ssl_big.test
mysql-test/r/ssl-big.result:
  BitKeeper file /Users/dkatz/50/mysql-test/r/ssl-big.result
client/mysqltest.c:
  Added new command to mysqltest to send a quit command to the server, but to not close the actual socket on our end.
  
  Also changed code to reuse connection slots, so that the tests can open and close sockets in a loop.
extra/yassl/include/openssl/ssl.h:
  Added C accessors to the quietShutdown option.
extra/yassl/include/yassl_int.hpp:
  Added quietShutdown_ member and accessor methods to the SSL class.
extra/yassl/src/ssl.cpp:
  Added accessors to get/set the quietShutdown option and to not perform the shutdown handshake if quietShutdown is set.
extra/yassl/src/yassl_int.cpp:
  Added quietShutdown_ member and accessor methods to the SSL class.
vio/viossl.c:
  Added line to set the quiet_shutdown option before shutting down the socket.
mysql-test/t/ssl-big.test:
  Added a test that causes an unpatched server to hang during SSL socket shutdown.

4 files changed, 34 insertions, 3 deletions

diff --git a/extra/yassl/include/openssl/ssl.h b/extra/yassl/include/openssl/ssl.h
index 7dd33e3fcad..efd0dec75b6 100644
--- a/extra/yassl/include/openssl/ssl.h
+++ b/extra/yassl/include/openssl/ssl.h
@@ -277,6 +277,8 @@ int  SSL_session_reused(SSL*);
 int  SSL_set_rfd(SSL*, int);
 int  SSL_set_wfd(SSL*, int);
 void SSL_set_shutdown(SSL*, int);
+void SSL_set_quiet_shutdown(SSL *ssl,int mode);
+int SSL_get_quiet_shutdown(SSL *ssl);
 
 int SSL_want_read(SSL*);
 int SSL_want_write(SSL*);
diff --git a/extra/yassl/include/yassl_int.hpp b/extra/yassl/include/yassl_int.hpp
index 94cb85c3300..b207f0bffbd 100644
--- a/extra/yassl/include/yassl_int.hpp
+++ b/extra/yassl/include/yassl_int.hpp
@@ -584,6 +584,7 @@ class SSL {
     Socket              socket_;                // socket wrapper
     Buffers             buffers_;               // buffered handshakes and data
     Log                 log_;                   // logger
+    bool                quietShutdown_;
 
     // optimization variables
     bool                has_data_;              // buffered data ready?
@@ -610,6 +611,7 @@ public:
     Buffers&   useBuffers();
 
     bool       HasData() const;
+    bool       GetQuietShutdown() const;
 
     // sets
     void set_pending(Cipher suite);
@@ -621,6 +623,7 @@ public:
     void SetError(YasslError);
     int  SetCompression();
     void UnSetCompression();
+    void SetQuietShutdown(bool mode);
 
     // helpers
     bool isTLS() const;
diff --git a/extra/yassl/src/ssl.cpp b/extra/yassl/src/ssl.cpp
index 86dfa1c6ebd..c3d580a93ab 100644
--- a/extra/yassl/src/ssl.cpp
+++ b/extra/yassl/src/ssl.cpp
@@ -411,8 +411,10 @@ int SSL_clear(SSL* ssl)
 
 int SSL_shutdown(SSL* ssl)
 {
-    Alert alert(warning, close_notify);
-    sendAlert(*ssl, alert);
+    if (!ssl->GetQuietShutdown()) {
+      Alert alert(warning, close_notify);
+      sendAlert(*ssl, alert);
+    }
     ssl->useLog().ShowTCP(ssl->getSocket().get_fd(), true);
 
     GetErrors().Remove();
@@ -421,6 +423,18 @@ int SSL_shutdown(SSL* ssl)
 }
 
 
+void SSL_set_quiet_shutdown(SSL *ssl,int mode)
+{
+    ssl->SetQuietShutdown(mode != 0);
+}
+
+
+int SSL_get_quiet_shutdown(SSL *ssl)
+{
+    return ssl->GetQuietShutdown();
+}
+
+
 /* on by default but allow user to turn off */
 long SSL_CTX_set_session_cache_mode(SSL_CTX* ctx, long mode)
 {
diff --git a/extra/yassl/src/yassl_int.cpp b/extra/yassl/src/yassl_int.cpp
index ae16abf9e49..ba4678d70b9 100644
--- a/extra/yassl/src/yassl_int.cpp
+++ b/extra/yassl/src/yassl_int.cpp
@@ -291,7 +291,7 @@ const ClientKeyFactory& sslFactory::getClientKey() const
 SSL::SSL(SSL_CTX* ctx) 
     : secure_(ctx->getMethod()->getVersion(), crypto_.use_random(),
               ctx->getMethod()->getSide(), ctx->GetCiphers(), ctx,
-              ctx->GetDH_Parms().set_), has_data_(false)
+              ctx->GetDH_Parms().set_), has_data_(false), quietShutdown_(false)
 {
     if (int err = crypto_.get_random().GetError()) {
         SetError(YasslError(err));
@@ -773,6 +773,12 @@ void SSL::SetError(YasslError ye)
     // TODO: add string here
 }
 
+// set the quiet shutdown mode (close_nofiy not sent or received on shutdown)
+void SSL::SetQuietShutdown(bool mode)
+{
+  quietShutdown_ = mode;
+}
+
 
 Buffers& SSL::useBuffers()
 {
@@ -1330,6 +1336,12 @@ YasslError SSL::GetError() const
 }
 
 
+bool SSL::GetQuietShutdown() const
+{
+    return quietShutdown_;
+}
+
+
 bool SSL::GetMultiProtocol() const
 {
     return secure_.GetContext()->getMethod()->multipleProtocol();