SERVER-28530 Prevent SSLThreadInfo destruction construction cycle

(cherry picked from commit 169f0f1178347f367d56b8ef8ec608b11ca71307)

2 files changed, 35 insertions, 51 deletions

diff --git a/buildscripts/cpplint.py b/buildscripts/cpplint.py
index 2f3440c0652..acf0a0af505 100755
--- a/buildscripts/cpplint.py
+++ b/buildscripts/cpplint.py
@@ -1639,7 +1639,7 @@ def make_polyfill_regex():
     'shared_lock,',
     'shared_mutex',
     'shared_timed_mutex',
-    'this_thread',
+    'this_thread(?!::at_thread_exit)',
     'thread',
     'timed_mutex',
     'try_to_lock',
diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp
index 21a40a22fad..da8af7e23e1 100644
--- a/src/mongo/util/net/ssl_manager.cpp
+++ b/src/mongo/util/net/ssl_manager.cpp
@@ -34,7 +34,7 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/thread/recursive_mutex.hpp>
-#include <boost/thread/tss.hpp>
+#include <boost/thread/thread.hpp>
 #include <iostream>
 #include <sstream>
 #include <string>
@@ -47,6 +47,7 @@
 #include "mongo/platform/atomic_word.h"
 #include "mongo/stdx/memory.h"
 #include "mongo/util/concurrency/mutex.h"
+#include "mongo/util/concurrency/threadlocal.h"
 #include "mongo/util/debug_util.h"
 #include "mongo/util/exit.h"
 #include "mongo/util/log.h"
@@ -128,31 +129,32 @@ IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(ASN1_SEQUENCE_ANY, ASN1_SET_ANY, ASN
  * Multithreaded Support for SSL.
  *
  * In order to allow OpenSSL to work in a multithreaded environment, you
- * must provide some callbacks for it to use for locking.  The following code
- * sets up a vector of mutexes and uses thread-local storage to assign an id
- * to each thread.
+ * may need to provide some callbacks for it to use for locking. The following code
+ * sets up a vector of mutexes and provides a thread unique ID number.
  * The so-called SSLThreadInfo class encapsulates most of the logic required for
  * OpenSSL multithreaded support.
+ *
+ * OpenSSL before version 1.1.0 requires applications provide a callback which emits a thread
+ * identifier. This ID is used to store thread specific ERR information. When a thread is
+ * terminated, it must call ERR_remove_state or ERR_remove_thread_state. These functions may
+ * themselves invoke the application provided callback.
  */
-
-unsigned long _ssl_id_callback();
-void _ssl_locking_callback(int mode, int type, const char* file, int line);
-
 class SSLThreadInfo {
 public:
-    SSLThreadInfo() {
-        _id = _next.fetchAndAdd(1);
-    }
-
-    ~SSLThreadInfo() {
-        ERR_remove_state(0);
-    }
+    static unsigned long getID() {
+        enforceCleanupOnShutdown();
 
-    unsigned long id() const {
-        return _id;
+#ifdef _WIN32
+        return GetCurrentThreadId();
+#else
+        static_assert(sizeof(void*) == sizeof(unsigned long),
+                      "OpenSSL needs the address of a thread-unique object to be castable to"
+                      "unsigned long");
+        return reinterpret_cast<unsigned long>(&errno);
+#endif
     }
 
-    void lock_callback(int mode, int type, const char* file, int line) {
+    static void lockingCallback(int mode, int type, const char* file, int line) {
         if (mode & CRYPTO_LOCK) {
             _mutex[type]->lock();
         } else {
@@ -161,38 +163,32 @@ public:
     }
 
     static void init() {
+        CRYPTO_set_id_callback(&SSLThreadInfo::getID);
+        CRYPTO_set_locking_callback(&SSLThreadInfo::lockingCallback);
+
         while ((int)_mutex.size() < CRYPTO_num_locks()) {
             _mutex.emplace_back(stdx::make_unique<stdx::recursive_mutex>());
         }
     }
 
-    static SSLThreadInfo* get() {
-        SSLThreadInfo* me = _thread.get();
-        if (!me) {
-            me = new SSLThreadInfo();
-            _thread.reset(me);
+private:
+    SSLThreadInfo() = delete;
+
+    // When called, ensures that this thread will, on termination, call ERR_remove_state.
+    static void enforceCleanupOnShutdown() {
+        static MONGO_TRIVIALLY_CONSTRUCTIBLE_THREAD_LOCAL bool firstCall = true;
+        if (firstCall) {
+            boost::this_thread::at_thread_exit([] { ERR_remove_state(0); });
+            firstCall = false;
         }
-        return me;
     }
 
-private:
-    unsigned _id;
-
-    static AtomicUInt32 _next;
     // Note: see SERVER-8734 for why we are using a recursive mutex here.
     // Once the deadlock fix in OpenSSL is incorporated into most distros of
     // Linux, this can be changed back to a nonrecursive mutex.
     static std::vector<std::unique_ptr<stdx::recursive_mutex>> _mutex;
-    static boost::thread_specific_ptr<SSLThreadInfo> _thread;
 };
-
-unsigned long _ssl_id_callback() {
-    return SSLThreadInfo::get()->id();
-}
-
-void _ssl_locking_callback(int mode, int type, const char* file, int line) {
-    SSLThreadInfo::get()->lock_callback(mode, type, file, line);
-}
+std::vector<std::unique_ptr<stdx::recursive_mutex>> SSLThreadInfo::_mutex;
 
 // We only want to free SSL_CTX objects if they have been populated. OpenSSL seems to perform this
 // check before freeing them, but because it does not document this, we should protect ourselves.
@@ -202,10 +198,6 @@ void _free_ssl_context(SSL_CTX* ctx) {
     }
 }
 
-AtomicUInt32 SSLThreadInfo::_next;
-std::vector<std::unique_ptr<stdx::recursive_mutex>> SSLThreadInfo::_mutex;
-boost::thread_specific_ptr<SSLThreadInfo> SSLThreadInfo::_thread;
-
 ////////////////////////////////////////////////////////////////
 
 SimpleMutex sslManagerMtx;
@@ -384,12 +376,8 @@ MONGO_INITIALIZER(SetupOpenSSL)(InitializerContext*) {
     // so that encryption/decryption is backwards compatible
     OpenSSL_add_all_algorithms();
 
-    // Setup OpenSSL multithreading callbacks
-    CRYPTO_set_id_callback(_ssl_id_callback);
-    CRYPTO_set_locking_callback(_ssl_locking_callback);
-
+    // Setup OpenSSL multithreading callbacks and mutexes
     SSLThreadInfo::init();
-    SSLThreadInfo::get();
 
     return Status::OK();
 }
@@ -435,10 +423,6 @@ std::string getCertificateSubjectName(X509* cert) {
 
 SSLConnection::SSLConnection(SSL_CTX* context, Socket* sock, const char* initialBytes, int len)
     : socket(sock) {
-    // This just ensures that SSL multithreading support is set up for this thread,
-    // if it's not already.
-    SSLThreadInfo::get();
-
     ssl = SSL_new(context);
 
     std::string sslErr =