SERVER-23103 Unify exit handling

31 files changed, 333 insertions, 365 deletions

diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 3ffe57a0b08..e8af00437ed 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -92,6 +92,7 @@ baseSource=[
     'util/base64.cpp',
     'util/concurrency/thread_name.cpp',
     'util/exception_filter_win32.cpp',
+    'util/exit.cpp',
     'util/hex.cpp',
     'util/itoa.cpp',
     'util/log.cpp',
diff --git a/src/mongo/client/SConscript b/src/mongo/client/SConscript
index 71a65c735a2..bfd559b7ed4 100644
--- a/src/mongo/client/SConscript
+++ b/src/mongo/client/SConscript
@@ -145,11 +145,6 @@ env.Library(
         'connection_string',
         'read_preference',
     ],
-    LIBDEPS_TAGS=[
-        # Depends on mongo::inShutdown
-        'incomplete'
-    ],
-
 )
 
 env.Library(
diff --git a/src/mongo/client/scoped_db_conn_test.cpp b/src/mongo/client/scoped_db_conn_test.cpp
index 342342895d8..122aff1da85 100644
--- a/src/mongo/client/scoped_db_conn_test.cpp
+++ b/src/mongo/client/scoped_db_conn_test.cpp
@@ -66,38 +66,10 @@ using std::vector;
 class Client;
 class OperationContext;
 
-namespace {
-
-stdx::mutex shutDownMutex;
-bool shuttingDown = false;
-
-}  // namespace
-
-// Symbols defined to build the binary correctly.
-bool inShutdown() {
-    stdx::lock_guard<stdx::mutex> sl(shutDownMutex);
-    return shuttingDown;
-}
-
-void signalShutdown() {}
-
 DBClientBase* createDirectClient(OperationContext* txn) {
     return NULL;
 }
 
-void dbexit(ExitCode rc, const char* why) {
-    {
-        stdx::lock_guard<stdx::mutex> sl(shutDownMutex);
-        shuttingDown = true;
-    }
-
-    quickExit(rc);
-}
-
-void exitCleanly(ExitCode rc) {
-    dbexit(rc, "");
-}
-
 namespace {
 
 const string TARGET_HOST = "localhost:27017";
@@ -169,11 +141,6 @@ public:
         MessageServer::Options options;
         options.port = _port;
 
-        {
-            stdx::lock_guard<stdx::mutex> sl(shutDownMutex);
-            shuttingDown = false;
-        }
-
         _server.reset(createServer(options, std::move(messsageHandler)));
         _serverThread = stdx::thread(runServer, _server.get());
     }
@@ -186,11 +153,6 @@ public:
             return;
         }
 
-        {
-            stdx::lock_guard<stdx::mutex> sl(shutDownMutex);
-            shuttingDown = true;
-        }
-
         ListeningSockets::get()->closeAll();
         _serverThread.join();
 
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index daf898821e1..c7ae1d3ac1c 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -364,10 +364,6 @@ env.Library(
         '$BUILD_DIR/mongo/util/concurrency/synchronization',
         'range_arithmetic',
     ],
-    LIBDEPS_TAGS=[
-        # Needs mongo::inShutdown
-        'incomplete',
-    ]
 )
 
 env.CppUnitTest(
diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript
index f8935e8d588..13355f2a049 100644
--- a/src/mongo/db/auth/SConscript
+++ b/src/mongo/db/auth/SConscript
@@ -143,10 +143,6 @@ env.Library('authmongos',
                 '$BUILD_DIR/mongo/s/catalog/dist_lock_manager',
                 '$BUILD_DIR/mongo/s/coreshard',
             ],
-            LIBDEPS_TAGS=[
-                # Depends on inShutdown
-                'incomplete',
-            ],
 )
 
 env.Library(
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index d36f661e414..84b9d5ab02d 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -75,7 +75,7 @@ env.Library(
         'server_status_core',
     ],
     LIBDEPS_TAGS=[
-        # Depends on mongo::exitCleanly.
+        # Depends on coreshard.
         'incomplete',
     ],
 )
diff --git a/src/mongo/db/commands/generic.cpp b/src/mongo/db/commands/generic.cpp
index 26746480db6..c2459534f4d 100644
--- a/src/mongo/db/commands/generic.cpp
+++ b/src/mongo/db/commands/generic.cpp
@@ -317,7 +317,7 @@ void CmdShutdown::shutdownHelper() {
 #if defined(_WIN32)
     // Signal the ServiceMain thread to shutdown.
     if (ntservice::shouldStartService()) {
-        signalShutdown();
+        shutdownNoTerminate();
 
         // Client expects us to abruptly close the socket as part of exiting
         // so this function is not allowed to return.
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 5ca96a91f11..aba223690b3 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -45,6 +45,7 @@
 #include "mongo/base/initializer.h"
 #include "mongo/base/status.h"
 #include "mongo/config.h"
+#include "mongo/db/audit.h"
 #include "mongo/db/auth/auth_index_d.h"
 #include "mongo/db/auth/authorization_manager.h"
 #include "mongo/db/auth/authorization_manager_global.h"
@@ -57,6 +58,7 @@
 #include "mongo/db/client.h"
 #include "mongo/db/clientcursor.h"
 #include "mongo/db/concurrency/d_concurrency.h"
+#include "mongo/db/concurrency/lock_state.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/dbdirectclient.h"
 #include "mongo/db/dbmessage.h"
@@ -84,6 +86,7 @@
 #include "mongo/db/repl/storage_interface_impl.h"
 #include "mongo/db/repl/topology_coordinator_impl.h"
 #include "mongo/db/restapi.h"
+#include "mongo/db/s/sharding_state.h"
 #include "mongo/db/s/sharding_state_recovery.h"
 #include "mongo/db/server_options.h"
 #include "mongo/db/server_parameters.h"
@@ -347,7 +350,7 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) {
             log() << "****";
             log() << "cannot do this upgrade without an upgrade in the middle";
             log() << "please do a --repair with 2.6 and then start this version";
-            dbexit(EXIT_NEED_UPGRADE);
+            quickExit(EXIT_NEED_UPGRADE);
             return;
         }
 
@@ -855,9 +858,95 @@ static void reportEventToSystemImpl(const char* msg) {
 }  // namespace mongo
 #endif  // if defined(_WIN32)
 
+static void shutdownServer() {
+    log(LogComponent::kNetwork) << "shutdown: going to close listening sockets..." << endl;
+    ListeningSockets::get()->closeAll();
+
+    log(LogComponent::kNetwork) << "shutdown: going to flush diaglog..." << endl;
+    _diaglog.flush();
+
+    /* must do this before unmapping mem or you may get a seg fault */
+    log(LogComponent::kNetwork) << "shutdown: going to close sockets..." << endl;
+    stdx::thread close_socket_thread(stdx::bind(MessagingPort::closeAllSockets, 0));
+    close_socket_thread.detach();
+
+    // We drop the scope cache because leak sanitizer can't see across the
+    // thread we use for proxying MozJS requests. Dropping the cache cleans up
+    // the memory and makes leak sanitizer happy.
+    ScriptEngine::dropScopeCache();
+
+    getGlobalServiceContext()->shutdownGlobalStorageEngineCleanly();
+}
+
+// NOTE: This function may be called at any time after
+// registerShutdownTask is called below. It must not depend on the
+// prior execution of mongo initializers or the existence of threads.
+static void shutdownTask() {
+    // Global storage engine may not be started in all cases before we exit
+    if (getGlobalServiceContext()->getGlobalStorageEngine() == NULL) {
+        return;
+    }
+
+    // Shutdown Full-Time Data Capture
+    stopFTDC();
+
+    getGlobalServiceContext()->setKillAllOperations();
+
+    repl::getGlobalReplicationCoordinator()->shutdown();
+
+    Client& client = cc();
+    ServiceContext::UniqueOperationContext uniqueTxn;
+    OperationContext* txn = client.getOperationContext();
+    if (!txn) {
+        uniqueTxn = client.makeOperationContext();
+        txn = uniqueTxn.get();
+    }
+
+    ShardingState::get(txn)->shutDown(txn);
+
+    // We should always be able to acquire the global lock at shutdown.
+    //
+    // TODO: This call chain uses the locker directly, because we do not want to start an
+    // operation context, which also instantiates a recovery unit. Also, using the
+    // lockGlobalBegin/lockGlobalComplete sequence, we avoid taking the flush lock. This will
+    // all go away if we start acquiring the global/flush lock as part of ScopedTransaction.
+    //
+    // For a Windows service, dbexit does not call exit(), so we must leak the lock outside
+    // of this function to prevent any operations from running that need a lock.
+    //
+    DefaultLockerImpl* globalLocker = new DefaultLockerImpl();
+    LockResult result = globalLocker->lockGlobalBegin(MODE_X);
+    if (result == LOCK_WAITING) {
+        result = globalLocker->lockGlobalComplete(UINT_MAX);
+    }
+
+    invariant(LOCK_OK == result);
+
+    log(LogComponent::kControl) << "now exiting" << endl;
+
+    // Execute the graceful shutdown tasks, such as flushing the outstanding journal
+    // and data files, close sockets, etc.
+    try {
+        shutdownServer();
+    } catch (const DBException& ex) {
+        severe() << "shutdown failed with DBException " << ex;
+        std::terminate();
+    } catch (const std::exception& ex) {
+        severe() << "shutdown failed with std::exception: " << ex.what();
+        std::terminate();
+    } catch (...) {
+        severe() << "shutdown failed with exception";
+        std::terminate();
+    }
+
+    audit::logShutdown(&cc());
+}
+
 static int mongoDbMain(int argc, char* argv[], char** envp) {
     static StaticObserver staticObserver;
 
+    registerShutdownTask(shutdownTask);
+
 #if defined(_WIN32)
     mongo::reportEventToSystem = &mongo::reportEventToSystemImpl;
 #endif
diff --git a/src/mongo/db/instance.cpp b/src/mongo/db/instance.cpp
index 96bcf7d64ad..59f13ce1957 100644
--- a/src/mongo/db/instance.cpp
+++ b/src/mongo/db/instance.cpp
@@ -1223,135 +1223,6 @@ void receivedInsert(OperationContext* txn, const NamespaceString& nsString, Mess
     _receivedInsert(txn, nsString, ns, multi, keepGoing, op, false);
 }
 
-static AtomicUInt32 shutdownInProgress(0);
-
-bool inShutdown() {
-    return shutdownInProgress.loadRelaxed() != 0;
-}
-
-bool inShutdownStrict() {
-    return shutdownInProgress.load() != 0;
-}
-
-static void shutdownServer() {
-    log(LogComponent::kNetwork) << "shutdown: going to close listening sockets..." << endl;
-    ListeningSockets::get()->closeAll();
-
-    log(LogComponent::kNetwork) << "shutdown: going to flush diaglog..." << endl;
-    _diaglog.flush();
-
-    /* must do this before unmapping mem or you may get a seg fault */
-    log(LogComponent::kNetwork) << "shutdown: going to close sockets..." << endl;
-    stdx::thread close_socket_thread(stdx::bind(MessagingPort::closeAllSockets, 0));
-    close_socket_thread.detach();
-
-    // We drop the scope cache because leak sanitizer can't see across the
-    // thread we use for proxying MozJS requests. Dropping the cache cleans up
-    // the memory and makes leak sanitizer happy.
-    ScriptEngine::dropScopeCache();
-
-    getGlobalServiceContext()->shutdownGlobalStorageEngineCleanly();
-}
-
-// shutdownLock
-//
-// Protects:
-//  Ensures shutdown is single threaded.
-// Lock Ordering:
-//  No restrictions
-stdx::mutex shutdownLock;
-
-void signalShutdown() {
-    // Notify all threads shutdown has started
-    shutdownInProgress.fetchAndAdd(1);
-}
-
-void exitCleanly(ExitCode code) {
-    // Notify all threads shutdown has started
-    shutdownInProgress.fetchAndAdd(1);
-
-    // Grab the shutdown lock to prevent concurrent callers
-    stdx::lock_guard<stdx::mutex> lockguard(shutdownLock);
-
-    // Shutdown Full-Time Data Capture
-    stopFTDC();
-
-    // Global storage engine may not be started in all cases before we exit
-    if (getGlobalServiceContext()->getGlobalStorageEngine() == NULL) {
-        dbexit(code);  // returns only under a windows service
-        invariant(code == EXIT_WINDOWS_SERVICE_STOP);
-        return;
-    }
-
-    getGlobalServiceContext()->setKillAllOperations();
-
-    repl::getGlobalReplicationCoordinator()->shutdown();
-
-    Client& client = cc();
-    ServiceContext::UniqueOperationContext uniqueTxn;
-    OperationContext* txn = client.getOperationContext();
-    if (!txn) {
-        uniqueTxn = client.makeOperationContext();
-        txn = uniqueTxn.get();
-    }
-
-    ShardingState::get(txn)->shutDown(txn);
-
-    // We should always be able to acquire the global lock at shutdown.
-    //
-    // TODO: This call chain uses the locker directly, because we do not want to start an
-    // operation context, which also instantiates a recovery unit. Also, using the
-    // lockGlobalBegin/lockGlobalComplete sequence, we avoid taking the flush lock. This will
-    // all go away if we start acquiring the global/flush lock as part of ScopedTransaction.
-    //
-    // For a Windows service, dbexit does not call exit(), so we must leak the lock outside
-    // of this function to prevent any operations from running that need a lock.
-    //
-    DefaultLockerImpl* globalLocker = new DefaultLockerImpl();
-    LockResult result = globalLocker->lockGlobalBegin(MODE_X);
-    if (result == LOCK_WAITING) {
-        result = globalLocker->lockGlobalComplete(UINT_MAX);
-    }
-
-    invariant(LOCK_OK == result);
-
-    log(LogComponent::kControl) << "now exiting" << endl;
-
-    // Execute the graceful shutdown tasks, such as flushing the outstanding journal
-    // and data files, close sockets, etc.
-    try {
-        shutdownServer();
-    } catch (const DBException& ex) {
-        severe() << "shutdown failed with DBException " << ex;
-        std::terminate();
-    } catch (const std::exception& ex) {
-        severe() << "shutdown failed with std::exception: " << ex.what();
-        std::terminate();
-    } catch (...) {
-        severe() << "shutdown failed with exception";
-        std::terminate();
-    }
-
-    dbexit(code);
-}
-
-NOINLINE_DECL void dbexit(ExitCode rc, const char* why) {
-    audit::logShutdown(&cc());
-
-    log(LogComponent::kControl) << "dbexit: " << why << " rc: " << rc;
-
-#ifdef _WIN32
-    // Windows Service Controller wants to be told when we are down,
-    //  so don't call quickExit() yet, or say "really exiting now"
-    //
-    if (rc == EXIT_WINDOWS_SERVICE_STOP) {
-        return;
-    }
-#endif
-
-    quickExit(rc);
-}
-
 // ----- BEGIN Diaglog -----
 DiagLog::DiagLog() : f(0), level(0) {}
 
diff --git a/src/mongo/db/repl/master_slave.cpp b/src/mongo/db/repl/master_slave.cpp
index c3a0cc260e9..dd60bd07c6f 100644
--- a/src/mongo/db/repl/master_slave.cpp
+++ b/src/mongo/db/repl/master_slave.cpp
@@ -71,6 +71,7 @@
 #include "mongo/stdx/thread.h"
 #include "mongo/util/exit.h"
 #include "mongo/util/log.h"
+#include "mongo/util/quick_exit.h"
 
 using std::cout;
 using std::endl;
@@ -282,14 +283,14 @@ void ReplSource::loadAll(OperationContext* txn, SourceVector& v) {
                 log() << "http://dochub.mongodb.org/core/masterslave" << endl;
                 log() << "terminating mongod after 30 seconds" << endl;
                 sleepsecs(30);
-                dbexit(EXIT_REPLICATION_ERROR);
+                quickExit(EXIT_REPLICATION_ERROR);
             }
             if (tmp.only != replSettings.getOnly()) {
                 log() << "--only " << replSettings.getOnly() << " != " << tmp.only
                       << " from local.sources collection" << endl;
                 log() << "terminating after 30 seconds" << endl;
                 sleepsecs(30);
-                dbexit(EXIT_REPLICATION_ERROR);
+                quickExit(EXIT_REPLICATION_ERROR);
             }
         }
         uassert(17065, "Internal error reading from local.sources", PlanExecutor::IS_EOF == state);
@@ -305,7 +306,7 @@ void ReplSource::loadAll(OperationContext* txn, SourceVector& v) {
         try {
             massert(10384, "--only requires use of --source", replSettings.getOnly().empty());
         } catch (...) {
-            dbexit(EXIT_BADOPTIONS);
+            quickExit(EXIT_BADOPTIONS);
         }
     }
 
diff --git a/src/mongo/dbtests/SConscript b/src/mongo/dbtests/SConscript
index a7dd427ed0e..37cf400ffba 100644
--- a/src/mongo/dbtests/SConscript
+++ b/src/mongo/dbtests/SConscript
@@ -33,7 +33,7 @@ env.Library(
         'framework_options',
     ],
     LIBDEPS_TAGS=[
-        # Depends on exitCleanly
+        # Depends on service_context_d.cpp which is in serveronly
         'incomplete',
     ],
 )
diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index a99be665ed3..2c4e07b605f 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -212,10 +212,6 @@ env.Library(
         'write_ops/cluster_write_op',
         'write_ops/cluster_write_op_conversion',
     ],
-    LIBDEPS_TAGS=[
-        # Depends on inShutdown
-        'incomplete',
-    ],
 )
 
 env.CppUnitTest(
diff --git a/src/mongo/s/query/SConscript b/src/mongo/s/query/SConscript
index 18022507aa2..afec3a3dd34 100644
--- a/src/mongo/s/query/SConscript
+++ b/src/mongo/s/query/SConscript
@@ -150,10 +150,6 @@ env.Library(
         "$BUILD_DIR/mongo/s/coreshard",
         "$BUILD_DIR/mongo/util/background_job",
     ],
-    LIBDEPS_TAGS=[
-        # Needs mongo::inShutDown()
-        'incomplete',
-    ],
 )
 
 env.CppUnitTest(
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp
index ae6b221c596..9b31f4796b5 100644
--- a/src/mongo/s/server.cpp
+++ b/src/mongo/s/server.cpp
@@ -108,30 +108,10 @@ ntservice::NtServiceDefaultStrings defaultServiceStrings = {
 static ExitCode initService();
 #endif
 
-static AtomicUInt32 shutdownInProgress(0);
-
-bool inShutdown() {
-    return shutdownInProgress.loadRelaxed() != 0;
-}
-void signalShutdown() {
-    // Notify all threads shutdown has started
-    shutdownInProgress.fetchAndAdd(1);
-}
-
-// shutdownLock
-//
-// Protects:
-//  Ensures shutdown is single threaded.
-// Lock Ordering:
-//  No restrictions
-stdx::mutex shutdownLock;
-
-void exitCleanly(ExitCode code) {
-    signalShutdown();
-
-    // Grab the shutdown lock to prevent concurrent callers
-    stdx::lock_guard<stdx::mutex> lockguard(shutdownLock);
-
+// NOTE: This function may be called at any time after
+// registerShutdownTask is called below. It must not depend on the
+// prior execution of mongo initializers or the existence of threads.
+static void cleanupTask() {
     {
         Client& client = cc();
         ServiceContext::UniqueOperationContext uniqueTxn;
@@ -147,24 +127,7 @@ void exitCleanly(ExitCode code) {
         grid.catalogManager(txn)->shutDown(txn);
     }
 
-    dbexit(code);
-}
-
-void dbexit(ExitCode rc, const char* why) {
     audit::logShutdown(ClientBasic::getCurrent());
-
-#if defined(_WIN32)
-    // Windows Service Controller wants to be told when we are done shutting down
-    // and call quickExit itself.
-    //
-    if (rc == EXIT_WINDOWS_SERVICE_STOP) {
-        log() << "dbexit: exiting because Windows service was stopped";
-        return;
-    }
-#endif
-
-    log() << "dbexit: " << why << " rc:" << rc;
-    quickExit(rc);
 }
 
 static BSONObj buildErrReply(const DBException& ex) {
@@ -283,7 +246,7 @@ static ExitCode runMongosServer() {
     DBClientReplicaSet::setAuthPooledSecondaryConn(false);
 
     if (getHostName().empty()) {
-        dbexit(EXIT_BADOPTIONS);
+        quickExit(EXIT_BADOPTIONS);
     }
 
     {
@@ -449,6 +412,8 @@ int mongoSMain(int argc, char* argv[], char** envp) {
     if (argc < 1)
         return EXIT_FAILURE;
 
+    registerShutdownTask(cleanupTask);
+
     setupSignalHandlers();
 
     Status status = mongo::runGlobalInitializers(argc, argv, envp);
diff --git a/src/mongo/shell/clientAndShell.cpp b/src/mongo/shell/clientAndShell.cpp
index abbb03be656..f1ecde12eeb 100644
--- a/src/mongo/shell/clientAndShell.cpp
+++ b/src/mongo/shell/clientAndShell.cpp
@@ -47,29 +47,6 @@ class Client;
 class DBClientBase;
 class OperationContext;
 
-
-bool dbexitCalled = false;
-
-void dbexit(ExitCode returnCode, const char* whyMsg) {
-    {
-        stdx::lock_guard<stdx::mutex> lk(shell_utils::mongoProgramOutputMutex);
-        dbexitCalled = true;
-    }
-
-    log() << "dbexit called" << endl;
-
-    if (whyMsg) {
-        log() << " b/c " << whyMsg << endl;
-    }
-
-    log() << "exiting" << endl;
-    quickExit(returnCode);
-}
-
-bool inShutdown() {
-    return dbexitCalled;
-}
-
 bool haveLocalShardingInfo(OperationContext* txn, const string& ns) {
     return false;
 }
diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp
index c064c292e83..1c45cb71f6e 100644
--- a/src/mongo/shell/dbshell.cpp
+++ b/src/mongo/shell/dbshell.cpp
@@ -54,7 +54,7 @@
 #include "mongo/shell/shell_options.h"
 #include "mongo/shell/shell_utils.h"
 #include "mongo/shell/shell_utils_launcher.h"
-#include "mongo/util/exit_code.h"
+#include "mongo/util/exit.h"
 #include "mongo/util/file.h"
 #include "mongo/util/log.h"
 #include "mongo/util/net/ssl_options.h"
@@ -88,8 +88,6 @@ static volatile bool atPrompt = false;  // can eval before getting to prompt
 namespace mongo {
 
 Scope* shellMainScope;
-
-extern bool dbexitCalled;
 }
 
 void generateCompletions(const string& prefix, vector<string>& all) {
@@ -179,21 +177,10 @@ void killOps() {
 // Stubs for signal_handlers.cpp
 namespace mongo {
 void logProcessDetailsForLogRotate() {}
-
-void exitCleanly(ExitCode code) {
-    {
-        stdx::lock_guard<stdx::mutex> lk(mongo::shell_utils::mongoProgramOutputMutex);
-        mongo::dbexitCalled = true;
-    }
-
-    ::killOps();
-    ::shellHistoryDone();
-    quickExit(0);
-}
 }
 
 void quitNicely(int sig) {
-    exitCleanly(EXIT_CLEAN);
+    shutdown(EXIT_CLEAN);
 }
 
 // the returned string is allocated with strdup() or malloc() and must be freed by calling free()
@@ -585,6 +572,14 @@ static void edit(const string& whatToEdit) {
 }
 
 int _main(int argc, char* argv[], char** envp) {
+    registerShutdownTask([] {
+        // NOTE: This function may be called at any time. It must not
+        // depend on the prior execution of mongo initializers or the
+        // existence of threads.
+        ::killOps();
+        ::shellHistoryDone();
+    });
+
     setupSignalHandlers();
     setupSignals();
 
@@ -905,10 +900,6 @@ int _main(int argc, char* argv[], char** envp) {
         shellHistoryDone();
     }
 
-    {
-        stdx::lock_guard<stdx::mutex> lk(mongo::shell_utils::mongoProgramOutputMutex);
-        mongo::dbexitCalled = true;
-    }
     return (lastLineSuccessful ? 0 : 1);
 }
 
diff --git a/src/mongo/shell/shell_utils_launcher.cpp b/src/mongo/shell/shell_utils_launcher.cpp
index beb80eff4a2..3fb2ec1489d 100644
--- a/src/mongo/shell/shell_utils_launcher.cpp
+++ b/src/mongo/shell/shell_utils_launcher.cpp
@@ -54,6 +54,7 @@
 #include "mongo/scripting/engine.h"
 #include "mongo/shell/shell_utils.h"
 #include "mongo/stdx/thread.h"
+#include "mongo/util/exit.h"
 #include "mongo/util/log.h"
 #include "mongo/util/net/hostandport.h"
 #include "mongo/util/quick_exit.h"
@@ -72,8 +73,6 @@ using std::string;
 using std::stringstream;
 using std::vector;
 
-extern bool dbexitCalled;
-
 #ifdef _WIN32
 inline int close(int fd) {
     return _close(fd);
@@ -192,14 +191,9 @@ void ProgramRegistry::insertHandleForPid(ProcessId pid, HANDLE handle) {
 
 ProgramRegistry& registry = *(new ProgramRegistry());
 
-void goingAwaySoon() {
-    stdx::lock_guard<stdx::mutex> lk(mongoProgramOutputMutex);
-    mongo::dbexitCalled = true;
-}
-
 void ProgramOutputMultiplexer::appendLine(int port, ProcessId pid, const char* line) {
     stdx::lock_guard<stdx::mutex> lk(mongoProgramOutputMutex);
-    uassert(28695, "program is terminating", !mongo::dbexitCalled);
+    uassert(28695, "program is terminating", !mongo::inShutdown());
     stringstream buf;
     string name = registry.programName(pid);
     if (port > 0)
@@ -342,7 +336,7 @@ void ProgramRunner::operator()() {
             }
             verify(lenToRead > 0);
             int ret = read(_pipe, (void*)start, lenToRead);
-            if (mongo::dbexitCalled)
+            if (mongo::inShutdown())
                 break;
             verify(ret != -1);
             start[ret] = '\0';
diff --git a/src/mongo/shell/shell_utils_launcher.h b/src/mongo/shell/shell_utils_launcher.h
index fd58c1bd614..8c4449aee41 100644
--- a/src/mongo/shell/shell_utils_launcher.h
+++ b/src/mongo/shell/shell_utils_launcher.h
@@ -54,7 +54,6 @@ struct MongoProgramScope {
 };
 void KillMongoProgramInstances();
 
-void goingAwaySoon();
 void installShellUtilsLauncher(Scope& scope);
 
 /** Record log lines from concurrent programs.  All public members are thread safe. */
diff --git a/src/mongo/tools/bridge.cpp b/src/mongo/tools/bridge.cpp
index 375473dee44..64ff17387e0 100644
--- a/src/mongo/tools/bridge.cpp
+++ b/src/mongo/tools/bridge.cpp
@@ -55,7 +55,7 @@
 #include "mongo/util/mongoutils/str.h"
 #include "mongo/util/net/listen.h"
 #include "mongo/util/net/message.h"
-#include "mongo/util/exit_code.h"
+#include "mongo/util/exit.h"
 #include "mongo/util/quick_exit.h"
 #include "mongo/util/static_observer.h"
 #include "mongo/util/signal_handlers.h"
@@ -334,13 +334,7 @@ public:
         t.detach();
     }
 
-    bool inShutdown() {
-        return _inShutdown.load();
-    }
-
     void shutdownAll() {
-        _inShutdown.store(true);
-
         stdx::lock_guard<stdx::mutex> lk(_portsMutex);
         for (auto mp : _ports) {
             mp->shutdown();
@@ -367,20 +361,19 @@ MONGO_INITIALIZER(SetGlobalEnvironment)(InitializerContext* context) {
 
 }  // namespace
 
-bool inShutdown() {
-    return listener->inShutdown();
-}
-
 void logProcessDetailsForLogRotate() {}
 
-void exitCleanly(ExitCode code) {
-    ListeningSockets::get()->closeAll();
-    listener->shutdownAll();
-    quickExit(code);
-}
-
 int bridgeMain(int argc, char** argv, char** envp) {
     static StaticObserver staticObserver;
+
+    registerShutdownTask([&] {
+        // NOTE: This function may be called at any time. It must not
+        // depend on the prior execution of mongo initializers or the
+        // existence of threads.
+        ListeningSockets::get()->closeAll();
+        listener->shutdownAll();
+    });
+
     setupSignalHandlers();
     runGlobalInitializersOrDie(argc, argv, envp);
     startSignalProcessingThread();
diff --git a/src/mongo/unittest/crutch.cpp b/src/mongo/unittest/crutch.cpp
index 61297a0f48c..628aa7b589c 100644
--- a/src/mongo/unittest/crutch.cpp
+++ b/src/mongo/unittest/crutch.cpp
@@ -43,10 +43,6 @@ namespace mongo {
 
 class Client;
 
-bool inShutdown() {
-    return false;
-}
-
 class OperationContext;
 
 DBClientBase* createDirectClient(OperationContext* txn) {
@@ -58,17 +54,7 @@ bool haveLocalShardingInfo(OperationContext* txn, const std::string& ns) {
     return false;
 }
 
-void dbexit(ExitCode rc, const char* why) {
-    invariant(!"unittests shouldn't call dbexit");
-}
-
-void exitCleanly(ExitCode code) {
-    invariant(!"unittests shouldn't call exitCleanly");
-}
-
 #ifdef _WIN32
-void signalShutdown() {}
-
 namespace ntservice {
 bool shouldStartService() {
     return false;
diff --git a/src/mongo/util/concurrency/SConscript b/src/mongo/util/concurrency/SConscript
index cea19bc91d5..3bc52f9acce 100644
--- a/src/mongo/util/concurrency/SConscript
+++ b/src/mongo/util/concurrency/SConscript
@@ -73,10 +73,6 @@ env.Library(
     LIBDEPS=[
         '$BUILD_DIR/mongo/util/background_job',
     ],
-    LIBDEPS_TAGS=[
-        # Depends on inShutdown
-        'incomplete',
-    ],
 )
 
 env.Library(
diff --git a/src/mongo/util/exit.cpp b/src/mongo/util/exit.cpp
new file mode 100644
index 00000000000..44cdf07f68b
--- /dev/null
+++ b/src/mongo/util/exit.cpp
@@ -0,0 +1,153 @@
+/**
+ *    Copyright (C) 2016 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kControl;
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/util/exit.h"
+
+#include <stack>
+
+#include "mongo/stdx/condition_variable.h"
+#include "mongo/stdx/functional.h"
+#include "mongo/stdx/mutex.h"
+#include "mongo/stdx/thread.h"
+#include "mongo/util/log.h"
+#include "mongo/util/quick_exit.h"
+
+namespace mongo {
+
+namespace {
+
+stdx::mutex shutdownMutex;
+stdx::condition_variable shutdownTasksComplete;
+bool shutdownTasksInProgress = false;
+AtomicUInt32 shutdownFlag;
+std::stack<stdx::function<void()>> shutdownTasks;
+stdx::thread::id shutdownTasksThreadId;
+
+void runTasks(decltype(shutdownTasks) tasks) {
+    while (!tasks.empty()) {
+        const auto& task = tasks.top();
+        try {
+            task();
+        } catch (...) {
+            std::terminate();
+        }
+        tasks.pop();
+    }
+}
+
+MONGO_COMPILER_NORETURN void logAndQuickExit(ExitCode code) {
+    log() << "shutting down with code:" << code;
+    quickExit(code);
+}
+
+}  // namespace
+
+bool inShutdown() {
+    return shutdownFlag.loadRelaxed() != 0;
+}
+
+bool inShutdownStrict() {
+    return shutdownFlag.load() != 0;
+}
+
+void registerShutdownTask(stdx::function<void()> task) {
+    stdx::lock_guard<stdx::mutex> lock(shutdownMutex);
+    invariant(!inShutdown());
+    shutdownTasks.emplace(std::move(task));
+}
+
+void shutdown(ExitCode code) {
+    decltype(shutdownTasks) localTasks;
+
+    {
+        stdx::unique_lock<stdx::mutex> lock(shutdownMutex);
+
+        if (shutdownTasksInProgress) {
+            // Someone better have called shutdown in some form already.
+            invariant(inShutdown());
+
+            // Re-entrant calls to shutdown are not allowed.
+            invariant(shutdownTasksThreadId != stdx::this_thread::get_id());
+
+            // Wait for the shutdown tasks to complete
+            while (shutdownTasksInProgress)
+                shutdownTasksComplete.wait(lock);
+
+            logAndQuickExit(code);
+        }
+
+        shutdownFlag.fetchAndAdd(1);
+        shutdownTasksInProgress = true;
+        shutdownTasksThreadId = stdx::this_thread::get_id();
+
+        localTasks.swap(shutdownTasks);
+    }
+
+    runTasks(std::move(localTasks));
+
+    {
+        stdx::lock_guard<stdx::mutex> lock(shutdownMutex);
+        shutdownTasksInProgress = false;
+    }
+
+    shutdownTasksComplete.notify_all();
+
+    logAndQuickExit(code);
+}
+
+void shutdownNoTerminate() {
+    decltype(shutdownTasks) localTasks;
+
+    {
+        stdx::lock_guard<stdx::mutex> lock(shutdownMutex);
+
+        if (inShutdown())
+            return;
+
+        shutdownFlag.fetchAndAdd(1);
+        shutdownTasksInProgress = true;
+        shutdownTasksThreadId = stdx::this_thread::get_id();
+
+        localTasks.swap(shutdownTasks);
+    }
+
+    runTasks(std::move(localTasks));
+
+    {
+        stdx::lock_guard<stdx::mutex> lock(shutdownMutex);
+        shutdownTasksInProgress = false;
+    }
+
+    shutdownTasksComplete.notify_all();
+}
+
+}  // namespace mongo
diff --git a/src/mongo/util/exit.h b/src/mongo/util/exit.h
index bb7c2de9adb..fa6a8bbb85b 100644
--- a/src/mongo/util/exit.h
+++ b/src/mongo/util/exit.h
@@ -28,13 +28,12 @@
 
 #pragma once
 
+#include "mongo/platform/compiler.h"
+#include "mongo/stdx/functional.h"
 #include "mongo/util/exit_code.h"
 
 namespace mongo {
 
-// Note: whyMsg can never be NULL.
-void dbexit(ExitCode returnCode, const char* whyMsg = "");
-
 /**
  * Quickly determines if the shutdown flag is set.  May not be definitive.
  */
@@ -45,4 +44,36 @@ bool inShutdown();
  * than inShutdown().
  */
 bool inShutdownStrict();
+
+/**
+ * Registers a new shutdown task to be called when shutdown or
+ * shutdownNoTerminate is called. If this function is invoked after
+ * shutdown or shutdownNoTerminate has been called, std::terminate is
+ * called.
+ */
+void registerShutdownTask(stdx::function<void()>);
+
+/**
+ * Toggles the shutdown flag to 'true', runs registered shutdown
+ * tasks, and then exits with the given code. It is safe to call this
+ * function from multiple threads, only the first caller executes
+ * shutdown tasks. It is illegal to reenter this function from a
+ * registered shutdown task. The function does not return.
+ */
+MONGO_COMPILER_NORETURN void shutdown(ExitCode code);
+
+/**
+ * Toggles the shutdown flag to 'true' and runs the registered
+ * shutdown tasks. It is safe to call this function from multiple
+ * threads, only the first caller executes shutdown tasks, subsequent
+ * callers return immediately. It is legal to call shutdownNoTerminate
+ * from a shutdown task.
+ */
+void shutdownNoTerminate();
+
+/** An alias for 'shutdown'. */
+MONGO_COMPILER_NORETURN inline void exitCleanly(ExitCode code) {
+    shutdown(code);
 }
+
+}  // namespace mongo
diff --git a/src/mongo/util/exit_code.h b/src/mongo/util/exit_code.h
index 209e27a27a6..c8020684a83 100644
--- a/src/mongo/util/exit_code.h
+++ b/src/mongo/util/exit_code.h
@@ -56,20 +56,4 @@ enum ExitCode : int {
     EXIT_TEST = 101
 };
 
-/**
- * Exit the current executable doing whatever cleanup is necessary.
- * Defined differently in different executables.
- * No database locks must be held by the thread when this function is called.
- */
-void exitCleanly(ExitCode code);
-
-/**
- * Signal main or  ServiceMain thread to exit
- * Important for the ServiceMain thread to do the exit when mongod/s are running as NT Services
- * on Windows.
- * It is not required to be called before exitCleanly in the general case, only for
- * proper NT Service shutdown.
- */
-void signalShutdown();
-
 }  // namespace mongo
diff --git a/src/mongo/util/net/SConscript b/src/mongo/util/net/SConscript
index af96061029f..b50629c10b5 100644
--- a/src/mongo/util/net/SConscript
+++ b/src/mongo/util/net/SConscript
@@ -45,10 +45,6 @@ env.Library(
         '$BUILD_DIR/mongo/util/options_parser/options_parser',
         'hostandport',
     ],
-    LIBDEPS_TAGS=[
-        # Depends on inShutdown
-        'incomplete',
-    ],
 )
 
 env.CppUnitTest(
@@ -104,10 +100,6 @@ env.Library(
         'network',
         '$BUILD_DIR/mongo/db/stats/counters',
     ],
-    LIBDEPS_TAGS=[
-        # Depends on inShutdown and dbexit
-        'incomplete',
-    ],
 )
 
 env.Library(
diff --git a/src/mongo/util/net/listen_test.cpp b/src/mongo/util/net/listen_test.cpp
index 098cb7c851a..db732bd57da 100644
--- a/src/mongo/util/net/listen_test.cpp
+++ b/src/mongo/util/net/listen_test.cpp
@@ -32,16 +32,12 @@
 #include "mongo/stdx/thread.h"
 #include "mongo/unittest/unittest.h"
 #include "mongo/util/log.h"
+#include "mongo/util/exit.h"
 #include "mongo/util/net/listen.h"
 #include "mongo/util/time_support.h"
 
 namespace mongo {
 
-static AtomicUInt32 myShutdownInProgress(0);
-bool inShutdown() {
-    return myShutdownInProgress.loadRelaxed() != 0;
-}
-
 namespace {
 
 const long long kSleepMillis = 5000;
@@ -61,7 +57,7 @@ TEST(Listener, ElapsedTimeCheck) {
     log() << "Listener elapsed time: " << listenDelta << std::endl;
     log() << "Clock elapsed time:    " << clockDelta << std::endl;
     ASSERT_APPROX_EQUAL(listenDelta, clockDelta, kEpsilon);
-    myShutdownInProgress.store(1);
+    shutdownNoTerminate();
     t.join();
 }
 
diff --git a/src/mongo/util/net/message_server_port.cpp b/src/mongo/util/net/message_server_port.cpp
index af230519ca7..fc61454203d 100644
--- a/src/mongo/util/net/message_server_port.cpp
+++ b/src/mongo/util/net/message_server_port.cpp
@@ -53,6 +53,7 @@
 #include "mongo/util/net/message_port.h"
 #include "mongo/util/net/message_server.h"
 #include "mongo/util/net/ssl_manager.h"
+#include "mongo/util/quick_exit.h"
 #include "mongo/util/scopeguard.h"
 
 #ifdef __linux__  // TODO: consider making this ifndef _WIN32
@@ -243,7 +244,7 @@ private:
             log() << "DBException handling request, closing client connection: " << e << endl;
         } catch (std::exception& e) {
             error() << "Uncaught std::exception: " << e.what() << ", terminating" << endl;
-            dbexit(EXIT_UNCAUGHT);
+            quickExit(EXIT_UNCAUGHT);
         }
         portWithHandler->shutdown();
 
diff --git a/src/mongo/util/ntservice.cpp b/src/mongo/util/ntservice.cpp
index 02f03b7e989..3bec1d69bf2 100644
--- a/src/mongo/util/ntservice.cpp
+++ b/src/mongo/util/ntservice.cpp
@@ -581,7 +581,7 @@ static void serviceShutdown(const char* controlCodeName) {
 
     // Note: This triggers _serviceCallback, ie  ServiceMain,
     // to stop by setting inShutdown() == true
-    signalShutdown();
+    shutdownNoTerminate();
 
     // Note: we will report exit status in initService
 }
diff --git a/src/mongo/util/ntservice_test.cpp b/src/mongo/util/ntservice_test.cpp
index e23cef12299..40ca8b05117 100644
--- a/src/mongo/util/ntservice_test.cpp
+++ b/src/mongo/util/ntservice_test.cpp
@@ -124,9 +124,4 @@ TEST(NtService, RegressionSERVER_7252) {
 namespace mongo {
 void Client::initThread(const char* desc, AbstractMessagingPort* mp) {}
 void removeControlCHandler() {}
-void signalShutdown() {}
-bool inShutdown() {
-    return false;
-}
-void exitCleanly(ExitCode code) {}
 }  // namespace mongo
diff --git a/src/mongo/util/quick_exit.cpp b/src/mongo/util/quick_exit.cpp
index d9e35e13528..e53c99420a4 100644
--- a/src/mongo/util/quick_exit.cpp
+++ b/src/mongo/util/quick_exit.cpp
@@ -39,6 +39,9 @@
 // This will probably get us _exit on non-unistd platforms like Windows.
 #include <cstdlib>
 
+// NOTE: Header only dependencies are OK in this library.
+#include "mongo/stdx/mutex.h"
+
 #if !defined(__has_feature)
 #define __has_feature(x) 0
 #endif
@@ -65,7 +68,16 @@ extern "C" void __gcov_flush();
 
 namespace mongo {
 
+namespace {
+stdx::mutex* const quickExitMutex = new stdx::mutex;
+}  // namespace
+
 void quickExit(int code) {
+    // Ensure that only one thread invokes the last rites here. No
+    // RAII here - we never want to unlock this.
+    if (quickExitMutex)
+        quickExitMutex->lock();
+
 #ifdef MONGO_GCOV
     __gcov_flush();
 #endif
diff --git a/src/mongo/util/signal_handlers.cpp b/src/mongo/util/signal_handlers.cpp
index 2b0683fbc63..14dd8ddd5e4 100644
--- a/src/mongo/util/signal_handlers.cpp
+++ b/src/mongo/util/signal_handlers.cpp
@@ -44,7 +44,7 @@
 #include "mongo/platform/process_id.h"
 #include "mongo/stdx/thread.h"
 #include "mongo/util/assert_util.h"
-#include "mongo/util/exit_code.h"
+#include "mongo/util/exit.h"
 #include "mongo/util/log.h"
 #include "mongo/util/quick_exit.h"
 #include "mongo/util/scopeguard.h"