SERVER-13429 Use logging system in mongod signal handlers, test exit logging.

Still not really safe for signal handlers, but no worse than before,
and correctly logs to the log file/syslog, so an improvement.

(cherry picked from commit f4e6aa9ee7c0dccbb4789d3dff1d92b086d67b88)

3 files changed, 100 insertions, 27 deletions

diff --git a/jstests/noPassthrough/exit_logging.js b/jstests/noPassthrough/exit_logging.js
new file mode 100644
index 00000000000..89615b34e8c
--- /dev/null
+++ b/jstests/noPassthrough/exit_logging.js
@@ -0,0 +1,57 @@
+/**
+ * Tests that various forms of normal and abnormal shutdown write to the log files as expected.
+ */
+
+(function () {
+
+    function makeShutdownByCrashFn(crashHow) {
+        return function (conn) {
+            var admin = conn.getDB("admin");
+            assert.commandWorked(admin.runCommand({
+                configureFailPoint: "crashOnShutdown",
+                mode: "alwaysOn",
+                data: { how: crashHow }
+            }));
+            admin.shutdownServer();
+        }
+    }
+
+    function makeRegExMatchFn(pattern) {
+        return function (text) {
+            assert(pattern.test(text),
+                   "Log contents did not match " + pattern);
+        }
+    }
+
+    function testShutdownLogging(crashFn, matchFn) {
+        var logFileName = MongoRunner.dataPath + "mongod.log";
+        var opts = { logpath: logFileName, nojournal: "" };
+        var conn = MongoRunner.runMongod(opts);
+        try {
+            crashFn(conn);
+        }
+        finally {
+            MongoRunner.stopMongod(conn);
+        }
+        var logContents = cat(logFileName);
+        matchFn(logContents);
+    }
+
+    if (_isWindows()) {
+        print("SKIPPING TEST ON WINDOWS");
+        return;
+    }
+
+    // testShutdownLogging(
+    //     function (conn) { conn.getDB('admin').shutdownServer() },
+    //     makeRegExMatchFn(/shutdown command received[\s\S]*dbexit: really exiting now/));
+
+    testShutdownLogging(
+        makeShutdownByCrashFn('fault'),
+        makeRegExMatchFn(/Invalid access at address[\s\S]*abruptQuitWithAddrSignal/));
+
+    testShutdownLogging(
+        makeShutdownByCrashFn('abort'),
+        makeRegExMatchFn(/Got signal[\s\S]*abruptQuit[^W]/));
+
+}());
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 1b68642561f..d9bd9907b82 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -1004,38 +1004,36 @@ namespace mongo {
 
 namespace mongo {
 
-    void abruptQuit(int x) {
-        ostringstream ossSig;
-        ossSig << "Got signal: " << x << " (" << strsignal( x ) << ")." << endl;
-        rawOut( ossSig.str() );
-
-        /*
-        ostringstream ossOp;
-        ossOp << "Last op: " << currentOp.infoNoauth() << endl;
-        rawOut( ossOp.str() );
-        */
-
-        ostringstream oss;
-        oss << "Backtrace:" << endl;
-        printStackTrace( oss );
-        rawOut( oss.str() );
+    void abruptQuit(int signalNum) {
+        {
+            logger::LogstreamBuilder logBuilder(logger::globalLogDomain(),
+                                                getThreadName(),
+                                                logger::LogSeverity::Severe());
+            logBuilder.stream() <<
+                "Got signal: " << signalNum << " (" << strsignal(signalNum) << ").\nBacktrace:";
+            printStackTrace(logBuilder.stream());
+        }
 
         // Don't go through normal shutdown procedure. It may make things worse.
         ::_exit(EXIT_ABRUPT);
-
     }
 
-    void abruptQuitWithAddrSignal( int signal, siginfo_t *siginfo, void * ) {
-        ostringstream oss;
-        oss << "Invalid";
-        if ( signal == SIGSEGV || signal == SIGBUS ) {
-            oss << " access";
-        } else {
-            oss << " operation";
+    void abruptQuitWithAddrSignal( int signalNum, siginfo_t *siginfo, void * ) {
+        {
+            logger::LogstreamBuilder logBuilder(logger::globalLogDomain(),
+                                                getThreadName(),
+                                                logger::LogSeverity::Severe());
+
+            std::ostream& oss = logBuilder.stream();
+            oss << "Invalid";
+            if ( signalNum == SIGSEGV || signalNum == SIGBUS ) {
+                oss << " access";
+            } else {
+                oss << " operation";
+            }
+            oss << " at address: " << siginfo->si_addr;
         }
-        oss << " at address: " << siginfo->si_addr << " from thread: " << getThreadName() << endl;
-        rawOut( oss.str() );
-        abruptQuit( signal );
+        abruptQuit( signalNum );
     }
 
     sigset_t asyncSignals;
@@ -1052,6 +1050,9 @@ namespace mongo {
                 fassert(16782, rotateLogs());
                 logProcessDetailsForLogRotate();
                 break;
+            case SIGQUIT:
+                log() << "Received SIGQUIT; terminating.";
+                _exit(EXIT_ABRUPT);
             default:
                 // interrupt/terminate signal
                 Client::initThread( "signalProcessingThread" );
@@ -1095,7 +1096,6 @@ namespace mongo {
         verify( sigaction(SIGFPE, &addrSignals, 0) == 0 );
 
         verify( signal(SIGABRT, abruptQuit) != SIG_ERR );
-        verify( signal(SIGQUIT, abruptQuit) != SIG_ERR );
         verify( signal(SIGPIPE, SIG_IGN) != SIG_ERR );
 
         setupSIGTRAPforGDB();
@@ -1106,6 +1106,7 @@ namespace mongo {
         sigaddset( &asyncSignals, SIGHUP );
         sigaddset( &asyncSignals, SIGINT );
         sigaddset( &asyncSignals, SIGTERM );
+        sigaddset( &asyncSignals, SIGQUIT );
         sigaddset( &asyncSignals, SIGUSR1 );
         sigaddset( &asyncSignals, SIGXCPU );
 
diff --git a/src/mongo/db/dbcommands_generic.cpp b/src/mongo/db/dbcommands_generic.cpp
index 937721e3581..0767952e826 100644
--- a/src/mongo/db/dbcommands_generic.cpp
+++ b/src/mongo/db/dbcommands_generic.cpp
@@ -55,6 +55,8 @@
 #include "mongo/db/stats/counters.h"
 #include "mongo/scripting/engine.h"
 #include "mongo/server.h"
+#include "mongo/util/fail_point.h"
+#include "mongo/util/fail_point_service.h"
 #include "mongo/util/lruishmap.h"
 #include "mongo/util/md5.hpp"
 #include "mongo/util/processinfo.h"
@@ -288,7 +290,20 @@ namespace mongo {
 
     } listCommandsCmd;
 
+    namespace {
+        MONGO_FP_DECLARE(crashOnShutdown);
+
+        int* volatile illegalAddress;
+    }  // namespace
+
     bool CmdShutdown::shutdownHelper() {
+        MONGO_FAIL_POINT_BLOCK(crashOnShutdown, crashBlock) {
+            const std::string crashHow = crashBlock.getData()["how"].str();
+            if (crashHow == "fault") {
+                ++*illegalAddress;
+            }
+            ::abort();
+        }
         Client * c = currentClient.get();
         if ( c ) {
             c->shutdown();