SERVER-38994 step down on SIGTERM

Check to see if we've entered shutdown from a shutdown command.  If not,
and if the replication machinery is up, attempt a shutdown in the style
of a default shutdown command.

(cherry picked from commit 9f5b13ee93e7eaeafa97ebd1d2d24c66b93cc974)

14 files changed, 194 insertions, 22 deletions

diff --git a/jstests/replsets/election_handoff_via_signal.js b/jstests/replsets/election_handoff_via_signal.js
new file mode 100644
index 00000000000..4bc58c95d40
--- /dev/null
+++ b/jstests/replsets/election_handoff_via_signal.js
@@ -0,0 +1,25 @@
+/**
+ * This is a basic test that checks that, when election handoff is enabled, a primary that is sent a
+ * non-terminal signal sends a ReplSetStepUp request to an eligible candidate.
+ */
+
+(function() {
+    "use strict";
+    load("jstests/replsets/libs/election_handoff.js");
+
+    const testName = "election_handoff_via_signal";
+    const numNodes = 3;
+    const rst = ReplSetTest({name: testName, nodes: numNodes});
+    const nodes = rst.nodeList();
+    rst.startSet();
+
+    // Make sure there are no election timeouts firing for the duration of the test. This helps
+    // ensure that the test will only pass if the election handoff succeeds.
+    const config = rst.getReplSetConfig();
+    config.settings = {"electionTimeoutMillis": 12 * 60 * 60 * 1000};
+    rst.initiate(config);
+
+    ElectionHandoffTest.testElectionHandoff(rst, 0, 1, {stepDownBySignal: true});
+
+    rst.stopSet();
+})();
diff --git a/jstests/replsets/libs/election_handoff.js b/jstests/replsets/libs/election_handoff.js
index 6bb903c1b2c..b81aca51363 100644
--- a/jstests/replsets/libs/election_handoff.js
+++ b/jstests/replsets/libs/election_handoff.js
@@ -10,13 +10,19 @@ var ElectionHandoffTest = (function() {
     load("jstests/replsets/rslib.js");
 
     const kStepDownPeriodSecs = 30;
+    const kSIGTERM = 15;
 
     /**
      * Exercises and validates an election handoff scenario by stepping down the primary and
      * ensuring that the node at "expectedCandidateId" is stepped up in its place. The desired
      * configuration of the replica set is passed in as its ReplSetTest instance.
+     *
+     * The options parameter contains extra options for the handoff.  Currently supported options
+     * are:
+     *   stepDownBySignal - When this option is set, the primary will be stepped down by stopping
+     *                      and restarting with sigterm, rather than with a replSetStepDown command
      */
-    function testElectionHandoff(rst, initialPrimaryId, expectedCandidateId) {
+    function testElectionHandoff(rst, initialPrimaryId, expectedCandidateId, options = {}) {
         const config = rst.getReplSetConfigFromNode();
         const numNodes = config.members.length;
         const memberInfo = config.members[expectedCandidateId];
@@ -50,10 +56,15 @@ var ElectionHandoffTest = (function() {
         rst.awaitNodesAgreeOnAppliedOpTime();
 
         // Step down the current primary.
-        assert.adminCommandWorkedAllowingNetworkError(primary, {
-            replSetStepDown: kStepDownPeriodSecs,
-            secondaryCatchUpPeriodSecs: kStepDownPeriodSecs / 2
-        });
+        if (options["stepDownBySignal"]) {
+            rst.stop(initialPrimaryId, kSIGTERM, {}, {forRestart: true});
+            rst.start(initialPrimaryId, {}, true);
+        } else {
+            assert.adminCommandWorkedAllowingNetworkError(primary, {
+                replSetStepDown: kStepDownPeriodSecs,
+                secondaryCatchUpPeriodSecs: kStepDownPeriodSecs / 2
+            });
+        }
 
         jsTestLog(`Checking that the secondary with id ${expectedCandidateId} is stepped up...`);
 
@@ -77,4 +88,4 @@ var ElectionHandoffTest = (function() {
 
     return {testElectionHandoff: testElectionHandoff, stepDownPeriodSecs: kStepDownPeriodSecs};
 
-})();
\ No newline at end of file
+})();
diff --git a/src/mongo/db/commands/generic_servers.cpp b/src/mongo/db/commands/generic_servers.cpp
index 7ae401f0846..93ad3389461 100644
--- a/src/mongo/db/commands/generic_servers.cpp
+++ b/src/mongo/db/commands/generic_servers.cpp
@@ -331,6 +331,9 @@ void CmdShutdown::addRequiredPrivileges(const std::string& dbname,
 }
 
 void CmdShutdown::shutdownHelper(const BSONObj& cmdObj) {
+    ShutdownTaskArgs shutdownArgs;
+    shutdownArgs.isUserInitiated = true;
+
     MONGO_FAIL_POINT_BLOCK(crashOnShutdown, crashBlock) {
         const std::string crashHow = crashBlock.getData()["how"].str();
         if (crashHow == "fault") {
@@ -344,7 +347,7 @@ void CmdShutdown::shutdownHelper(const BSONObj& cmdObj) {
 #if defined(_WIN32)
     // Signal the ServiceMain thread to shutdown.
     if (ntservice::shouldStartService()) {
-        shutdownNoTerminate();
+        shutdownNoTerminate(shutdownArgs);
 
         // Client expects us to abruptly close the socket as part of exiting
         // so this function is not allowed to return.
@@ -354,7 +357,7 @@ void CmdShutdown::shutdownHelper(const BSONObj& cmdObj) {
     } else
 #endif
     {
-        exitCleanly(EXIT_CLEAN);  // this never returns
+        shutdown(EXIT_CLEAN, shutdownArgs);  // this never returns
     }
 }
 
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 83e64191759..8824c3f0940 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -867,12 +867,39 @@ MONGO_INITIALIZER_GENERAL(setSSLManagerType, MONGO_NO_PREREQUISITES, ("SSLManage
 
 // NOTE: This function may be called at any time after registerShutdownTask is called below. It
 // must not depend on the prior execution of mongo initializers or the existence of threads.
-void shutdownTask() {
+void shutdownTask(const ShutdownTaskArgs& shutdownArgs) {
     Client::initThreadIfNotAlready();
 
     auto const client = Client::getCurrent();
     auto const serviceContext = client->getServiceContext();
 
+    // If we don't have shutdownArgs, we're shutting down from a signal, or other clean shutdown
+    // path.
+    //
+    // In that case, do a default step down, still shutting down if stepDown fails.
+    {
+        auto replCoord = repl::ReplicationCoordinator::get(serviceContext);
+        if (replCoord && !shutdownArgs.isUserInitiated) {
+            replCoord->enterTerminalShutdown();
+            ServiceContext::UniqueOperationContext uniqueOpCtx;
+            OperationContext* opCtx = client->getOperationContext();
+            if (!opCtx) {
+                uniqueOpCtx = client->makeOperationContext();
+                opCtx = uniqueOpCtx.get();
+            }
+
+            try {
+                uassertStatusOK(
+                    replCoord->stepDown(opCtx, false /* force */, Seconds(10), Seconds(120)));
+            } catch (const ExceptionFor<ErrorCodes::NotMaster>&) {
+                // ignore not master errors
+            } catch (const DBException& e) {
+                log() << "Failed to stepDown in non-command initiated shutdown path "
+                      << e.toString();
+            }
+        }
+    }
+
     // Terminate the balancer thread so it doesn't leak memory.
     if (auto balancer = Balancer::get(serviceContext)) {
         balancer->interruptBalancer();
diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h
index a09f61c6120..55d70436847 100644
--- a/src/mongo/db/repl/replication_coordinator.h
+++ b/src/mongo/db/repl/replication_coordinator.h
@@ -114,6 +114,15 @@ public:
     virtual void startup(OperationContext* opCtx) = 0;
 
     /**
+     * Start terminal shutdown.  This causes the topology coordinator to refuse to vote in any
+     * further elections.  This should only be called from global shutdown after we've passed the
+     * point of no return.
+     *
+     * This should be called once we are sure to call shutdown().
+     */
+    virtual void enterTerminalShutdown() = 0;
+
+    /**
      * Does whatever cleanup is required to stop replication, including instructing the other
      * components of the replication system to shut down and stop any threads they are using,
      * blocking until all replication-related shutdown tasks are complete.
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index f2ddcd13ac5..024ed39bd0b 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -827,6 +827,11 @@ void ReplicationCoordinatorImpl::startup(OperationContext* opCtx) {
     }
 }
 
+void ReplicationCoordinatorImpl::enterTerminalShutdown() {
+    stdx::lock_guard<stdx::mutex> lk(_mutex);
+    _inTerminalShutdown = true;
+}
+
 void ReplicationCoordinatorImpl::shutdown(OperationContext* opCtx) {
     // Shutdown must:
     // * prevent new threads from blocking in awaitReplication
@@ -3370,6 +3375,13 @@ Status ReplicationCoordinatorImpl::processReplSetRequestVotes(
 
     {
         stdx::lock_guard<stdx::mutex> lk(_mutex);
+
+        // We should only enter terminal shutdown from global terminal exit.  In that case, rather
+        // than voting in a term we don't plan to stay alive in, refuse to vote.
+        if (_inTerminalShutdown) {
+            return Status(ErrorCodes::ShutdownInProgress, "In the process of shutting down");
+        }
+
         _topCoord->processReplSetRequestVotes(args, response);
     }
 
diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h
index b5315e9ba8d..bd1d4a26f86 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.h
+++ b/src/mongo/db/repl/replication_coordinator_impl.h
@@ -104,6 +104,8 @@ public:
 
     virtual void startup(OperationContext* opCtx) override;
 
+    virtual void enterTerminalShutdown() override;
+
     virtual void shutdown(OperationContext* opCtx) override;
 
     virtual const ReplSettings& getSettings() const override;
@@ -1350,6 +1352,9 @@ private:
     // When we decide to step down due to hearing about a higher term, we remember the term we heard
     // here so we can update our term to match as part of finishing stepdown.
     boost::optional<long long> _pendingTermUpdateDuringStepDown;  // (M)
+
+    // If we're in terminal shutdown.  If true, we'll refuse to vote in elections.
+    bool _inTerminalShutdown = false;  // (M)
 };
 
 }  // namespace repl
diff --git a/src/mongo/db/repl/replication_coordinator_impl_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_test.cpp
index 8b583073c2e..dc1d39bb82a 100644
--- a/src/mongo/db/repl/replication_coordinator_impl_test.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl_test.cpp
@@ -5998,6 +5998,58 @@ TEST_F(ReplCoordTest, NodeFailsVoteRequestIfItFailsToStoreLastVote) {
     ASSERT_EQUALS(lastVote.getCandidateIndex(), 0);
 }
 
+TEST_F(ReplCoordTest, NodeNodesNotGrantVoteIfInTerminalShutdown) {
+    // Set up a 2-node replica set config.
+    assertStartSuccess(BSON("_id"
+                            << "mySet"
+                            << "version"
+                            << 2
+                            << "members"
+                            << BSON_ARRAY(BSON("host"
+                                               << "node1:12345"
+                                               << "_id"
+                                               << 0)
+                                          << BSON("host"
+                                                  << "node2:12345"
+                                                  << "_id"
+                                                  << 1))),
+                       HostAndPort("node1", 12345));
+    auto time = OpTimeWithTermOne(100, 1);
+    ASSERT_OK(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY));
+    getReplCoord()->setMyLastAppliedOpTime(time);
+    getReplCoord()->setMyLastDurableOpTime(time);
+    simulateSuccessfulV1Election();
+
+    // Get our current term, as primary.
+    ASSERT(getReplCoord()->getMemberState().primary());
+    auto initTerm = getReplCoord()->getTerm();
+
+    auto opCtx = makeOperationContext();
+
+    ReplSetRequestVotesArgs args;
+    ASSERT_OK(args.initialize(BSON("replSetRequestVotes" << 1 << "setName"
+                                                         << "mySet"
+                                                         << "term"
+                                                         << initTerm + 1  // term of new candidate.
+                                                         << "candidateIndex"
+                                                         << 1LL
+                                                         << "configVersion"
+                                                         << 2LL
+                                                         << "dryRun"
+                                                         << false
+                                                         << "lastCommittedOp"
+                                                         << time.asOpTime().toBSON())));
+    ReplSetRequestVotesResponse response;
+
+    getReplCoord()->enterTerminalShutdown();
+
+    auto r = getReplCoord()->processReplSetRequestVotes(opCtx.get(), args, &response);
+
+    ASSERT_NOT_OK(r);
+    ASSERT_EQUALS("In the process of shutting down", r.reason());
+    ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, r.code());
+}
+
 // TODO(schwerin): Unit test election id updating
 }  // namespace
 }  // namespace repl
diff --git a/src/mongo/db/repl/replication_coordinator_mock.cpp b/src/mongo/db/repl/replication_coordinator_mock.cpp
index e49bc521cd1..56717fe4ff0 100644
--- a/src/mongo/db/repl/replication_coordinator_mock.cpp
+++ b/src/mongo/db/repl/replication_coordinator_mock.cpp
@@ -71,6 +71,10 @@ void ReplicationCoordinatorMock::startup(OperationContext* opCtx) {
     // TODO
 }
 
+void ReplicationCoordinatorMock::enterTerminalShutdown() {
+    // TODO
+}
+
 void ReplicationCoordinatorMock::shutdown(OperationContext*) {
     // TODO
 }
diff --git a/src/mongo/db/repl/replication_coordinator_mock.h b/src/mongo/db/repl/replication_coordinator_mock.h
index 4c92a83f72b..a936e81c739 100644
--- a/src/mongo/db/repl/replication_coordinator_mock.h
+++ b/src/mongo/db/repl/replication_coordinator_mock.h
@@ -64,6 +64,8 @@ public:
 
     virtual void startup(OperationContext* opCtx);
 
+    virtual void enterTerminalShutdown();
+
     virtual void shutdown(OperationContext* opCtx);
 
     virtual void appendDiagnosticBSON(BSONObjBuilder* bob) override {}
diff --git a/src/mongo/embedded/replication_coordinator_embedded.cpp b/src/mongo/embedded/replication_coordinator_embedded.cpp
index 9f7fb99cfe4..62a2eb36d0d 100644
--- a/src/mongo/embedded/replication_coordinator_embedded.cpp
+++ b/src/mongo/embedded/replication_coordinator_embedded.cpp
@@ -50,6 +50,8 @@ ReplicationCoordinatorEmbedded::~ReplicationCoordinatorEmbedded() = default;
 
 void ReplicationCoordinatorEmbedded::startup(OperationContext* opCtx) {}
 
+void ReplicationCoordinatorEmbedded::enterTerminalShutdown() {}
+
 void ReplicationCoordinatorEmbedded::shutdown(OperationContext* opCtx) {}
 
 const ReplSettings& ReplicationCoordinatorEmbedded::getSettings() const {
diff --git a/src/mongo/embedded/replication_coordinator_embedded.h b/src/mongo/embedded/replication_coordinator_embedded.h
index f4f3cf2afd8..679db98dd9c 100644
--- a/src/mongo/embedded/replication_coordinator_embedded.h
+++ b/src/mongo/embedded/replication_coordinator_embedded.h
@@ -48,6 +48,8 @@ public:
 
     void startup(OperationContext* opCtx) override;
 
+    void enterTerminalShutdown() override;
+
     void shutdown(OperationContext* opCtx) override;
 
     // Returns the ServiceContext where this instance runs.
diff --git a/src/mongo/util/exit.cpp b/src/mongo/util/exit.cpp
index 3269cec3b39..fc874c0eea4 100644
--- a/src/mongo/util/exit.cpp
+++ b/src/mongo/util/exit.cpp
@@ -35,10 +35,10 @@
 #include "mongo/util/exit.h"
 
 #include <boost/optional.hpp>
+#include <functional>
 #include <stack>
 
 #include "mongo/stdx/condition_variable.h"
-#include "mongo/stdx/functional.h"
 #include "mongo/stdx/mutex.h"
 #include "mongo/stdx/thread.h"
 #include "mongo/util/log.h"
@@ -53,14 +53,14 @@ stdx::condition_variable shutdownTasksComplete;
 boost::optional<ExitCode> shutdownExitCode;
 bool shutdownTasksInProgress = false;
 AtomicUInt32 shutdownFlag;
-std::stack<stdx::function<void()>> shutdownTasks;
+std::stack<std::function<void(const ShutdownTaskArgs&)>> shutdownTasks;
 stdx::thread::id shutdownTasksThreadId;
 
-void runTasks(decltype(shutdownTasks) tasks) {
+void runTasks(decltype(shutdownTasks) tasks, const ShutdownTaskArgs& shutdownArgs) {
     while (!tasks.empty()) {
         const auto& task = tasks.top();
         try {
-            task();
+            task(shutdownArgs);
         } catch (...) {
             std::terminate();
         }
@@ -97,13 +97,13 @@ ExitCode waitForShutdown() {
     return shutdownExitCode.get();
 }
 
-void registerShutdownTask(stdx::function<void()> task) {
+void registerShutdownTask(std::function<void(const ShutdownTaskArgs&)> task) {
     stdx::lock_guard<stdx::mutex> lock(shutdownMutex);
     invariant(!globalInShutdownDeprecated());
     shutdownTasks.emplace(std::move(task));
 }
 
-void shutdown(ExitCode code) {
+void shutdown(ExitCode code, const ShutdownTaskArgs& shutdownArgs) {
     decltype(shutdownTasks) localTasks;
 
     {
@@ -139,7 +139,7 @@ void shutdown(ExitCode code) {
         localTasks.swap(shutdownTasks);
     }
 
-    runTasks(std::move(localTasks));
+    runTasks(std::move(localTasks), shutdownArgs);
 
     {
         stdx::lock_guard<stdx::mutex> lock(shutdownMutex);
@@ -151,7 +151,7 @@ void shutdown(ExitCode code) {
     }
 }
 
-void shutdownNoTerminate() {
+void shutdownNoTerminate(const ShutdownTaskArgs& shutdownArgs) {
     decltype(shutdownTasks) localTasks;
 
     {
@@ -167,7 +167,7 @@ void shutdownNoTerminate() {
         localTasks.swap(shutdownTasks);
     }
 
-    runTasks(std::move(localTasks));
+    runTasks(std::move(localTasks), shutdownArgs);
 
     {
         stdx::lock_guard<stdx::mutex> lock(shutdownMutex);
diff --git a/src/mongo/util/exit.h b/src/mongo/util/exit.h
index b7ca4482663..a6e51aa9ec3 100644
--- a/src/mongo/util/exit.h
+++ b/src/mongo/util/exit.h
@@ -30,13 +30,24 @@
 
 #pragma once
 
+#include <boost/optional.hpp>
+#include <functional>
+
 #include "mongo/platform/compiler.h"
-#include "mongo/stdx/functional.h"
 #include "mongo/util/exit_code.h"
 
 namespace mongo {
 
 /**
+ * ShutdownTaskArgs holds any arguments we might like to pass from a manual invocation of the
+ * shutdown command.  It is meant to give a default shutdown when default constructed.
+ */
+struct ShutdownTaskArgs {
+    // This should be set to true if we called shutdown from the shutdown command
+    bool isUserInitiated = false;
+};
+
+/**
  * Determines if the shutdown flag is set.
  *
  * Calling this function is deprecated because modules that consult it
@@ -58,7 +69,14 @@ ExitCode waitForShutdown();
  * shutdown or shutdownNoTerminate has been called, std::terminate is
  * called.
  */
-void registerShutdownTask(stdx::function<void()>);
+void registerShutdownTask(std::function<void(const ShutdownTaskArgs& shutdownArgs)>);
+
+/**
+ * For shutdown tasks that don't care to distinguish if they're called from command shutdown
+ */
+inline void registerShutdownTask(std::function<void()> task) {
+    registerShutdownTask([task = std::move(task)](const ShutdownTaskArgs&) { task(); });
+}
 
 /**
  * Toggles the shutdown flag to 'true', runs registered shutdown
@@ -67,7 +85,7 @@ void registerShutdownTask(stdx::function<void()>);
  * shutdown tasks. It is illegal to reenter this function from a
  * registered shutdown task. The function does not return.
  */
-MONGO_COMPILER_NORETURN void shutdown(ExitCode code);
+MONGO_COMPILER_NORETURN void shutdown(ExitCode code, const ShutdownTaskArgs& shutdownArgs = {});
 
 /**
  * Toggles the shutdown flag to 'true' and runs the registered
@@ -76,7 +94,7 @@ MONGO_COMPILER_NORETURN void shutdown(ExitCode code);
  * callers return immediately. It is legal to call shutdownNoTerminate
  * from a shutdown task.
  */
-void shutdownNoTerminate();
+void shutdownNoTerminate(const ShutdownTaskArgs& shutdownArgs = {});
 
 /** An alias for 'shutdown'. */
 MONGO_COMPILER_NORETURN inline void exitCleanly(ExitCode code) {