diff options
author | Jason Carey <jcarey@argv.me> | 2019-03-06 13:53:03 -0500 |
---|---|---|
committer | Jason Carey <jcarey@argv.me> | 2019-03-25 11:38:31 -0400 |
commit | 6b5def2ef0f4c798c67a04f390e81aa9f3bb9415 (patch) | |
tree | cfb110e73558dd61500904bb52d77c6db07a67de | |
parent | 941a6dc4a709ea3014786c3c5effe1f92a4acc23 (diff) | |
download | mongo-6b5def2ef0f4c798c67a04f390e81aa9f3bb9415.tar.gz |
SERVER-38994 step down on SIGTERM
Check to see if we've entered shutdown from a shutdown command. If not,
and if the replication machinery is up, attempt a shutdown in the style
of a default shutdown command.
(cherry picked from commit 9f5b13ee93e7eaeafa97ebd1d2d24c66b93cc974)
-rw-r--r-- | jstests/replsets/election_handoff_via_signal.js | 25 | ||||
-rw-r--r-- | jstests/replsets/libs/election_handoff.js | 23 | ||||
-rw-r--r-- | src/mongo/db/commands/generic_servers.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/db.cpp | 29 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator.h | 9 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl.cpp | 12 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl.h | 5 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_test.cpp | 52 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_mock.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_mock.h | 2 | ||||
-rw-r--r-- | src/mongo/embedded/replication_coordinator_embedded.cpp | 2 | ||||
-rw-r--r-- | src/mongo/embedded/replication_coordinator_embedded.h | 2 | ||||
-rw-r--r-- | src/mongo/util/exit.cpp | 18 | ||||
-rw-r--r-- | src/mongo/util/exit.h | 26 |
14 files changed, 194 insertions, 22 deletions
diff --git a/jstests/replsets/election_handoff_via_signal.js b/jstests/replsets/election_handoff_via_signal.js new file mode 100644 index 00000000000..4bc58c95d40 --- /dev/null +++ b/jstests/replsets/election_handoff_via_signal.js @@ -0,0 +1,25 @@ +/** + * This is a basic test that checks that, when election handoff is enabled, a primary that is sent a + * non-terminal signal sends a ReplSetStepUp request to an eligible candidate. + */ + +(function() { + "use strict"; + load("jstests/replsets/libs/election_handoff.js"); + + const testName = "election_handoff_via_signal"; + const numNodes = 3; + const rst = ReplSetTest({name: testName, nodes: numNodes}); + const nodes = rst.nodeList(); + rst.startSet(); + + // Make sure there are no election timeouts firing for the duration of the test. This helps + // ensure that the test will only pass if the election handoff succeeds. + const config = rst.getReplSetConfig(); + config.settings = {"electionTimeoutMillis": 12 * 60 * 60 * 1000}; + rst.initiate(config); + + ElectionHandoffTest.testElectionHandoff(rst, 0, 1, {stepDownBySignal: true}); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/libs/election_handoff.js b/jstests/replsets/libs/election_handoff.js index 6bb903c1b2c..b81aca51363 100644 --- a/jstests/replsets/libs/election_handoff.js +++ b/jstests/replsets/libs/election_handoff.js @@ -10,13 +10,19 @@ var ElectionHandoffTest = (function() { load("jstests/replsets/rslib.js"); const kStepDownPeriodSecs = 30; + const kSIGTERM = 15; /** * Exercises and validates an election handoff scenario by stepping down the primary and * ensuring that the node at "expectedCandidateId" is stepped up in its place. The desired * configuration of the replica set is passed in as its ReplSetTest instance. + * + * The options parameter contains extra options for the handoff. Currently supported options + * are: + * stepDownBySignal - When this option is set, the primary will be stepped down by stopping + * and restarting with sigterm, rather than with a replSetStepDown command */ - function testElectionHandoff(rst, initialPrimaryId, expectedCandidateId) { + function testElectionHandoff(rst, initialPrimaryId, expectedCandidateId, options = {}) { const config = rst.getReplSetConfigFromNode(); const numNodes = config.members.length; const memberInfo = config.members[expectedCandidateId]; @@ -50,10 +56,15 @@ var ElectionHandoffTest = (function() { rst.awaitNodesAgreeOnAppliedOpTime(); // Step down the current primary. - assert.adminCommandWorkedAllowingNetworkError(primary, { - replSetStepDown: kStepDownPeriodSecs, - secondaryCatchUpPeriodSecs: kStepDownPeriodSecs / 2 - }); + if (options["stepDownBySignal"]) { + rst.stop(initialPrimaryId, kSIGTERM, {}, {forRestart: true}); + rst.start(initialPrimaryId, {}, true); + } else { + assert.adminCommandWorkedAllowingNetworkError(primary, { + replSetStepDown: kStepDownPeriodSecs, + secondaryCatchUpPeriodSecs: kStepDownPeriodSecs / 2 + }); + } jsTestLog(`Checking that the secondary with id ${expectedCandidateId} is stepped up...`); @@ -77,4 +88,4 @@ var ElectionHandoffTest = (function() { return {testElectionHandoff: testElectionHandoff, stepDownPeriodSecs: kStepDownPeriodSecs}; -})();
\ No newline at end of file +})(); diff --git a/src/mongo/db/commands/generic_servers.cpp b/src/mongo/db/commands/generic_servers.cpp index 7ae401f0846..93ad3389461 100644 --- a/src/mongo/db/commands/generic_servers.cpp +++ b/src/mongo/db/commands/generic_servers.cpp @@ -331,6 +331,9 @@ void CmdShutdown::addRequiredPrivileges(const std::string& dbname, } void CmdShutdown::shutdownHelper(const BSONObj& cmdObj) { + ShutdownTaskArgs shutdownArgs; + shutdownArgs.isUserInitiated = true; + MONGO_FAIL_POINT_BLOCK(crashOnShutdown, crashBlock) { const std::string crashHow = crashBlock.getData()["how"].str(); if (crashHow == "fault") { @@ -344,7 +347,7 @@ void CmdShutdown::shutdownHelper(const BSONObj& cmdObj) { #if defined(_WIN32) // Signal the ServiceMain thread to shutdown. if (ntservice::shouldStartService()) { - shutdownNoTerminate(); + shutdownNoTerminate(shutdownArgs); // Client expects us to abruptly close the socket as part of exiting // so this function is not allowed to return. @@ -354,7 +357,7 @@ void CmdShutdown::shutdownHelper(const BSONObj& cmdObj) { } else #endif { - exitCleanly(EXIT_CLEAN); // this never returns + shutdown(EXIT_CLEAN, shutdownArgs); // this never returns } } diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 83e64191759..8824c3f0940 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -867,12 +867,39 @@ MONGO_INITIALIZER_GENERAL(setSSLManagerType, MONGO_NO_PREREQUISITES, ("SSLManage // NOTE: This function may be called at any time after registerShutdownTask is called below. It // must not depend on the prior execution of mongo initializers or the existence of threads. -void shutdownTask() { +void shutdownTask(const ShutdownTaskArgs& shutdownArgs) { Client::initThreadIfNotAlready(); auto const client = Client::getCurrent(); auto const serviceContext = client->getServiceContext(); + // If we don't have shutdownArgs, we're shutting down from a signal, or other clean shutdown + // path. + // + // In that case, do a default step down, still shutting down if stepDown fails. + { + auto replCoord = repl::ReplicationCoordinator::get(serviceContext); + if (replCoord && !shutdownArgs.isUserInitiated) { + replCoord->enterTerminalShutdown(); + ServiceContext::UniqueOperationContext uniqueOpCtx; + OperationContext* opCtx = client->getOperationContext(); + if (!opCtx) { + uniqueOpCtx = client->makeOperationContext(); + opCtx = uniqueOpCtx.get(); + } + + try { + uassertStatusOK( + replCoord->stepDown(opCtx, false /* force */, Seconds(10), Seconds(120))); + } catch (const ExceptionFor<ErrorCodes::NotMaster>&) { + // ignore not master errors + } catch (const DBException& e) { + log() << "Failed to stepDown in non-command initiated shutdown path " + << e.toString(); + } + } + } + // Terminate the balancer thread so it doesn't leak memory. if (auto balancer = Balancer::get(serviceContext)) { balancer->interruptBalancer(); diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h index a09f61c6120..55d70436847 100644 --- a/src/mongo/db/repl/replication_coordinator.h +++ b/src/mongo/db/repl/replication_coordinator.h @@ -114,6 +114,15 @@ public: virtual void startup(OperationContext* opCtx) = 0; /** + * Start terminal shutdown. This causes the topology coordinator to refuse to vote in any + * further elections. This should only be called from global shutdown after we've passed the + * point of no return. + * + * This should be called once we are sure to call shutdown(). + */ + virtual void enterTerminalShutdown() = 0; + + /** * Does whatever cleanup is required to stop replication, including instructing the other * components of the replication system to shut down and stop any threads they are using, * blocking until all replication-related shutdown tasks are complete. diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index f2ddcd13ac5..024ed39bd0b 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -827,6 +827,11 @@ void ReplicationCoordinatorImpl::startup(OperationContext* opCtx) { } } +void ReplicationCoordinatorImpl::enterTerminalShutdown() { + stdx::lock_guard<stdx::mutex> lk(_mutex); + _inTerminalShutdown = true; +} + void ReplicationCoordinatorImpl::shutdown(OperationContext* opCtx) { // Shutdown must: // * prevent new threads from blocking in awaitReplication @@ -3370,6 +3375,13 @@ Status ReplicationCoordinatorImpl::processReplSetRequestVotes( { stdx::lock_guard<stdx::mutex> lk(_mutex); + + // We should only enter terminal shutdown from global terminal exit. In that case, rather + // than voting in a term we don't plan to stay alive in, refuse to vote. + if (_inTerminalShutdown) { + return Status(ErrorCodes::ShutdownInProgress, "In the process of shutting down"); + } + _topCoord->processReplSetRequestVotes(args, response); } diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h index b5315e9ba8d..bd1d4a26f86 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.h +++ b/src/mongo/db/repl/replication_coordinator_impl.h @@ -104,6 +104,8 @@ public: virtual void startup(OperationContext* opCtx) override; + virtual void enterTerminalShutdown() override; + virtual void shutdown(OperationContext* opCtx) override; virtual const ReplSettings& getSettings() const override; @@ -1350,6 +1352,9 @@ private: // When we decide to step down due to hearing about a higher term, we remember the term we heard // here so we can update our term to match as part of finishing stepdown. boost::optional<long long> _pendingTermUpdateDuringStepDown; // (M) + + // If we're in terminal shutdown. If true, we'll refuse to vote in elections. + bool _inTerminalShutdown = false; // (M) }; } // namespace repl diff --git a/src/mongo/db/repl/replication_coordinator_impl_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_test.cpp index 8b583073c2e..dc1d39bb82a 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_test.cpp @@ -5998,6 +5998,58 @@ TEST_F(ReplCoordTest, NodeFailsVoteRequestIfItFailsToStoreLastVote) { ASSERT_EQUALS(lastVote.getCandidateIndex(), 0); } +TEST_F(ReplCoordTest, NodeNodesNotGrantVoteIfInTerminalShutdown) { + // Set up a 2-node replica set config. + assertStartSuccess(BSON("_id" + << "mySet" + << "version" + << 2 + << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" + << 0) + << BSON("host" + << "node2:12345" + << "_id" + << 1))), + HostAndPort("node1", 12345)); + auto time = OpTimeWithTermOne(100, 1); + ASSERT_OK(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + getReplCoord()->setMyLastAppliedOpTime(time); + getReplCoord()->setMyLastDurableOpTime(time); + simulateSuccessfulV1Election(); + + // Get our current term, as primary. + ASSERT(getReplCoord()->getMemberState().primary()); + auto initTerm = getReplCoord()->getTerm(); + + auto opCtx = makeOperationContext(); + + ReplSetRequestVotesArgs args; + ASSERT_OK(args.initialize(BSON("replSetRequestVotes" << 1 << "setName" + << "mySet" + << "term" + << initTerm + 1 // term of new candidate. + << "candidateIndex" + << 1LL + << "configVersion" + << 2LL + << "dryRun" + << false + << "lastCommittedOp" + << time.asOpTime().toBSON()))); + ReplSetRequestVotesResponse response; + + getReplCoord()->enterTerminalShutdown(); + + auto r = getReplCoord()->processReplSetRequestVotes(opCtx.get(), args, &response); + + ASSERT_NOT_OK(r); + ASSERT_EQUALS("In the process of shutting down", r.reason()); + ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, r.code()); +} + // TODO(schwerin): Unit test election id updating } // namespace } // namespace repl diff --git a/src/mongo/db/repl/replication_coordinator_mock.cpp b/src/mongo/db/repl/replication_coordinator_mock.cpp index e49bc521cd1..56717fe4ff0 100644 --- a/src/mongo/db/repl/replication_coordinator_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_mock.cpp @@ -71,6 +71,10 @@ void ReplicationCoordinatorMock::startup(OperationContext* opCtx) { // TODO } +void ReplicationCoordinatorMock::enterTerminalShutdown() { + // TODO +} + void ReplicationCoordinatorMock::shutdown(OperationContext*) { // TODO } diff --git a/src/mongo/db/repl/replication_coordinator_mock.h b/src/mongo/db/repl/replication_coordinator_mock.h index 4c92a83f72b..a936e81c739 100644 --- a/src/mongo/db/repl/replication_coordinator_mock.h +++ b/src/mongo/db/repl/replication_coordinator_mock.h @@ -64,6 +64,8 @@ public: virtual void startup(OperationContext* opCtx); + virtual void enterTerminalShutdown(); + virtual void shutdown(OperationContext* opCtx); virtual void appendDiagnosticBSON(BSONObjBuilder* bob) override {} diff --git a/src/mongo/embedded/replication_coordinator_embedded.cpp b/src/mongo/embedded/replication_coordinator_embedded.cpp index 9f7fb99cfe4..62a2eb36d0d 100644 --- a/src/mongo/embedded/replication_coordinator_embedded.cpp +++ b/src/mongo/embedded/replication_coordinator_embedded.cpp @@ -50,6 +50,8 @@ ReplicationCoordinatorEmbedded::~ReplicationCoordinatorEmbedded() = default; void ReplicationCoordinatorEmbedded::startup(OperationContext* opCtx) {} +void ReplicationCoordinatorEmbedded::enterTerminalShutdown() {} + void ReplicationCoordinatorEmbedded::shutdown(OperationContext* opCtx) {} const ReplSettings& ReplicationCoordinatorEmbedded::getSettings() const { diff --git a/src/mongo/embedded/replication_coordinator_embedded.h b/src/mongo/embedded/replication_coordinator_embedded.h index f4f3cf2afd8..679db98dd9c 100644 --- a/src/mongo/embedded/replication_coordinator_embedded.h +++ b/src/mongo/embedded/replication_coordinator_embedded.h @@ -48,6 +48,8 @@ public: void startup(OperationContext* opCtx) override; + void enterTerminalShutdown() override; + void shutdown(OperationContext* opCtx) override; // Returns the ServiceContext where this instance runs. diff --git a/src/mongo/util/exit.cpp b/src/mongo/util/exit.cpp index 3269cec3b39..fc874c0eea4 100644 --- a/src/mongo/util/exit.cpp +++ b/src/mongo/util/exit.cpp @@ -35,10 +35,10 @@ #include "mongo/util/exit.h" #include <boost/optional.hpp> +#include <functional> #include <stack> #include "mongo/stdx/condition_variable.h" -#include "mongo/stdx/functional.h" #include "mongo/stdx/mutex.h" #include "mongo/stdx/thread.h" #include "mongo/util/log.h" @@ -53,14 +53,14 @@ stdx::condition_variable shutdownTasksComplete; boost::optional<ExitCode> shutdownExitCode; bool shutdownTasksInProgress = false; AtomicUInt32 shutdownFlag; -std::stack<stdx::function<void()>> shutdownTasks; +std::stack<std::function<void(const ShutdownTaskArgs&)>> shutdownTasks; stdx::thread::id shutdownTasksThreadId; -void runTasks(decltype(shutdownTasks) tasks) { +void runTasks(decltype(shutdownTasks) tasks, const ShutdownTaskArgs& shutdownArgs) { while (!tasks.empty()) { const auto& task = tasks.top(); try { - task(); + task(shutdownArgs); } catch (...) { std::terminate(); } @@ -97,13 +97,13 @@ ExitCode waitForShutdown() { return shutdownExitCode.get(); } -void registerShutdownTask(stdx::function<void()> task) { +void registerShutdownTask(std::function<void(const ShutdownTaskArgs&)> task) { stdx::lock_guard<stdx::mutex> lock(shutdownMutex); invariant(!globalInShutdownDeprecated()); shutdownTasks.emplace(std::move(task)); } -void shutdown(ExitCode code) { +void shutdown(ExitCode code, const ShutdownTaskArgs& shutdownArgs) { decltype(shutdownTasks) localTasks; { @@ -139,7 +139,7 @@ void shutdown(ExitCode code) { localTasks.swap(shutdownTasks); } - runTasks(std::move(localTasks)); + runTasks(std::move(localTasks), shutdownArgs); { stdx::lock_guard<stdx::mutex> lock(shutdownMutex); @@ -151,7 +151,7 @@ void shutdown(ExitCode code) { } } -void shutdownNoTerminate() { +void shutdownNoTerminate(const ShutdownTaskArgs& shutdownArgs) { decltype(shutdownTasks) localTasks; { @@ -167,7 +167,7 @@ void shutdownNoTerminate() { localTasks.swap(shutdownTasks); } - runTasks(std::move(localTasks)); + runTasks(std::move(localTasks), shutdownArgs); { stdx::lock_guard<stdx::mutex> lock(shutdownMutex); diff --git a/src/mongo/util/exit.h b/src/mongo/util/exit.h index b7ca4482663..a6e51aa9ec3 100644 --- a/src/mongo/util/exit.h +++ b/src/mongo/util/exit.h @@ -30,13 +30,24 @@ #pragma once +#include <boost/optional.hpp> +#include <functional> + #include "mongo/platform/compiler.h" -#include "mongo/stdx/functional.h" #include "mongo/util/exit_code.h" namespace mongo { /** + * ShutdownTaskArgs holds any arguments we might like to pass from a manual invocation of the + * shutdown command. It is meant to give a default shutdown when default constructed. + */ +struct ShutdownTaskArgs { + // This should be set to true if we called shutdown from the shutdown command + bool isUserInitiated = false; +}; + +/** * Determines if the shutdown flag is set. * * Calling this function is deprecated because modules that consult it @@ -58,7 +69,14 @@ ExitCode waitForShutdown(); * shutdown or shutdownNoTerminate has been called, std::terminate is * called. */ -void registerShutdownTask(stdx::function<void()>); +void registerShutdownTask(std::function<void(const ShutdownTaskArgs& shutdownArgs)>); + +/** + * For shutdown tasks that don't care to distinguish if they're called from command shutdown + */ +inline void registerShutdownTask(std::function<void()> task) { + registerShutdownTask([task = std::move(task)](const ShutdownTaskArgs&) { task(); }); +} /** * Toggles the shutdown flag to 'true', runs registered shutdown @@ -67,7 +85,7 @@ void registerShutdownTask(stdx::function<void()>); * shutdown tasks. It is illegal to reenter this function from a * registered shutdown task. The function does not return. */ -MONGO_COMPILER_NORETURN void shutdown(ExitCode code); +MONGO_COMPILER_NORETURN void shutdown(ExitCode code, const ShutdownTaskArgs& shutdownArgs = {}); /** * Toggles the shutdown flag to 'true' and runs the registered @@ -76,7 +94,7 @@ MONGO_COMPILER_NORETURN void shutdown(ExitCode code); * callers return immediately. It is legal to call shutdownNoTerminate * from a shutdown task. */ -void shutdownNoTerminate(); +void shutdownNoTerminate(const ShutdownTaskArgs& shutdownArgs = {}); /** An alias for 'shutdown'. */ MONGO_COMPILER_NORETURN inline void exitCleanly(ExitCode code) { |