diff options
author | Xuerui Fa <xuerui.fa@mongodb.com> | 2019-10-01 17:51:45 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-10-01 17:51:45 +0000 |
commit | b5e2e827f65459fd5cf5f88c081dda01b801867d (patch) | |
tree | a71757a1302f351f1a1bda1df42fa1a8ccb96c01 /src/mongo | |
parent | 065bc52eec452e5bedfd7d9d8df569ef4309616d (diff) | |
download | mongo-b5e2e827f65459fd5cf5f88c081dda01b801867d.tar.gz |
SERVER-41506 Added tracking for metrics around a nodes calling an election
(cherry picked from commit 87c9601a762048ea93559b13c3b6043b4017b024)
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl.cpp | 12 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl.h | 20 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp | 54 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp | 35 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp | 41 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_metrics.cpp | 54 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_metrics.h | 14 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_metrics.idl | 41 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator.cpp | 87 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator.h | 23 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator_v1_test.cpp | 20 |
11 files changed, 232 insertions, 169 deletions
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index aafca514fd2..0f686fd0e40 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -2179,8 +2179,7 @@ void ReplicationCoordinatorImpl::_handleTimePassing( // For election protocol v1, call _startElectSelfIfEligibleV1 to avoid race // against other elections caused by events like election timeout, replSetStepUp etc. - _startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason::kSingleNodePromptElection); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kSingleNodePromptElection); } bool ReplicationCoordinatorImpl::isMasterForReportingPurposes() { @@ -2546,8 +2545,7 @@ Status ReplicationCoordinatorImpl::processReplSetFreeze(int secs, BSONObjBuilder result.getValue()) { // For election protocol v1, call _startElectSelfIfEligibleV1 to avoid race // against other elections caused by events like election timeout, replSetStepUp etc. - _startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason::kSingleNodePromptElection); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kSingleNodePromptElection); } return Status::OK(); @@ -3000,7 +2998,7 @@ void ReplicationCoordinatorImpl::_performPostMemberStateUpdateAction( case kActionStartSingleNodeElection: // In protocol version 1, single node replset will run an election instead of // kActionWinElection as in protocol version 0. - _startElectSelfV1(TopologyCoordinator::StartElectionReason::kElectionTimeout); + _startElectSelfV1(StartElectionReasonEnum::kElectionTimeout); break; default: severe() << "Unknown post member state update action " << static_cast<int>(action); @@ -4065,8 +4063,8 @@ CallbackFn ReplicationCoordinatorImpl::_wrapAsCallbackFn(const stdx::function<vo Status ReplicationCoordinatorImpl::stepUpIfEligible(bool skipDryRun) { - auto reason = skipDryRun ? TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun - : TopologyCoordinator::StartElectionReason::kStepUpRequest; + auto reason = skipDryRun ? StartElectionReasonEnum::kStepUpRequestSkipDryRun + : StartElectionReasonEnum::kStepUpRequest; _startElectSelfIfEligibleV1(reason); EventHandle finishEvent; diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h index 8d6753f6f58..ad0273da3b2 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.h +++ b/src/mongo/db/repl/replication_coordinator_impl.h @@ -1023,8 +1023,8 @@ private: * _startVoteRequester_inlock() * _onVoteRequestComplete() */ - void _startElectSelfV1_inlock(TopologyCoordinator::StartElectionReason reason); - void _startElectSelfV1(TopologyCoordinator::StartElectionReason reason); + void _startElectSelfV1_inlock(StartElectionReasonEnum reason); + void _startElectSelfV1(StartElectionReasonEnum reason); /** * Callback called when the dryRun VoteRequester has completed; checks the results and @@ -1032,15 +1032,13 @@ private: * "originalTerm" was the term during which the dry run began, if the term has since * changed, do not run for election. */ - void _processDryRunResult(long long originalTerm, - TopologyCoordinator::StartElectionReason reason); + void _processDryRunResult(long long originalTerm, StartElectionReasonEnum reason); /** * Begins executing a real election. This is called either a successful dry run, or when the * dry run was skipped (which may be specified for a ReplSetStepUp). */ - void _startRealElection_inlock(long long originalTerm, - TopologyCoordinator::StartElectionReason reason); + void _startRealElection_inlock(long long originalTerm, StartElectionReasonEnum reason); /** * Writes the last vote in persistent storage after completing dry run successfully. @@ -1048,13 +1046,12 @@ private: */ void _writeLastVoteForMyElection(LastVote lastVote, const executor::TaskExecutor::CallbackArgs& cbData, - TopologyCoordinator::StartElectionReason reason); + StartElectionReasonEnum reason); /** * Starts VoteRequester to run the real election when last vote write has completed. */ - void _startVoteRequester_inlock(long long newTerm, - TopologyCoordinator::StartElectionReason reason); + void _startVoteRequester_inlock(long long newTerm, StartElectionReasonEnum reason); /** * Callback called when the VoteRequester has completed; checks the results and @@ -1062,8 +1059,7 @@ private: * "originalTerm" was the term during which the election began, if the term has since * changed, do not step up as primary. */ - void _onVoteRequestComplete(long long originalTerm, - TopologyCoordinator::StartElectionReason reason); + void _onVoteRequestComplete(long long originalTerm, StartElectionReasonEnum reason); /** * Removes 'host' from the sync source blacklist. If 'host' isn't found, it's simply @@ -1281,7 +1277,7 @@ private: /** * Callback which starts an election if this node is electable and using protocolVersion 1. */ - void _startElectSelfIfEligibleV1(TopologyCoordinator::StartElectionReason reason); + void _startElectSelfIfEligibleV1(StartElectionReasonEnum reason); /** * Schedules work to be run no sooner than 'when' and returns handle to callback. diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp index b1464c0de9b..04f191ad81c 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp @@ -92,15 +92,12 @@ public: } }; - -void ReplicationCoordinatorImpl::_startElectSelfV1( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startElectSelfV1(StartElectionReasonEnum reason) { stdx::lock_guard<stdx::mutex> lk(_mutex); _startElectSelfV1_inlock(reason); } -void ReplicationCoordinatorImpl::_startElectSelfV1_inlock( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(StartElectionReasonEnum reason) { invariant(!_voteRequester); switch (_rsConfigState) { @@ -146,10 +143,7 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock( long long term = _topCoord->getTerm(); int primaryIndex = -1; - Date_t now = _replExecutor->now(); - ReplicationMetrics::get(getServiceContext()).setElectionCandidateMetrics(now); - - if (reason == TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun) { + if (reason == StartElectionReasonEnum::kStepUpRequestSkipDryRun) { long long newTerm = term + 1; log() << "skipping dry run and running for election in term " << newTerm; _startRealElection_inlock(newTerm, reason); @@ -161,7 +155,7 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock( _voteRequester.reset(new VoteRequester); // Only set primaryIndex if the primary's vote is required during the dry run. - if (reason == TopologyCoordinator::StartElectionReason::kCatchupTakeover) { + if (reason == StartElectionReasonEnum::kCatchupTakeover) { primaryIndex = _topCoord->getCurrentPrimaryIndex(); } StatusWith<executor::TaskExecutor::EventHandle> nextPhaseEvh = @@ -185,8 +179,8 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock( lossGuard.dismiss(); } -void ReplicationCoordinatorImpl::_processDryRunResult( - long long originalTerm, TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_processDryRunResult(long long originalTerm, + StartElectionReasonEnum reason) { stdx::lock_guard<stdx::mutex> lk(_mutex); LoseElectionDryRunGuardV1 lossGuard(this); @@ -221,8 +215,30 @@ void ReplicationCoordinatorImpl::_processDryRunResult( lossGuard.dismiss(); } -void ReplicationCoordinatorImpl::_startRealElection_inlock( - long long newTerm, TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startRealElection_inlock(long long newTerm, + StartElectionReasonEnum reason) { + + const Date_t now = _replExecutor->now(); + const OpTime lastCommittedOpTime = _topCoord->getLastCommittedOpTime(); + const OpTime lastSeenOpTime = _topCoord->latestKnownOpTime(); + const int numVotesNeeded = _rsConfig.getMajorityVoteCount(); + const double priorityAtElection = _rsConfig.getMemberAt(_selfIndex).getPriority(); + const Milliseconds electionTimeoutMillis = _rsConfig.getElectionTimeoutPeriod(); + const int priorPrimaryIndex = _topCoord->getCurrentPrimaryIndex(); + const boost::optional<int> priorPrimaryMemberId = (priorPrimaryIndex == -1) + ? boost::none + : boost::make_optional(_rsConfig.getMemberAt(priorPrimaryIndex).getId().getData()); + + ReplicationMetrics::get(getServiceContext()) + .setElectionCandidateMetrics(reason, + now, + newTerm, + lastCommittedOpTime, + lastSeenOpTime, + numVotesNeeded, + priorityAtElection, + electionTimeoutMillis, + priorPrimaryMemberId); ReplicationMetrics::get(getServiceContext()).incrementNumElectionsCalledForReason(reason); LoseElectionDryRunGuardV1 lossGuard(this); @@ -254,7 +270,7 @@ void ReplicationCoordinatorImpl::_startRealElection_inlock( void ReplicationCoordinatorImpl::_writeLastVoteForMyElection( LastVote lastVote, const executor::TaskExecutor::CallbackArgs& cbData, - TopologyCoordinator::StartElectionReason reason) { + StartElectionReasonEnum reason) { // storeLocalLastVoteDocument can call back in to the replication coordinator, // so _mutex must be unlocked here. However, we cannot return until we // lock it because we want to lose the election on cancel or error and @@ -292,8 +308,8 @@ void ReplicationCoordinatorImpl::_writeLastVoteForMyElection( lossGuard.dismiss(); } -void ReplicationCoordinatorImpl::_startVoteRequester_inlock( - long long newTerm, TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startVoteRequester_inlock(long long newTerm, + StartElectionReasonEnum reason) { const auto lastOpTime = _getMyLastAppliedOpTime_inlock(); _voteRequester.reset(new VoteRequester); @@ -313,8 +329,8 @@ void ReplicationCoordinatorImpl::_startVoteRequester_inlock( MONGO_FAIL_POINT_DEFINE(electionHangsBeforeUpdateMemberState); -void ReplicationCoordinatorImpl::_onVoteRequestComplete( - long long newTerm, TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long newTerm, + StartElectionReasonEnum reason) { stdx::lock_guard<stdx::mutex> lk(_mutex); LoseElectionGuardV1 lossGuard(this); diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp index f54883200ea..ed6c0febe18 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp @@ -374,9 +374,9 @@ TEST_F(ReplCoordTest, ElectionFailsWhenInsufficientVotesAreReceivedDuringDryRun) << false << "reason" << "don't like him much"))); voteRequests++; - // Check that the node's election candidate metrics are set once it has called an - // election. - ASSERT_BSONOBJ_NE( + + // Check that the node's election candidate metrics are not set if a dry run fails. + ASSERT_BSONOBJ_EQ( BSONObj(), ReplicationMetrics::get(getServiceContext()).getElectionCandidateMetricsBSON()); } else { @@ -883,7 +883,7 @@ public: } void performSuccessfulTakeover(Date_t takeoverTime, - TopologyCoordinator::StartElectionReason reason, + StartElectionReasonEnum reason, const LastVote& lastVoteExpected) { startCapturingLogMessages(); simulateSuccessfulV1ElectionAt(takeoverTime); @@ -898,7 +898,7 @@ public: ASSERT_EQ(lastVoteExpected.getCandidateIndex(), lastVote.getValue().getCandidateIndex()); ASSERT_EQ(lastVoteExpected.getTerm(), lastVote.getValue().getTerm()); - if (reason == TopologyCoordinator::StartElectionReason::kPriorityTakeover) { + if (reason == StartElectionReasonEnum::kPriorityTakeover) { ASSERT_EQUALS(1, countLogLinesContaining("Starting an election for a priority takeover")); } @@ -1448,9 +1448,8 @@ TEST_F(TakeoverTest, SuccessfulCatchupTakeover) { ASSERT_EQUALS(1, countLogLinesContaining("Starting an election for a catchup takeover")); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(catchupTakeoverTime, - TopologyCoordinator::StartElectionReason::kCatchupTakeover, - lastVoteExpected); + performSuccessfulTakeover( + catchupTakeoverTime, StartElectionReasonEnum::kCatchupTakeover, lastVoteExpected); // Check that the numCatchUpTakeoversCalled and the numCatchUpTakeoversSuccessful election // metrics have been incremented, and that none of the metrics that track the number of @@ -1707,9 +1706,8 @@ TEST_F(TakeoverTest, PrimaryCatchesUpBeforeHighPriorityNodeCatchupTakeover) { config, now + longElectionTimeout, HostAndPort("node2", 12345), currentOptime); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(priorityTakeoverTime, - TopologyCoordinator::StartElectionReason::kPriorityTakeover, - lastVoteExpected); + performSuccessfulTakeover( + priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected); } TEST_F(TakeoverTest, SchedulesPriorityTakeoverIfNodeHasHigherPriorityThanCurrentPrimary) { @@ -1801,9 +1799,8 @@ TEST_F(TakeoverTest, SuccessfulPriorityTakeover) { config, now + halfElectionTimeout, HostAndPort("node2", 12345), myOptime); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(priorityTakeoverTime, - TopologyCoordinator::StartElectionReason::kPriorityTakeover, - lastVoteExpected); + performSuccessfulTakeover( + priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected); // Check that the numPriorityTakeoversCalled and the numPriorityTakeoversSuccessful election // metrics have been incremented, and that none of the metrics that track the number of @@ -1892,9 +1889,8 @@ TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedSameSecond) { Date_t() + Seconds(closeEnoughOpTime.getSecs())); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(priorityTakeoverTime, - TopologyCoordinator::StartElectionReason::kPriorityTakeover, - lastVoteExpected); + performSuccessfulTakeover( + priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected); } TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedDifferentSecond) { @@ -1967,9 +1963,8 @@ TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedDifferentSecond) { Date_t() + Seconds(closeEnoughOpTime.getSecs())); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(priorityTakeoverTime, - TopologyCoordinator::StartElectionReason::kPriorityTakeover, - lastVoteExpected); + performSuccessfulTakeover( + priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected); } TEST_F(ReplCoordTest, NodeCancelsElectionUponReceivingANewConfigDuringDryRun) { diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index 6fa02d27a39..faec4c34d41 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -304,8 +304,7 @@ stdx::unique_lock<stdx::mutex> ReplicationCoordinatorImpl::_handleHeartbeatRespo LOG_FOR_ELECTION(0) << "Scheduling priority takeover at " << _priorityTakeoverWhen; _priorityTakeoverCbh = _scheduleWorkAt( _priorityTakeoverWhen, [=](const mongo::executor::TaskExecutor::CallbackArgs&) { - _startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason::kPriorityTakeover); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kPriorityTakeover); }); } break; @@ -318,8 +317,7 @@ stdx::unique_lock<stdx::mutex> ReplicationCoordinatorImpl::_handleHeartbeatRespo LOG_FOR_ELECTION(0) << "Scheduling catchup takeover at " << _catchupTakeoverWhen; _catchupTakeoverCbh = _scheduleWorkAt( _catchupTakeoverWhen, [=](const mongo::executor::TaskExecutor::CallbackArgs&) { - _startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason::kCatchupTakeover); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kCatchupTakeover); }); } break; @@ -875,15 +873,14 @@ void ReplicationCoordinatorImpl::_cancelAndRescheduleElectionTimeout_inlock() { _handleElectionTimeoutWhen = when; _handleElectionTimeoutCbh = _scheduleWorkAt(when, [=](const mongo::executor::TaskExecutor::CallbackArgs&) { - _startElectSelfIfEligibleV1(TopologyCoordinator::StartElectionReason::kElectionTimeout); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kElectionTimeout); }); } -void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1(StartElectionReasonEnum reason) { stdx::lock_guard<stdx::mutex> lock(_mutex); // If it is not a single node replica set, no need to start an election after stepdown timeout. - if (reason == TopologyCoordinator::StartElectionReason::kSingleNodePromptElection && + if (reason == StartElectionReasonEnum::kSingleNodePromptElection && _rsConfig.getNumMembers() != 1) { return; } @@ -903,52 +900,56 @@ void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1( const auto status = _topCoord->becomeCandidateIfElectable(_replExecutor->now(), reason); if (!status.isOK()) { switch (reason) { - case TopologyCoordinator::StartElectionReason::kElectionTimeout: + case StartElectionReasonEnum::kElectionTimeout: LOG_FOR_ELECTION(0) << "Not starting an election, since we are not electable due to: " << status.reason(); break; - case TopologyCoordinator::StartElectionReason::kPriorityTakeover: + case StartElectionReasonEnum::kPriorityTakeover: LOG_FOR_ELECTION(0) << "Not starting an election for a priority takeover, " << "since we are not electable due to: " << status.reason(); break; - case TopologyCoordinator::StartElectionReason::kStepUpRequest: - case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: + case StartElectionReasonEnum::kStepUpRequest: + case StartElectionReasonEnum::kStepUpRequestSkipDryRun: LOG_FOR_ELECTION(0) << "Not starting an election for a replSetStepUp request, " << "since we are not electable due to: " << status.reason(); break; - case TopologyCoordinator::StartElectionReason::kCatchupTakeover: + case StartElectionReasonEnum::kCatchupTakeover: LOG_FOR_ELECTION(0) << "Not starting an election for a catchup takeover, " << "since we are not electable due to: " << status.reason(); break; - case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: + case StartElectionReasonEnum::kSingleNodePromptElection: LOG_FOR_ELECTION(0) << "Not starting an election for a single node replica set prompt election, " << "since we are not electable due to: " << status.reason(); break; + default: + MONGO_UNREACHABLE; } return; } switch (reason) { - case TopologyCoordinator::StartElectionReason::kElectionTimeout: + case StartElectionReasonEnum::kElectionTimeout: LOG_FOR_ELECTION(0) << "Starting an election, since we've seen no PRIMARY in the past " << _rsConfig.getElectionTimeoutPeriod(); break; - case TopologyCoordinator::StartElectionReason::kPriorityTakeover: + case StartElectionReasonEnum::kPriorityTakeover: LOG_FOR_ELECTION(0) << "Starting an election for a priority takeover"; break; - case TopologyCoordinator::StartElectionReason::kStepUpRequest: - case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: + case StartElectionReasonEnum::kStepUpRequest: + case StartElectionReasonEnum::kStepUpRequestSkipDryRun: LOG_FOR_ELECTION(0) << "Starting an election due to step up request"; break; - case TopologyCoordinator::StartElectionReason::kCatchupTakeover: + case StartElectionReasonEnum::kCatchupTakeover: LOG_FOR_ELECTION(0) << "Starting an election for a catchup takeover"; break; - case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: + case StartElectionReasonEnum::kSingleNodePromptElection: LOG_FOR_ELECTION(0) << "Starting an election due to single node replica set prompt election"; break; + default: + MONGO_UNREACHABLE; } _startElectSelfV1_inlock(reason); diff --git a/src/mongo/db/repl/replication_metrics.cpp b/src/mongo/db/repl/replication_metrics.cpp index 55508674562..0794c43fd0c 100644 --- a/src/mongo/db/repl/replication_metrics.cpp +++ b/src/mongo/db/repl/replication_metrics.cpp @@ -56,32 +56,31 @@ ReplicationMetrics::ReplicationMetrics() ReplicationMetrics::~ReplicationMetrics() {} -void ReplicationMetrics::incrementNumElectionsCalledForReason( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationMetrics::incrementNumElectionsCalledForReason(StartElectionReasonEnum reason) { stdx::lock_guard<stdx::mutex> lk(_mutex); switch (reason) { - case TopologyCoordinator::StartElectionReason::kStepUpRequest: - case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: { + case StartElectionReasonEnum::kStepUpRequest: + case StartElectionReasonEnum::kStepUpRequestSkipDryRun: { ElectionReasonCounter& stepUpCmd = _electionMetrics.getStepUpCmd(); stepUpCmd.incrementCalled(); break; } - case TopologyCoordinator::StartElectionReason::kPriorityTakeover: { + case StartElectionReasonEnum::kPriorityTakeover: { ElectionReasonCounter& priorityTakeover = _electionMetrics.getPriorityTakeover(); priorityTakeover.incrementCalled(); break; } - case TopologyCoordinator::StartElectionReason::kCatchupTakeover: { + case StartElectionReasonEnum::kCatchupTakeover: { ElectionReasonCounter& catchUpTakeover = _electionMetrics.getCatchUpTakeover(); catchUpTakeover.incrementCalled(); break; } - case TopologyCoordinator::StartElectionReason::kElectionTimeout: { + case StartElectionReasonEnum::kElectionTimeout: { ElectionReasonCounter& electionTimeout = _electionMetrics.getElectionTimeout(); electionTimeout.incrementCalled(); break; } - case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: { + case StartElectionReasonEnum::kSingleNodePromptElection: { ElectionReasonCounter& freezeTimeout = _electionMetrics.getFreezeTimeout(); freezeTimeout.incrementCalled(); break; @@ -89,32 +88,31 @@ void ReplicationMetrics::incrementNumElectionsCalledForReason( } } -void ReplicationMetrics::incrementNumElectionsSuccessfulForReason( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationMetrics::incrementNumElectionsSuccessfulForReason(StartElectionReasonEnum reason) { stdx::lock_guard<stdx::mutex> lk(_mutex); switch (reason) { - case TopologyCoordinator::StartElectionReason::kStepUpRequest: - case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: { + case StartElectionReasonEnum::kStepUpRequest: + case StartElectionReasonEnum::kStepUpRequestSkipDryRun: { ElectionReasonCounter& stepUpCmd = _electionMetrics.getStepUpCmd(); stepUpCmd.incrementSuccessful(); break; } - case TopologyCoordinator::StartElectionReason::kPriorityTakeover: { + case StartElectionReasonEnum::kPriorityTakeover: { ElectionReasonCounter& priorityTakeover = _electionMetrics.getPriorityTakeover(); priorityTakeover.incrementSuccessful(); break; } - case TopologyCoordinator::StartElectionReason::kCatchupTakeover: { + case StartElectionReasonEnum::kCatchupTakeover: { ElectionReasonCounter& catchUpTakeover = _electionMetrics.getCatchUpTakeover(); catchUpTakeover.incrementSuccessful(); break; } - case TopologyCoordinator::StartElectionReason::kElectionTimeout: { + case StartElectionReasonEnum::kElectionTimeout: { ElectionReasonCounter& electionTimeout = _electionMetrics.getElectionTimeout(); electionTimeout.incrementSuccessful(); break; } - case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: { + case StartElectionReasonEnum::kSingleNodePromptElection: { ElectionReasonCounter& freezeTimeout = _electionMetrics.getFreezeTimeout(); freezeTimeout.incrementSuccessful(); break; @@ -263,10 +261,30 @@ long ReplicationMetrics::getNumCatchUpsFailedWithReplSetAbortPrimaryCatchUpCmd_f return _electionMetrics.getNumCatchUpsFailedWithReplSetAbortPrimaryCatchUpCmd(); } -void ReplicationMetrics::setElectionCandidateMetrics(Date_t lastElectionDate) { +void ReplicationMetrics::setElectionCandidateMetrics( + const StartElectionReasonEnum reason, + const Date_t lastElectionDate, + const long long termAtElection, + const OpTime lastCommittedOpTime, + const OpTime lastSeenOpTime, + const int numVotesNeeded, + const double priorityAtElection, + const Milliseconds electionTimeout, + const boost::optional<int> priorPrimaryMemberId) { + stdx::lock_guard<stdx::mutex> lk(_mutex); - _electionCandidateMetrics.setLastElectionDate(lastElectionDate); + _nodeIsCandidateOrPrimary = true; + _electionCandidateMetrics.setLastElectionReason(reason); + _electionCandidateMetrics.setLastElectionDate(lastElectionDate); + _electionCandidateMetrics.setTermAtElection(termAtElection); + _electionCandidateMetrics.setLastCommittedOpTimeAtElection(lastCommittedOpTime); + _electionCandidateMetrics.setLastSeenOpTimeAtElection(lastSeenOpTime); + _electionCandidateMetrics.setNumVotesNeeded(numVotesNeeded); + _electionCandidateMetrics.setPriorityAtElection(priorityAtElection); + long long electionTimeoutMillis = durationCount<Milliseconds>(electionTimeout); + _electionCandidateMetrics.setElectionTimeoutMillis(electionTimeoutMillis); + _electionCandidateMetrics.setPriorPrimaryMemberId(priorPrimaryMemberId); } void ReplicationMetrics::setTargetCatchupOpTime(OpTime opTime) { diff --git a/src/mongo/db/repl/replication_metrics.h b/src/mongo/db/repl/replication_metrics.h index 816b1fc39bc..0d8c025e18b 100644 --- a/src/mongo/db/repl/replication_metrics.h +++ b/src/mongo/db/repl/replication_metrics.h @@ -49,8 +49,8 @@ public: ~ReplicationMetrics(); // Election metrics - void incrementNumElectionsCalledForReason(TopologyCoordinator::StartElectionReason reason); - void incrementNumElectionsSuccessfulForReason(TopologyCoordinator::StartElectionReason reason); + void incrementNumElectionsCalledForReason(StartElectionReasonEnum reason); + void incrementNumElectionsSuccessfulForReason(StartElectionReasonEnum reason); void incrementNumStepDownsCausedByHigherTerm(); void incrementNumCatchUps(); void incrementNumCatchUpsConcludedForReason( @@ -81,7 +81,15 @@ public: // All the election candidate metrics that should be set when a node calls an election are set // in this one function, so that the 'electionCandidateMetrics' section of replSetStatus shows a // consistent state. - void setElectionCandidateMetrics(Date_t lastElectionDate); + void setElectionCandidateMetrics(const StartElectionReasonEnum reason, + const Date_t lastElectionDate, + const long long termAtElection, + const OpTime lastCommittedOpTime, + const OpTime lastSeenOpTime, + const int numVotesNeeded, + const double priorityAtElection, + const Milliseconds electionTimeoutMillis, + const boost::optional<int> priorPrimary); void setTargetCatchupOpTime(OpTime opTime); void setNumCatchUpOps(int numCatchUpOps); void setNewTermStartDate(Date_t newTermStartDate); diff --git a/src/mongo/db/repl/replication_metrics.idl b/src/mongo/db/repl/replication_metrics.idl index 6c5746cef92..e214c2b0dc8 100644 --- a/src/mongo/db/repl/replication_metrics.idl +++ b/src/mongo/db/repl/replication_metrics.idl @@ -40,6 +40,18 @@ imports: - "mongo/idl/basic_types.idl" - "mongo/db/repl/replication_types.idl" +enums: + StartElectionReason: + description: "Reason why a node called for an election" + type: string + values: + kElectionTimeout: "electionTimeout" + kPriorityTakeover: "priorityTakeover" + kStepUpRequest: "stepUpRequest" + kStepUpRequestSkipDryRun: "stepUpRequestSkipDryRun" + kCatchupTakeover: "catchupTakeover" + kSingleNodePromptElection: "singleNodePromptElection" + types: ElectionReasonCounter: bson_serialization_type: any @@ -126,10 +138,35 @@ structs: candidate" strict: true fields: + lastElectionReason: + description: "Reason why the node called for the election" + type: StartElectionReason lastElectionDate: - description: "Time the node called the dry run election, or the actual election if - it skipped dry run" + description: "Time the node called for the election" type: date + termAtElection: + description: "New term at the time of election" + type: long + lastCommittedOpTimeAtElection: + description: "Last OpTime the node committed before calling the election" + type: optime + lastSeenOpTimeAtElection: + description: "Last OpTime the node saw before calling the election" + type: optime + numVotesNeeded: + description: "Number of votes the node needed to win the election" + type: int + priorityAtElection: + description: "Priority of the node at the time of the election" + type: double + electionTimeoutMillis: + description: "Timeout duration before a node initiates a new election" + type: long + priorPrimaryMemberId: + description: "MemberId of the prior primary, according to the node. Not set if + there was no prior primary" + type: int + optional: true targetCatchupOpTime: description: "The node's target opTime for catchup" type: optime diff --git a/src/mongo/db/repl/topology_coordinator.cpp b/src/mongo/db/repl/topology_coordinator.cpp index 9d4e1235ba6..8ee1a117226 100644 --- a/src/mongo/db/repl/topology_coordinator.cpp +++ b/src/mongo/db/repl/topology_coordinator.cpp @@ -48,12 +48,8 @@ #include "mongo/db/mongod_options.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/heartbeat_response_action.h" -#include "mongo/db/repl/is_master_response.h" #include "mongo/db/repl/isself.h" #include "mongo/db/repl/member_data.h" -#include "mongo/db/repl/repl_set_heartbeat_args_v1.h" -#include "mongo/db/repl/repl_set_heartbeat_response.h" -#include "mongo/db/repl/repl_set_request_votes_args.h" #include "mongo/db/repl/rslog.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -922,7 +918,8 @@ bool TopologyCoordinator::setMemberAsDown(Date_t now, const int memberIndex) { MemberData& hbData = _memberData.at(memberIndex); hbData.setDownValues(now, "no response within election timeout period"); - if (CannotSeeMajority & _getMyUnelectableReason(now, StartElectionReason::kElectionTimeout)) { + if (CannotSeeMajority & + _getMyUnelectableReason(now, StartElectionReasonEnum::kElectionTimeout)) { return true; } @@ -1213,7 +1210,7 @@ int TopologyCoordinator::_findHealthyPrimaryOfEqualOrGreaterPriority( } bool TopologyCoordinator::_amIFreshEnoughForPriorityTakeover() const { - const OpTime latestKnownOpTime = _latestKnownOpTime(); + const OpTime ourLatestKnownOpTime = latestKnownOpTime(); // Rules are: // - If the terms don't match, we don't call for priority takeover. @@ -1227,23 +1224,24 @@ bool TopologyCoordinator::_amIFreshEnoughForPriorityTakeover() const { // passes the timestamp component of the last oplog entry. const OpTime ourLastOpApplied = getMyLastAppliedOpTime(); - if (ourLastOpApplied.getTerm() != latestKnownOpTime.getTerm()) { + if (ourLastOpApplied.getTerm() != ourLatestKnownOpTime.getTerm()) { return false; } - if (ourLastOpApplied.getTimestamp().getSecs() != latestKnownOpTime.getTimestamp().getSecs()) { + if (ourLastOpApplied.getTimestamp().getSecs() != + ourLatestKnownOpTime.getTimestamp().getSecs()) { return ourLastOpApplied.getTimestamp().getSecs() + gPriorityTakeoverFreshnessWindowSeconds >= - latestKnownOpTime.getTimestamp().getSecs(); + ourLatestKnownOpTime.getTimestamp().getSecs(); } else { return ourLastOpApplied.getTimestamp().getInc() + 1000 >= - latestKnownOpTime.getTimestamp().getInc(); + ourLatestKnownOpTime.getTimestamp().getInc(); } } bool TopologyCoordinator::_amIFreshEnoughForCatchupTakeover() const { - const OpTime latestKnownOpTime = _latestKnownOpTime(); + const OpTime ourLatestKnownOpTime = latestKnownOpTime(); // Rules are: // - We must have the freshest optime of all the up nodes. @@ -1255,7 +1253,7 @@ bool TopologyCoordinator::_amIFreshEnoughForCatchupTakeover() const { // There is no point to a catchup takeover if we aren't the freshest node because // another node would immediately perform another catchup takeover when we become primary. const OpTime ourLastOpApplied = getMyLastAppliedOpTime(); - if (ourLastOpApplied < latestKnownOpTime) { + if (ourLastOpApplied < ourLatestKnownOpTime) { return false; } @@ -1284,34 +1282,6 @@ bool TopologyCoordinator::_iAmPrimary() const { return false; } -OpTime TopologyCoordinator::_latestKnownOpTime() const { - OpTime latest = getMyLastAppliedOpTime(); - for (std::vector<MemberData>::const_iterator it = _memberData.begin(); it != _memberData.end(); - ++it) { - // Ignore self - // TODO(russotto): Simplify when heartbeat and spanning tree times are combined. - if (it->isSelf()) { - continue; - } - // Ignore down members - if (!it->up()) { - continue; - } - // Ignore removed nodes (not in config, so not valid). - if (it->getState().removed()) { - continue; - } - - OpTime optime = it->getHeartbeatAppliedOpTime(); - - if (optime > latest) { - latest = optime; - } - } - - return latest; -} - bool TopologyCoordinator::prepareForUnconditionalStepDown() { if (_leaderMode == LeaderMode::kSteppingDown) { // Can only be processing one required stepdown at a time. @@ -1983,7 +1953,7 @@ TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getUnelectableR } TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getMyUnelectableReason( - const Date_t now, StartElectionReason reason) const { + const Date_t now, StartElectionReasonEnum reason) const { UnelectableReasonMask result = None; const OpTime lastApplied = getMyLastAppliedOpTime(); if (lastApplied.isNull()) { @@ -2011,11 +1981,13 @@ TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getMyUnelectabl result |= NotSecondary; } - if (reason == StartElectionReason::kPriorityTakeover && !_amIFreshEnoughForPriorityTakeover()) { + if (reason == StartElectionReasonEnum::kPriorityTakeover && + !_amIFreshEnoughForPriorityTakeover()) { result |= NotCloseEnoughToLatestForPriorityTakeover; } - if (reason == StartElectionReason::kCatchupTakeover && !_amIFreshEnoughForCatchupTakeover()) { + if (reason == StartElectionReasonEnum::kCatchupTakeover && + !_amIFreshEnoughForCatchupTakeover()) { result |= NotFreshEnoughForCatchupTakeover; } return result; @@ -2781,7 +2753,7 @@ void TopologyCoordinator::setPrimaryIndex(long long primaryIndex) { } Status TopologyCoordinator::becomeCandidateIfElectable(const Date_t now, - StartElectionReason reason) { + StartElectionReasonEnum reason) { if (_role == Role::kLeader) { return {ErrorCodes::NodeNotElectable, "Not standing for election again; already primary"}; } @@ -2814,6 +2786,33 @@ void TopologyCoordinator::restartHeartbeats() { } } +OpTime TopologyCoordinator::latestKnownOpTime() const { + OpTime latest = getMyLastAppliedOpTime(); + for (std::vector<MemberData>::const_iterator it = _memberData.begin(); it != _memberData.end(); + ++it) { + // Ignore self + if (it->isSelf()) { + continue; + } + // Ignore down members + if (!it->up()) { + continue; + } + // Ignore removed nodes (not in config, so not valid). + if (it->getState().removed()) { + continue; + } + + OpTime optime = it->getHeartbeatAppliedOpTime(); + + if (optime > latest) { + latest = optime; + } + } + + return latest; +} + boost::optional<OpTime> TopologyCoordinator::latestKnownOpTimeSinceHeartbeatRestart() const { // The smallest OpTime in PV1. OpTime latest(Timestamp(0, 0), 0); diff --git a/src/mongo/db/repl/topology_coordinator.h b/src/mongo/db/repl/topology_coordinator.h index e29453292a4..e23f820fef3 100644 --- a/src/mongo/db/repl/topology_coordinator.h +++ b/src/mongo/db/repl/topology_coordinator.h @@ -32,9 +32,13 @@ #include <iosfwd> #include <string> +#include "mongo/db/repl/is_master_response.h" #include "mongo/db/repl/last_vote.h" +#include "mongo/db/repl/repl_set_heartbeat_args_v1.h" #include "mongo/db/repl/repl_set_heartbeat_response.h" +#include "mongo/db/repl/repl_set_request_votes_args.h" #include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/repl/replication_metrics_gen.h" #include "mongo/db/repl/split_horizon.h" #include "mongo/db/repl/update_position_args.h" #include "mongo/db/server_options.h" @@ -650,19 +654,10 @@ public: */ int getCurrentPrimaryIndex() const; - enum StartElectionReason { - kElectionTimeout, - kPriorityTakeover, - kStepUpRequest, - kStepUpRequestSkipDryRun, - kCatchupTakeover, - kSingleNodePromptElection - }; - /** * Transitions to the candidate role if the node is electable. */ - Status becomeCandidateIfElectable(const Date_t now, StartElectionReason reason); + Status becomeCandidateIfElectable(const Date_t now, StartElectionReasonEnum reason); /** * Updates the storage engine read committed support in the TopologyCoordinator options after @@ -675,6 +670,9 @@ public: */ void restartHeartbeats(); + // Scans through all members that are 'up' and returns the latest known optime. + OpTime latestKnownOpTime() const; + /** * Scans through all members that are 'up' and return the latest known optime, if we have * received (successful or failed) heartbeats from all nodes since heartbeat restart. @@ -814,7 +812,7 @@ private: // Returns reason why "self" member is unelectable UnelectableReasonMask _getMyUnelectableReason(const Date_t now, - StartElectionReason reason) const; + StartElectionReasonEnum reason) const; // Returns reason why memberIndex is unelectable UnelectableReasonMask _getUnelectableReason(int memberIndex) const; @@ -825,9 +823,6 @@ private: // Return true if we are currently primary bool _iAmPrimary() const; - // Scans through all members that are 'up' and return the latest known optime. - OpTime _latestKnownOpTime() const; - // Helper shortcut to self config const MemberConfig& _selfConfig() const; diff --git a/src/mongo/db/repl/topology_coordinator_v1_test.cpp b/src/mongo/db/repl/topology_coordinator_v1_test.cpp index 9dd69ddd6cb..04d0a152219 100644 --- a/src/mongo/db/repl/topology_coordinator_v1_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_v1_test.cpp @@ -2486,7 +2486,7 @@ TEST_F(TopoCoordTest, NodeReturnsArbiterWhenGetMemberStateRunsAgainstArbiter) { TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileRemovedFromTheConfig) { const auto status = getTopoCoord().becomeCandidateIfElectable( - now()++, TopologyCoordinator::StartElectionReason::kElectionTimeout); + now()++, StartElectionReasonEnum::kElectionTimeout); ASSERT_NOT_OK(status); ASSERT_STRING_CONTAINS(status.reason(), "not a member of a valid replica set config"); } @@ -3690,8 +3690,8 @@ TEST_F(TopoCoordTest, FreshestNodeDoesCatchupTakeover) { StatusWith<ReplSetHeartbeatResponse>(hbResp)); getTopoCoord().updateTerm(1, Date_t()); - ASSERT_OK(getTopoCoord().becomeCandidateIfElectable( - Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover)); + ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(Date_t(), + StartElectionReasonEnum::kCatchupTakeover)); } TEST_F(TopoCoordTest, StaleNodeDoesntDoCatchupTakeover) { @@ -3742,7 +3742,7 @@ TEST_F(TopoCoordTest, StaleNodeDoesntDoCatchupTakeover) { getTopoCoord().updateTerm(1, Date_t()); Status result = getTopoCoord().becomeCandidateIfElectable( - Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover); + Date_t(), StartElectionReasonEnum::kCatchupTakeover); ASSERT_NOT_OK(result); ASSERT_STRING_CONTAINS(result.reason(), "member is either not the most up-to-date member or not ahead of the " @@ -3794,7 +3794,7 @@ TEST_F(TopoCoordTest, NodeDoesntDoCatchupTakeoverHeartbeatSaysPrimaryCaughtUp) { getTopoCoord().updateTerm(1, Date_t()); Status result = getTopoCoord().becomeCandidateIfElectable( - Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover); + Date_t(), StartElectionReasonEnum::kCatchupTakeover); ASSERT_NOT_OK(result); ASSERT_STRING_CONTAINS(result.reason(), "member is either not the most up-to-date member or not ahead of the " @@ -3851,7 +3851,7 @@ TEST_F(TopoCoordTest, NodeDoesntDoCatchupTakeoverIfTermNumbersSayPrimaryCaughtUp getTopoCoord().updateTerm(1, Date_t()); Status result = getTopoCoord().becomeCandidateIfElectable( - Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover); + Date_t(), StartElectionReasonEnum::kCatchupTakeover); ASSERT_NOT_OK(result); ASSERT_STRING_CONTAINS(result.reason(), "member is either not the most up-to-date member or not ahead of the " @@ -5005,8 +5005,8 @@ TEST_F(HeartbeatResponseTestV1, ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole()); // We are electable now. - ASSERT_OK(getTopoCoord().becomeCandidateIfElectable( - now(), TopologyCoordinator::StartElectionReason::kElectionTimeout)); + ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(), + StartElectionReasonEnum::kElectionTimeout)); ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole()); } @@ -5031,8 +5031,8 @@ TEST_F(HeartbeatResponseTestV1, ScheduleElectionWhenPrimaryIsMarkedDownAndWeAreE ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole()); // We are electable now. - ASSERT_OK(getTopoCoord().becomeCandidateIfElectable( - now(), TopologyCoordinator::StartElectionReason::kElectionTimeout)); + ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(), + StartElectionReasonEnum::kElectionTimeout)); ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole()); } |