From 87c9601a762048ea93559b13c3b6043b4017b024 Mon Sep 17 00:00:00 2001 From: Xuerui Fa Date: Tue, 24 Sep 2019 22:05:25 +0000 Subject: SERVER-41506 Added tracking for metrics around a nodes calling an election --- src/mongo/db/repl/replication_coordinator_impl.cpp | 12 ++- src/mongo/db/repl/replication_coordinator_impl.h | 20 ++--- .../repl/replication_coordinator_impl_elect_v1.cpp | 54 +++++++++----- .../replication_coordinator_impl_elect_v1_test.cpp | 35 ++++----- .../replication_coordinator_impl_heartbeat.cpp | 41 +++++----- src/mongo/db/repl/replication_metrics.cpp | 54 +++++++++----- src/mongo/db/repl/replication_metrics.h | 14 +++- src/mongo/db/repl/replication_metrics.idl | 41 +++++++++- src/mongo/db/repl/topology_coordinator.cpp | 87 +++++++++++----------- src/mongo/db/repl/topology_coordinator.h | 23 +++--- src/mongo/db/repl/topology_coordinator_v1_test.cpp | 20 ++--- 11 files changed, 232 insertions(+), 169 deletions(-) (limited to 'src/mongo') diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 8587b3a56a0..1430bc05faa 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -2185,8 +2185,7 @@ void ReplicationCoordinatorImpl::_handleTimePassing( // For election protocol v1, call _startElectSelfIfEligibleV1 to avoid race // against other elections caused by events like election timeout, replSetStepUp etc. - _startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason::kSingleNodePromptElection); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kSingleNodePromptElection); } bool ReplicationCoordinatorImpl::isMasterForReportingPurposes() { @@ -2552,8 +2551,7 @@ Status ReplicationCoordinatorImpl::processReplSetFreeze(int secs, BSONObjBuilder result.getValue()) { // For election protocol v1, call _startElectSelfIfEligibleV1 to avoid race // against other elections caused by events like election timeout, replSetStepUp etc. - _startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason::kSingleNodePromptElection); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kSingleNodePromptElection); } return Status::OK(); @@ -3005,7 +3003,7 @@ void ReplicationCoordinatorImpl::_performPostMemberStateUpdateAction( case kActionStartSingleNodeElection: // In protocol version 1, single node replset will run an election instead of // kActionWinElection as in protocol version 0. - _startElectSelfV1(TopologyCoordinator::StartElectionReason::kElectionTimeout); + _startElectSelfV1(StartElectionReasonEnum::kElectionTimeout); break; default: severe() << "Unknown post member state update action " << static_cast(action); @@ -4116,8 +4114,8 @@ CallbackFn ReplicationCoordinatorImpl::_wrapAsCallbackFn(const std::function lk(_mutex); _startElectSelfV1_inlock(reason); } -void ReplicationCoordinatorImpl::_startElectSelfV1_inlock( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(StartElectionReasonEnum reason) { invariant(!_voteRequester); switch (_rsConfigState) { @@ -146,10 +143,7 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock( long long term = _topCoord->getTerm(); int primaryIndex = -1; - Date_t now = _replExecutor->now(); - ReplicationMetrics::get(getServiceContext()).setElectionCandidateMetrics(now); - - if (reason == TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun) { + if (reason == StartElectionReasonEnum::kStepUpRequestSkipDryRun) { long long newTerm = term + 1; log() << "skipping dry run and running for election in term " << newTerm; _startRealElection_inlock(newTerm, reason); @@ -161,7 +155,7 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock( _voteRequester.reset(new VoteRequester); // Only set primaryIndex if the primary's vote is required during the dry run. - if (reason == TopologyCoordinator::StartElectionReason::kCatchupTakeover) { + if (reason == StartElectionReasonEnum::kCatchupTakeover) { primaryIndex = _topCoord->getCurrentPrimaryIndex(); } StatusWith nextPhaseEvh = @@ -185,8 +179,8 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock( lossGuard.dismiss(); } -void ReplicationCoordinatorImpl::_processDryRunResult( - long long originalTerm, TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_processDryRunResult(long long originalTerm, + StartElectionReasonEnum reason) { stdx::lock_guard lk(_mutex); LoseElectionDryRunGuardV1 lossGuard(this); @@ -221,8 +215,30 @@ void ReplicationCoordinatorImpl::_processDryRunResult( lossGuard.dismiss(); } -void ReplicationCoordinatorImpl::_startRealElection_inlock( - long long newTerm, TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startRealElection_inlock(long long newTerm, + StartElectionReasonEnum reason) { + + const Date_t now = _replExecutor->now(); + const OpTime lastCommittedOpTime = _topCoord->getLastCommittedOpTime(); + const OpTime lastSeenOpTime = _topCoord->latestKnownOpTime(); + const int numVotesNeeded = _rsConfig.getMajorityVoteCount(); + const double priorityAtElection = _rsConfig.getMemberAt(_selfIndex).getPriority(); + const Milliseconds electionTimeoutMillis = _rsConfig.getElectionTimeoutPeriod(); + const int priorPrimaryIndex = _topCoord->getCurrentPrimaryIndex(); + const boost::optional priorPrimaryMemberId = (priorPrimaryIndex == -1) + ? boost::none + : boost::make_optional(_rsConfig.getMemberAt(priorPrimaryIndex).getId().getData()); + + ReplicationMetrics::get(getServiceContext()) + .setElectionCandidateMetrics(reason, + now, + newTerm, + lastCommittedOpTime, + lastSeenOpTime, + numVotesNeeded, + priorityAtElection, + electionTimeoutMillis, + priorPrimaryMemberId); ReplicationMetrics::get(getServiceContext()).incrementNumElectionsCalledForReason(reason); LoseElectionDryRunGuardV1 lossGuard(this); @@ -254,7 +270,7 @@ void ReplicationCoordinatorImpl::_startRealElection_inlock( void ReplicationCoordinatorImpl::_writeLastVoteForMyElection( LastVote lastVote, const executor::TaskExecutor::CallbackArgs& cbData, - TopologyCoordinator::StartElectionReason reason) { + StartElectionReasonEnum reason) { // storeLocalLastVoteDocument can call back in to the replication coordinator, // so _mutex must be unlocked here. However, we cannot return until we // lock it because we want to lose the election on cancel or error and @@ -292,8 +308,8 @@ void ReplicationCoordinatorImpl::_writeLastVoteForMyElection( lossGuard.dismiss(); } -void ReplicationCoordinatorImpl::_startVoteRequester_inlock( - long long newTerm, TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startVoteRequester_inlock(long long newTerm, + StartElectionReasonEnum reason) { const auto lastOpTime = _getMyLastAppliedOpTime_inlock(); _voteRequester.reset(new VoteRequester); @@ -313,8 +329,8 @@ void ReplicationCoordinatorImpl::_startVoteRequester_inlock( MONGO_FAIL_POINT_DEFINE(electionHangsBeforeUpdateMemberState); -void ReplicationCoordinatorImpl::_onVoteRequestComplete( - long long newTerm, TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long newTerm, + StartElectionReasonEnum reason) { stdx::lock_guard lk(_mutex); LoseElectionGuardV1 lossGuard(this); diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp index c51f4ace311..2f9344f44f9 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp @@ -374,9 +374,9 @@ TEST_F(ReplCoordTest, ElectionFailsWhenInsufficientVotesAreReceivedDuringDryRun) << false << "reason" << "don't like him much"))); voteRequests++; - // Check that the node's election candidate metrics are set once it has called an - // election. - ASSERT_BSONOBJ_NE( + + // Check that the node's election candidate metrics are not set if a dry run fails. + ASSERT_BSONOBJ_EQ( BSONObj(), ReplicationMetrics::get(getServiceContext()).getElectionCandidateMetricsBSON()); } else { @@ -879,7 +879,7 @@ public: } void performSuccessfulTakeover(Date_t takeoverTime, - TopologyCoordinator::StartElectionReason reason, + StartElectionReasonEnum reason, const LastVote& lastVoteExpected) { startCapturingLogMessages(); simulateSuccessfulV1ElectionAt(takeoverTime); @@ -894,7 +894,7 @@ public: ASSERT_EQ(lastVoteExpected.getCandidateIndex(), lastVote.getValue().getCandidateIndex()); ASSERT_EQ(lastVoteExpected.getTerm(), lastVote.getValue().getTerm()); - if (reason == TopologyCoordinator::StartElectionReason::kPriorityTakeover) { + if (reason == StartElectionReasonEnum::kPriorityTakeover) { ASSERT_EQUALS(1, countLogLinesContaining("Starting an election for a priority takeover")); } @@ -1444,9 +1444,8 @@ TEST_F(TakeoverTest, SuccessfulCatchupTakeover) { ASSERT_EQUALS(1, countLogLinesContaining("Starting an election for a catchup takeover")); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(catchupTakeoverTime, - TopologyCoordinator::StartElectionReason::kCatchupTakeover, - lastVoteExpected); + performSuccessfulTakeover( + catchupTakeoverTime, StartElectionReasonEnum::kCatchupTakeover, lastVoteExpected); // Check that the numCatchUpTakeoversCalled and the numCatchUpTakeoversSuccessful election // metrics have been incremented, and that none of the metrics that track the number of @@ -1703,9 +1702,8 @@ TEST_F(TakeoverTest, PrimaryCatchesUpBeforeHighPriorityNodeCatchupTakeover) { config, now + longElectionTimeout, HostAndPort("node2", 12345), currentOptime); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(priorityTakeoverTime, - TopologyCoordinator::StartElectionReason::kPriorityTakeover, - lastVoteExpected); + performSuccessfulTakeover( + priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected); } TEST_F(TakeoverTest, SchedulesPriorityTakeoverIfNodeHasHigherPriorityThanCurrentPrimary) { @@ -1797,9 +1795,8 @@ TEST_F(TakeoverTest, SuccessfulPriorityTakeover) { config, now + halfElectionTimeout, HostAndPort("node2", 12345), myOptime); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(priorityTakeoverTime, - TopologyCoordinator::StartElectionReason::kPriorityTakeover, - lastVoteExpected); + performSuccessfulTakeover( + priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected); // Check that the numPriorityTakeoversCalled and the numPriorityTakeoversSuccessful election // metrics have been incremented, and that none of the metrics that track the number of @@ -1888,9 +1885,8 @@ TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedSameSecond) { Date_t() + Seconds(closeEnoughOpTime.getSecs())); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(priorityTakeoverTime, - TopologyCoordinator::StartElectionReason::kPriorityTakeover, - lastVoteExpected); + performSuccessfulTakeover( + priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected); } TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedDifferentSecond) { @@ -1963,9 +1959,8 @@ TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedDifferentSecond) { Date_t() + Seconds(closeEnoughOpTime.getSecs())); LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0); - performSuccessfulTakeover(priorityTakeoverTime, - TopologyCoordinator::StartElectionReason::kPriorityTakeover, - lastVoteExpected); + performSuccessfulTakeover( + priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected); } TEST_F(ReplCoordTest, NodeCancelsElectionUponReceivingANewConfigDuringDryRun) { diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index 4be3daac838..5ce1a02042e 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -287,8 +287,7 @@ stdx::unique_lock ReplicationCoordinatorImpl::_handleHeartbeatResponseAct LOG_FOR_ELECTION(0) << "Scheduling priority takeover at " << _priorityTakeoverWhen; _priorityTakeoverCbh = _scheduleWorkAt( _priorityTakeoverWhen, [=](const mongo::executor::TaskExecutor::CallbackArgs&) { - _startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason::kPriorityTakeover); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kPriorityTakeover); }); } break; @@ -301,8 +300,7 @@ stdx::unique_lock ReplicationCoordinatorImpl::_handleHeartbeatResponseAct LOG_FOR_ELECTION(0) << "Scheduling catchup takeover at " << _catchupTakeoverWhen; _catchupTakeoverCbh = _scheduleWorkAt( _catchupTakeoverWhen, [=](const mongo::executor::TaskExecutor::CallbackArgs&) { - _startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason::kCatchupTakeover); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kCatchupTakeover); }); } break; @@ -858,15 +856,14 @@ void ReplicationCoordinatorImpl::_cancelAndRescheduleElectionTimeout_inlock() { _handleElectionTimeoutWhen = when; _handleElectionTimeoutCbh = _scheduleWorkAt(when, [=](const mongo::executor::TaskExecutor::CallbackArgs&) { - _startElectSelfIfEligibleV1(TopologyCoordinator::StartElectionReason::kElectionTimeout); + _startElectSelfIfEligibleV1(StartElectionReasonEnum::kElectionTimeout); }); } -void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1(StartElectionReasonEnum reason) { stdx::lock_guard lock(_mutex); // If it is not a single node replica set, no need to start an election after stepdown timeout. - if (reason == TopologyCoordinator::StartElectionReason::kSingleNodePromptElection && + if (reason == StartElectionReasonEnum::kSingleNodePromptElection && _rsConfig.getNumMembers() != 1) { return; } @@ -886,52 +883,56 @@ void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1( const auto status = _topCoord->becomeCandidateIfElectable(_replExecutor->now(), reason); if (!status.isOK()) { switch (reason) { - case TopologyCoordinator::StartElectionReason::kElectionTimeout: + case StartElectionReasonEnum::kElectionTimeout: LOG_FOR_ELECTION(0) << "Not starting an election, since we are not electable due to: " << status.reason(); break; - case TopologyCoordinator::StartElectionReason::kPriorityTakeover: + case StartElectionReasonEnum::kPriorityTakeover: LOG_FOR_ELECTION(0) << "Not starting an election for a priority takeover, " << "since we are not electable due to: " << status.reason(); break; - case TopologyCoordinator::StartElectionReason::kStepUpRequest: - case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: + case StartElectionReasonEnum::kStepUpRequest: + case StartElectionReasonEnum::kStepUpRequestSkipDryRun: LOG_FOR_ELECTION(0) << "Not starting an election for a replSetStepUp request, " << "since we are not electable due to: " << status.reason(); break; - case TopologyCoordinator::StartElectionReason::kCatchupTakeover: + case StartElectionReasonEnum::kCatchupTakeover: LOG_FOR_ELECTION(0) << "Not starting an election for a catchup takeover, " << "since we are not electable due to: " << status.reason(); break; - case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: + case StartElectionReasonEnum::kSingleNodePromptElection: LOG_FOR_ELECTION(0) << "Not starting an election for a single node replica set prompt election, " << "since we are not electable due to: " << status.reason(); break; + default: + MONGO_UNREACHABLE; } return; } switch (reason) { - case TopologyCoordinator::StartElectionReason::kElectionTimeout: + case StartElectionReasonEnum::kElectionTimeout: LOG_FOR_ELECTION(0) << "Starting an election, since we've seen no PRIMARY in the past " << _rsConfig.getElectionTimeoutPeriod(); break; - case TopologyCoordinator::StartElectionReason::kPriorityTakeover: + case StartElectionReasonEnum::kPriorityTakeover: LOG_FOR_ELECTION(0) << "Starting an election for a priority takeover"; break; - case TopologyCoordinator::StartElectionReason::kStepUpRequest: - case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: + case StartElectionReasonEnum::kStepUpRequest: + case StartElectionReasonEnum::kStepUpRequestSkipDryRun: LOG_FOR_ELECTION(0) << "Starting an election due to step up request"; break; - case TopologyCoordinator::StartElectionReason::kCatchupTakeover: + case StartElectionReasonEnum::kCatchupTakeover: LOG_FOR_ELECTION(0) << "Starting an election for a catchup takeover"; break; - case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: + case StartElectionReasonEnum::kSingleNodePromptElection: LOG_FOR_ELECTION(0) << "Starting an election due to single node replica set prompt election"; break; + default: + MONGO_UNREACHABLE; } _startElectSelfV1_inlock(reason); diff --git a/src/mongo/db/repl/replication_metrics.cpp b/src/mongo/db/repl/replication_metrics.cpp index 14f01452775..7c63a9baefd 100644 --- a/src/mongo/db/repl/replication_metrics.cpp +++ b/src/mongo/db/repl/replication_metrics.cpp @@ -56,32 +56,31 @@ ReplicationMetrics::ReplicationMetrics() ReplicationMetrics::~ReplicationMetrics() {} -void ReplicationMetrics::incrementNumElectionsCalledForReason( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationMetrics::incrementNumElectionsCalledForReason(StartElectionReasonEnum reason) { stdx::lock_guard lk(_mutex); switch (reason) { - case TopologyCoordinator::StartElectionReason::kStepUpRequest: - case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: { + case StartElectionReasonEnum::kStepUpRequest: + case StartElectionReasonEnum::kStepUpRequestSkipDryRun: { ElectionReasonCounter& stepUpCmd = _electionMetrics.getStepUpCmd(); stepUpCmd.incrementCalled(); break; } - case TopologyCoordinator::StartElectionReason::kPriorityTakeover: { + case StartElectionReasonEnum::kPriorityTakeover: { ElectionReasonCounter& priorityTakeover = _electionMetrics.getPriorityTakeover(); priorityTakeover.incrementCalled(); break; } - case TopologyCoordinator::StartElectionReason::kCatchupTakeover: { + case StartElectionReasonEnum::kCatchupTakeover: { ElectionReasonCounter& catchUpTakeover = _electionMetrics.getCatchUpTakeover(); catchUpTakeover.incrementCalled(); break; } - case TopologyCoordinator::StartElectionReason::kElectionTimeout: { + case StartElectionReasonEnum::kElectionTimeout: { ElectionReasonCounter& electionTimeout = _electionMetrics.getElectionTimeout(); electionTimeout.incrementCalled(); break; } - case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: { + case StartElectionReasonEnum::kSingleNodePromptElection: { ElectionReasonCounter& freezeTimeout = _electionMetrics.getFreezeTimeout(); freezeTimeout.incrementCalled(); break; @@ -89,32 +88,31 @@ void ReplicationMetrics::incrementNumElectionsCalledForReason( } } -void ReplicationMetrics::incrementNumElectionsSuccessfulForReason( - TopologyCoordinator::StartElectionReason reason) { +void ReplicationMetrics::incrementNumElectionsSuccessfulForReason(StartElectionReasonEnum reason) { stdx::lock_guard lk(_mutex); switch (reason) { - case TopologyCoordinator::StartElectionReason::kStepUpRequest: - case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: { + case StartElectionReasonEnum::kStepUpRequest: + case StartElectionReasonEnum::kStepUpRequestSkipDryRun: { ElectionReasonCounter& stepUpCmd = _electionMetrics.getStepUpCmd(); stepUpCmd.incrementSuccessful(); break; } - case TopologyCoordinator::StartElectionReason::kPriorityTakeover: { + case StartElectionReasonEnum::kPriorityTakeover: { ElectionReasonCounter& priorityTakeover = _electionMetrics.getPriorityTakeover(); priorityTakeover.incrementSuccessful(); break; } - case TopologyCoordinator::StartElectionReason::kCatchupTakeover: { + case StartElectionReasonEnum::kCatchupTakeover: { ElectionReasonCounter& catchUpTakeover = _electionMetrics.getCatchUpTakeover(); catchUpTakeover.incrementSuccessful(); break; } - case TopologyCoordinator::StartElectionReason::kElectionTimeout: { + case StartElectionReasonEnum::kElectionTimeout: { ElectionReasonCounter& electionTimeout = _electionMetrics.getElectionTimeout(); electionTimeout.incrementSuccessful(); break; } - case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: { + case StartElectionReasonEnum::kSingleNodePromptElection: { ElectionReasonCounter& freezeTimeout = _electionMetrics.getFreezeTimeout(); freezeTimeout.incrementSuccessful(); break; @@ -263,10 +261,30 @@ long ReplicationMetrics::getNumCatchUpsFailedWithReplSetAbortPrimaryCatchUpCmd_f return _electionMetrics.getNumCatchUpsFailedWithReplSetAbortPrimaryCatchUpCmd(); } -void ReplicationMetrics::setElectionCandidateMetrics(Date_t lastElectionDate) { +void ReplicationMetrics::setElectionCandidateMetrics( + const StartElectionReasonEnum reason, + const Date_t lastElectionDate, + const long long termAtElection, + const OpTime lastCommittedOpTime, + const OpTime lastSeenOpTime, + const int numVotesNeeded, + const double priorityAtElection, + const Milliseconds electionTimeout, + const boost::optional priorPrimaryMemberId) { + stdx::lock_guard lk(_mutex); - _electionCandidateMetrics.setLastElectionDate(lastElectionDate); + _nodeIsCandidateOrPrimary = true; + _electionCandidateMetrics.setLastElectionReason(reason); + _electionCandidateMetrics.setLastElectionDate(lastElectionDate); + _electionCandidateMetrics.setTermAtElection(termAtElection); + _electionCandidateMetrics.setLastCommittedOpTimeAtElection(lastCommittedOpTime); + _electionCandidateMetrics.setLastSeenOpTimeAtElection(lastSeenOpTime); + _electionCandidateMetrics.setNumVotesNeeded(numVotesNeeded); + _electionCandidateMetrics.setPriorityAtElection(priorityAtElection); + long long electionTimeoutMillis = durationCount(electionTimeout); + _electionCandidateMetrics.setElectionTimeoutMillis(electionTimeoutMillis); + _electionCandidateMetrics.setPriorPrimaryMemberId(priorPrimaryMemberId); } void ReplicationMetrics::setTargetCatchupOpTime(OpTime opTime) { diff --git a/src/mongo/db/repl/replication_metrics.h b/src/mongo/db/repl/replication_metrics.h index a07f84e48cf..14e31f59eba 100644 --- a/src/mongo/db/repl/replication_metrics.h +++ b/src/mongo/db/repl/replication_metrics.h @@ -49,8 +49,8 @@ public: ~ReplicationMetrics(); // Election metrics - void incrementNumElectionsCalledForReason(TopologyCoordinator::StartElectionReason reason); - void incrementNumElectionsSuccessfulForReason(TopologyCoordinator::StartElectionReason reason); + void incrementNumElectionsCalledForReason(StartElectionReasonEnum reason); + void incrementNumElectionsSuccessfulForReason(StartElectionReasonEnum reason); void incrementNumStepDownsCausedByHigherTerm(); void incrementNumCatchUps(); void incrementNumCatchUpsConcludedForReason( @@ -81,7 +81,15 @@ public: // All the election candidate metrics that should be set when a node calls an election are set // in this one function, so that the 'electionCandidateMetrics' section of replSetStatus shows a // consistent state. - void setElectionCandidateMetrics(Date_t lastElectionDate); + void setElectionCandidateMetrics(const StartElectionReasonEnum reason, + const Date_t lastElectionDate, + const long long termAtElection, + const OpTime lastCommittedOpTime, + const OpTime lastSeenOpTime, + const int numVotesNeeded, + const double priorityAtElection, + const Milliseconds electionTimeoutMillis, + const boost::optional priorPrimary); void setTargetCatchupOpTime(OpTime opTime); void setNumCatchUpOps(int numCatchUpOps); void setNewTermStartDate(Date_t newTermStartDate); diff --git a/src/mongo/db/repl/replication_metrics.idl b/src/mongo/db/repl/replication_metrics.idl index 6c5746cef92..e214c2b0dc8 100644 --- a/src/mongo/db/repl/replication_metrics.idl +++ b/src/mongo/db/repl/replication_metrics.idl @@ -40,6 +40,18 @@ imports: - "mongo/idl/basic_types.idl" - "mongo/db/repl/replication_types.idl" +enums: + StartElectionReason: + description: "Reason why a node called for an election" + type: string + values: + kElectionTimeout: "electionTimeout" + kPriorityTakeover: "priorityTakeover" + kStepUpRequest: "stepUpRequest" + kStepUpRequestSkipDryRun: "stepUpRequestSkipDryRun" + kCatchupTakeover: "catchupTakeover" + kSingleNodePromptElection: "singleNodePromptElection" + types: ElectionReasonCounter: bson_serialization_type: any @@ -126,10 +138,35 @@ structs: candidate" strict: true fields: + lastElectionReason: + description: "Reason why the node called for the election" + type: StartElectionReason lastElectionDate: - description: "Time the node called the dry run election, or the actual election if - it skipped dry run" + description: "Time the node called for the election" type: date + termAtElection: + description: "New term at the time of election" + type: long + lastCommittedOpTimeAtElection: + description: "Last OpTime the node committed before calling the election" + type: optime + lastSeenOpTimeAtElection: + description: "Last OpTime the node saw before calling the election" + type: optime + numVotesNeeded: + description: "Number of votes the node needed to win the election" + type: int + priorityAtElection: + description: "Priority of the node at the time of the election" + type: double + electionTimeoutMillis: + description: "Timeout duration before a node initiates a new election" + type: long + priorPrimaryMemberId: + description: "MemberId of the prior primary, according to the node. Not set if + there was no prior primary" + type: int + optional: true targetCatchupOpTime: description: "The node's target opTime for catchup" type: optime diff --git a/src/mongo/db/repl/topology_coordinator.cpp b/src/mongo/db/repl/topology_coordinator.cpp index 1441a4cdeb5..77da01a202d 100644 --- a/src/mongo/db/repl/topology_coordinator.cpp +++ b/src/mongo/db/repl/topology_coordinator.cpp @@ -48,12 +48,8 @@ #include "mongo/db/mongod_options.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/heartbeat_response_action.h" -#include "mongo/db/repl/is_master_response.h" #include "mongo/db/repl/isself.h" #include "mongo/db/repl/member_data.h" -#include "mongo/db/repl/repl_set_heartbeat_args_v1.h" -#include "mongo/db/repl/repl_set_heartbeat_response.h" -#include "mongo/db/repl/repl_set_request_votes_args.h" #include "mongo/db/repl/rslog.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -922,7 +918,8 @@ bool TopologyCoordinator::setMemberAsDown(Date_t now, const int memberIndex) { MemberData& hbData = _memberData.at(memberIndex); hbData.setDownValues(now, "no response within election timeout period"); - if (CannotSeeMajority & _getMyUnelectableReason(now, StartElectionReason::kElectionTimeout)) { + if (CannotSeeMajority & + _getMyUnelectableReason(now, StartElectionReasonEnum::kElectionTimeout)) { return true; } @@ -1213,7 +1210,7 @@ int TopologyCoordinator::_findHealthyPrimaryOfEqualOrGreaterPriority( } bool TopologyCoordinator::_amIFreshEnoughForPriorityTakeover() const { - const OpTime latestKnownOpTime = _latestKnownOpTime(); + const OpTime ourLatestKnownOpTime = latestKnownOpTime(); // Rules are: // - If the terms don't match, we don't call for priority takeover. @@ -1227,23 +1224,24 @@ bool TopologyCoordinator::_amIFreshEnoughForPriorityTakeover() const { // passes the timestamp component of the last oplog entry. const OpTime ourLastOpApplied = getMyLastAppliedOpTime(); - if (ourLastOpApplied.getTerm() != latestKnownOpTime.getTerm()) { + if (ourLastOpApplied.getTerm() != ourLatestKnownOpTime.getTerm()) { return false; } - if (ourLastOpApplied.getTimestamp().getSecs() != latestKnownOpTime.getTimestamp().getSecs()) { + if (ourLastOpApplied.getTimestamp().getSecs() != + ourLatestKnownOpTime.getTimestamp().getSecs()) { return ourLastOpApplied.getTimestamp().getSecs() + gPriorityTakeoverFreshnessWindowSeconds >= - latestKnownOpTime.getTimestamp().getSecs(); + ourLatestKnownOpTime.getTimestamp().getSecs(); } else { return ourLastOpApplied.getTimestamp().getInc() + 1000 >= - latestKnownOpTime.getTimestamp().getInc(); + ourLatestKnownOpTime.getTimestamp().getInc(); } } bool TopologyCoordinator::_amIFreshEnoughForCatchupTakeover() const { - const OpTime latestKnownOpTime = _latestKnownOpTime(); + const OpTime ourLatestKnownOpTime = latestKnownOpTime(); // Rules are: // - We must have the freshest optime of all the up nodes. @@ -1255,7 +1253,7 @@ bool TopologyCoordinator::_amIFreshEnoughForCatchupTakeover() const { // There is no point to a catchup takeover if we aren't the freshest node because // another node would immediately perform another catchup takeover when we become primary. const OpTime ourLastOpApplied = getMyLastAppliedOpTime(); - if (ourLastOpApplied < latestKnownOpTime) { + if (ourLastOpApplied < ourLatestKnownOpTime) { return false; } @@ -1284,34 +1282,6 @@ bool TopologyCoordinator::_iAmPrimary() const { return false; } -OpTime TopologyCoordinator::_latestKnownOpTime() const { - OpTime latest = getMyLastAppliedOpTime(); - for (std::vector::const_iterator it = _memberData.begin(); it != _memberData.end(); - ++it) { - // Ignore self - // TODO(russotto): Simplify when heartbeat and spanning tree times are combined. - if (it->isSelf()) { - continue; - } - // Ignore down members - if (!it->up()) { - continue; - } - // Ignore removed nodes (not in config, so not valid). - if (it->getState().removed()) { - continue; - } - - OpTime optime = it->getHeartbeatAppliedOpTime(); - - if (optime > latest) { - latest = optime; - } - } - - return latest; -} - bool TopologyCoordinator::prepareForUnconditionalStepDown() { if (_leaderMode == LeaderMode::kSteppingDown) { // Can only be processing one required stepdown at a time. @@ -1981,7 +1951,7 @@ TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getUnelectableR } TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getMyUnelectableReason( - const Date_t now, StartElectionReason reason) const { + const Date_t now, StartElectionReasonEnum reason) const { UnelectableReasonMask result = None; const OpTime lastApplied = getMyLastAppliedOpTime(); if (lastApplied.isNull()) { @@ -2009,11 +1979,13 @@ TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getMyUnelectabl result |= NotSecondary; } - if (reason == StartElectionReason::kPriorityTakeover && !_amIFreshEnoughForPriorityTakeover()) { + if (reason == StartElectionReasonEnum::kPriorityTakeover && + !_amIFreshEnoughForPriorityTakeover()) { result |= NotCloseEnoughToLatestForPriorityTakeover; } - if (reason == StartElectionReason::kCatchupTakeover && !_amIFreshEnoughForCatchupTakeover()) { + if (reason == StartElectionReasonEnum::kCatchupTakeover && + !_amIFreshEnoughForCatchupTakeover()) { result |= NotFreshEnoughForCatchupTakeover; } return result; @@ -2779,7 +2751,7 @@ void TopologyCoordinator::setPrimaryIndex(long long primaryIndex) { } Status TopologyCoordinator::becomeCandidateIfElectable(const Date_t now, - StartElectionReason reason) { + StartElectionReasonEnum reason) { if (_role == Role::kLeader) { return {ErrorCodes::NodeNotElectable, "Not standing for election again; already primary"}; } @@ -2812,6 +2784,33 @@ void TopologyCoordinator::restartHeartbeats() { } } +OpTime TopologyCoordinator::latestKnownOpTime() const { + OpTime latest = getMyLastAppliedOpTime(); + for (std::vector::const_iterator it = _memberData.begin(); it != _memberData.end(); + ++it) { + // Ignore self + if (it->isSelf()) { + continue; + } + // Ignore down members + if (!it->up()) { + continue; + } + // Ignore removed nodes (not in config, so not valid). + if (it->getState().removed()) { + continue; + } + + OpTime optime = it->getHeartbeatAppliedOpTime(); + + if (optime > latest) { + latest = optime; + } + } + + return latest; +} + boost::optional TopologyCoordinator::latestKnownOpTimeSinceHeartbeatRestart() const { // The smallest OpTime in PV1. OpTime latest(Timestamp(0, 0), 0); diff --git a/src/mongo/db/repl/topology_coordinator.h b/src/mongo/db/repl/topology_coordinator.h index d81e9f52154..8747cd489de 100644 --- a/src/mongo/db/repl/topology_coordinator.h +++ b/src/mongo/db/repl/topology_coordinator.h @@ -33,9 +33,13 @@ #include #include +#include "mongo/db/repl/is_master_response.h" #include "mongo/db/repl/last_vote.h" +#include "mongo/db/repl/repl_set_heartbeat_args_v1.h" #include "mongo/db/repl/repl_set_heartbeat_response.h" +#include "mongo/db/repl/repl_set_request_votes_args.h" #include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/repl/replication_metrics_gen.h" #include "mongo/db/repl/split_horizon.h" #include "mongo/db/repl/update_position_args.h" #include "mongo/db/server_options.h" @@ -646,19 +650,10 @@ public: */ int getCurrentPrimaryIndex() const; - enum StartElectionReason { - kElectionTimeout, - kPriorityTakeover, - kStepUpRequest, - kStepUpRequestSkipDryRun, - kCatchupTakeover, - kSingleNodePromptElection - }; - /** * Transitions to the candidate role if the node is electable. */ - Status becomeCandidateIfElectable(const Date_t now, StartElectionReason reason); + Status becomeCandidateIfElectable(const Date_t now, StartElectionReasonEnum reason); /** * Updates the storage engine read committed support in the TopologyCoordinator options after @@ -671,6 +666,9 @@ public: */ void restartHeartbeats(); + // Scans through all members that are 'up' and returns the latest known optime. + OpTime latestKnownOpTime() const; + /** * Scans through all members that are 'up' and return the latest known optime, if we have * received (successful or failed) heartbeats from all nodes since heartbeat restart. @@ -810,7 +808,7 @@ private: // Returns reason why "self" member is unelectable UnelectableReasonMask _getMyUnelectableReason(const Date_t now, - StartElectionReason reason) const; + StartElectionReasonEnum reason) const; // Returns reason why memberIndex is unelectable UnelectableReasonMask _getUnelectableReason(int memberIndex) const; @@ -821,9 +819,6 @@ private: // Return true if we are currently primary bool _iAmPrimary() const; - // Scans through all members that are 'up' and return the latest known optime. - OpTime _latestKnownOpTime() const; - // Helper shortcut to self config const MemberConfig& _selfConfig() const; diff --git a/src/mongo/db/repl/topology_coordinator_v1_test.cpp b/src/mongo/db/repl/topology_coordinator_v1_test.cpp index 2a9eb7932f3..9c503dc6513 100644 --- a/src/mongo/db/repl/topology_coordinator_v1_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_v1_test.cpp @@ -2477,7 +2477,7 @@ TEST_F(TopoCoordTest, NodeReturnsArbiterWhenGetMemberStateRunsAgainstArbiter) { TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileRemovedFromTheConfig) { const auto status = getTopoCoord().becomeCandidateIfElectable( - now()++, TopologyCoordinator::StartElectionReason::kElectionTimeout); + now()++, StartElectionReasonEnum::kElectionTimeout); ASSERT_NOT_OK(status); ASSERT_STRING_CONTAINS(status.reason(), "not a member of a valid replica set config"); } @@ -3678,8 +3678,8 @@ TEST_F(TopoCoordTest, FreshestNodeDoesCatchupTakeover) { StatusWith(hbResp)); getTopoCoord().updateTerm(1, Date_t()); - ASSERT_OK(getTopoCoord().becomeCandidateIfElectable( - Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover)); + ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(Date_t(), + StartElectionReasonEnum::kCatchupTakeover)); } TEST_F(TopoCoordTest, StaleNodeDoesntDoCatchupTakeover) { @@ -3730,7 +3730,7 @@ TEST_F(TopoCoordTest, StaleNodeDoesntDoCatchupTakeover) { getTopoCoord().updateTerm(1, Date_t()); Status result = getTopoCoord().becomeCandidateIfElectable( - Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover); + Date_t(), StartElectionReasonEnum::kCatchupTakeover); ASSERT_NOT_OK(result); ASSERT_STRING_CONTAINS(result.reason(), "member is either not the most up-to-date member or not ahead of the " @@ -3782,7 +3782,7 @@ TEST_F(TopoCoordTest, NodeDoesntDoCatchupTakeoverHeartbeatSaysPrimaryCaughtUp) { getTopoCoord().updateTerm(1, Date_t()); Status result = getTopoCoord().becomeCandidateIfElectable( - Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover); + Date_t(), StartElectionReasonEnum::kCatchupTakeover); ASSERT_NOT_OK(result); ASSERT_STRING_CONTAINS(result.reason(), "member is either not the most up-to-date member or not ahead of the " @@ -3839,7 +3839,7 @@ TEST_F(TopoCoordTest, NodeDoesntDoCatchupTakeoverIfTermNumbersSayPrimaryCaughtUp getTopoCoord().updateTerm(1, Date_t()); Status result = getTopoCoord().becomeCandidateIfElectable( - Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover); + Date_t(), StartElectionReasonEnum::kCatchupTakeover); ASSERT_NOT_OK(result); ASSERT_STRING_CONTAINS(result.reason(), "member is either not the most up-to-date member or not ahead of the " @@ -4993,8 +4993,8 @@ TEST_F(HeartbeatResponseTestV1, ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole()); // We are electable now. - ASSERT_OK(getTopoCoord().becomeCandidateIfElectable( - now(), TopologyCoordinator::StartElectionReason::kElectionTimeout)); + ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(), + StartElectionReasonEnum::kElectionTimeout)); ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole()); } @@ -5019,8 +5019,8 @@ TEST_F(HeartbeatResponseTestV1, ScheduleElectionWhenPrimaryIsMarkedDownAndWeAreE ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole()); // We are electable now. - ASSERT_OK(getTopoCoord().becomeCandidateIfElectable( - now(), TopologyCoordinator::StartElectionReason::kElectionTimeout)); + ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(), + StartElectionReasonEnum::kElectionTimeout)); ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole()); } -- cgit v1.2.1