summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorXuerui Fa <xuerui.fa@mongodb.com>2019-10-01 17:51:45 +0000
committerevergreen <evergreen@mongodb.com>2019-10-01 17:51:45 +0000
commitb5e2e827f65459fd5cf5f88c081dda01b801867d (patch)
treea71757a1302f351f1a1bda1df42fa1a8ccb96c01
parent065bc52eec452e5bedfd7d9d8df569ef4309616d (diff)
downloadmongo-b5e2e827f65459fd5cf5f88c081dda01b801867d.tar.gz
SERVER-41506 Added tracking for metrics around a nodes calling an election
(cherry picked from commit 87c9601a762048ea93559b13c3b6043b4017b024)
-rw-r--r--jstests/replsets/election_candidate_and_participant_metrics.js138
-rw-r--r--jstests/replsets/election_handoff_basic.js6
-rw-r--r--jstests/replsets/election_handoff_flip.js6
-rw-r--r--jstests/replsets/election_handoff_via_signal.js6
-rw-r--r--src/mongo/db/repl/replication_coordinator_impl.cpp12
-rw-r--r--src/mongo/db/repl/replication_coordinator_impl.h20
-rw-r--r--src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp54
-rw-r--r--src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp35
-rw-r--r--src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp41
-rw-r--r--src/mongo/db/repl/replication_metrics.cpp54
-rw-r--r--src/mongo/db/repl/replication_metrics.h14
-rw-r--r--src/mongo/db/repl/replication_metrics.idl41
-rw-r--r--src/mongo/db/repl/topology_coordinator.cpp87
-rw-r--r--src/mongo/db/repl/topology_coordinator.h23
-rw-r--r--src/mongo/db/repl/topology_coordinator_v1_test.cpp20
15 files changed, 373 insertions, 184 deletions
diff --git a/jstests/replsets/election_candidate_and_participant_metrics.js b/jstests/replsets/election_candidate_and_participant_metrics.js
new file mode 100644
index 00000000000..7e19f1a06f1
--- /dev/null
+++ b/jstests/replsets/election_candidate_and_participant_metrics.js
@@ -0,0 +1,138 @@
+/**
+ * This test checks that the metrics around election candidates and voters are set correctly.
+ */
+
+(function() {
+"use strict";
+load("jstests/libs/check_log.js");
+load("jstests/replsets/libs/election_metrics.js");
+load("jstests/replsets/libs/election_handoff.js");
+
+const testName = jsTestName();
+const numNodes = 2;
+const rst = ReplSetTest({name: testName, nodes: numNodes});
+const nodes = rst.nodeList();
+rst.startSet();
+
+// Make sure there are no election timeouts firing for the duration of the test. This helps
+// ensure that the test will only pass if the election handoff succeeds.
+rst.initiateWithHighElectionTimeout();
+
+const expectedElectionTimeoutMillis = 24 * 60 * 60 * 1000;
+
+const originalPrimary = rst.getPrimary();
+let originalPrimaryReplSetGetStatus =
+ assert.commandWorked(originalPrimary.adminCommand({replSetGetStatus: 1}));
+let originalPrimaryElectionCandidateMetrics =
+ originalPrimaryReplSetGetStatus.electionCandidateMetrics;
+
+// Check that the 'electionCandidateMetrics' section of the replSetGetStatus response after
+// replica set startup has all of the required fields and that they are set correctly.
+assert(originalPrimaryElectionCandidateMetrics.lastElectionReason,
+ () => "Response should have an 'electionCandidateMetrics.lastElectionReason' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+assert.eq(originalPrimaryElectionCandidateMetrics.lastElectionReason, "electionTimeout");
+assert(originalPrimaryElectionCandidateMetrics.lastElectionDate,
+ () => "Response should have an 'electionCandidateMetrics.lastElectionDate' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+assert(originalPrimaryElectionCandidateMetrics.termAtElection,
+ () => "Response should have an 'electionCandidateMetrics.termAtElection' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+assert.eq(originalPrimaryElectionCandidateMetrics.termAtElection, 1);
+assert(
+ originalPrimaryElectionCandidateMetrics.lastCommittedOpTimeAtElection,
+ () =>
+ "Response should have an 'electionCandidateMetrics.lastCommittedOpTimeAtElection' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+assert(originalPrimaryElectionCandidateMetrics.lastSeenOpTimeAtElection,
+ () => "Response should have an 'electionCandidateMetrics.lastSeenOpTimeAtElection' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+assert(originalPrimaryElectionCandidateMetrics.numVotesNeeded,
+ () => "Response should have an 'electionCandidateMetrics.numVotesNeeded' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+assert.eq(originalPrimaryElectionCandidateMetrics.numVotesNeeded, 1);
+assert(originalPrimaryElectionCandidateMetrics.priorityAtElection,
+ () => "Response should have an 'electionCandidateMetrics.priorityAtElection' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+assert.eq(originalPrimaryElectionCandidateMetrics.priorityAtElection, 1.0);
+assert(originalPrimaryElectionCandidateMetrics.electionTimeoutMillis,
+ () => "Response should have an 'electionCandidateMetrics.electionTimeoutMillis' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+// The node runs its own election before receiving the configuration, so 'electionTimeoutMillis' is
+// set to the default value.
+assert.eq(originalPrimaryElectionCandidateMetrics.electionTimeoutMillis, 10000);
+assert(!originalPrimaryElectionCandidateMetrics.priorPrimaryMemberId,
+ () => "Response should not have an 'electionCandidateMetrics.priorPrimaryMemberId' field: " +
+ tojson(originalPrimaryElectionCandidateMetrics));
+
+ElectionHandoffTest.testElectionHandoff(rst, 0, 1);
+
+const newPrimary = rst.getPrimary();
+let newPrimaryReplSetGetStatus =
+ assert.commandWorked(newPrimary.adminCommand({replSetGetStatus: 1}));
+let newPrimaryElectionCandidateMetrics = newPrimaryReplSetGetStatus.electionCandidateMetrics;
+
+// Check that the 'electionCandidateMetrics' section of the replSetGetStatus response for the new
+// primary has all of the required fields and that they are set correctly.
+assert(newPrimaryElectionCandidateMetrics.lastElectionReason,
+ () => "Response should have an 'electionCandidateMetrics.lastElectionReason' field: " +
+ tojson(newPrimaryElectionCandidateMetrics));
+assert.eq(newPrimaryElectionCandidateMetrics.lastElectionReason, "stepUpRequestSkipDryRun");
+assert(newPrimaryElectionCandidateMetrics.lastElectionDate,
+ () => "Response should have an 'electionCandidateMetrics.lastElectionDate' field: " +
+ tojson(newPrimaryElectionCandidateMetrics));
+assert(newPrimaryElectionCandidateMetrics.termAtElection,
+ () => "Response should have an 'electionCandidateMetrics.termAtElection' field: " +
+ tojson(newPrimaryElectionCandidateMetrics));
+assert.eq(newPrimaryElectionCandidateMetrics.termAtElection, 2);
+assert(
+ newPrimaryElectionCandidateMetrics.lastCommittedOpTimeAtElection,
+ () =>
+ "Response should have an 'electionCandidateMetrics.lastCommittedOpTimeAtElection' field: " +
+ tojson(newPrimaryElectionCandidateMetrics));
+assert(newPrimaryElectionCandidateMetrics.lastSeenOpTimeAtElection,
+ () => "Response should have an 'electionCandidateMetrics.lastSeenOpTimeAtElection' field: " +
+ tojson(newPrimaryElectionCandidateMetrics));
+assert(newPrimaryElectionCandidateMetrics.numVotesNeeded,
+ () => "Response should have an 'electionCandidateMetrics.numVotesNeeded' field: " +
+ tojson(newPrimaryElectionCandidateMetrics));
+assert.eq(newPrimaryElectionCandidateMetrics.numVotesNeeded, 2);
+assert(newPrimaryElectionCandidateMetrics.priorityAtElection,
+ () => "Response should have an 'electionCandidateMetrics.priorityAtElection' field: " +
+ tojson(newPrimaryElectionCandidateMetrics));
+assert.eq(newPrimaryElectionCandidateMetrics.priorityAtElection, 1.0);
+assert(newPrimaryElectionCandidateMetrics.electionTimeoutMillis,
+ () => "Response should have an 'electionCandidateMetrics.electionTimeoutMillis' field: " +
+ tojson(newPrimaryElectionCandidateMetrics));
+assert.eq(newPrimaryElectionCandidateMetrics.electionTimeoutMillis, expectedElectionTimeoutMillis);
+// Since the previous primary's ID is 0, we directly assert that 0 is stored in the
+// priorPrimaryMemberId field.
+assert.eq(newPrimaryElectionCandidateMetrics.priorPrimaryMemberId, 0);
+
+// Step up the original primary.
+sleep(ElectionHandoffTest.stepDownPeriodSecs * 1000);
+ElectionHandoffTest.testElectionHandoff(rst, 1, 0);
+
+originalPrimaryReplSetGetStatus =
+ assert.commandWorked(originalPrimary.adminCommand({replSetGetStatus: 1}));
+originalPrimaryElectionCandidateMetrics = originalPrimaryReplSetGetStatus.electionCandidateMetrics;
+
+// Check that the original primary's metrics are also being set properly after the second election.
+assert.eq(originalPrimaryElectionCandidateMetrics.lastElectionReason, "stepUpRequestSkipDryRun");
+assert.eq(originalPrimaryElectionCandidateMetrics.termAtElection, 3);
+assert.eq(originalPrimaryElectionCandidateMetrics.numVotesNeeded, 2);
+assert.eq(originalPrimaryElectionCandidateMetrics.priorityAtElection, 1);
+assert.eq(originalPrimaryElectionCandidateMetrics.electionTimeoutMillis,
+ expectedElectionTimeoutMillis);
+assert.eq(originalPrimaryElectionCandidateMetrics.priorPrimaryMemberId, 1);
+
+newPrimaryReplSetGetStatus = assert.commandWorked(newPrimary.adminCommand({replSetGetStatus: 1}));
+newPrimaryElectionCandidateMetrics = newPrimaryReplSetGetStatus.electionCandidateMetrics;
+
+// The other node should not have an electionCandidateMetrics, as it just stepped down.
+assert(!newPrimaryElectionCandidateMetrics,
+ () => "Response should not have an 'electionCandidateMetrics' field: " +
+ tojson(newPrimaryReplSetGetStatus));
+
+rst.stopSet();
+})(); \ No newline at end of file
diff --git a/jstests/replsets/election_handoff_basic.js b/jstests/replsets/election_handoff_basic.js
index 2c1e27b6ece..2a4376689d8 100644
--- a/jstests/replsets/election_handoff_basic.js
+++ b/jstests/replsets/election_handoff_basic.js
@@ -16,11 +16,7 @@ rst.startSet();
// Make sure there are no election timeouts firing for the duration of the test. This helps
// ensure that the test will only pass if the election handoff succeeds.
-const config = rst.getReplSetConfig();
-config.settings = {
- "electionTimeoutMillis": 12 * 60 * 60 * 1000
-};
-rst.initiate(config);
+rst.initiateWithHighElectionTimeout();
ElectionHandoffTest.testElectionHandoff(rst, 0, 1);
diff --git a/jstests/replsets/election_handoff_flip.js b/jstests/replsets/election_handoff_flip.js
index c2576023048..8ce2b804ca8 100644
--- a/jstests/replsets/election_handoff_flip.js
+++ b/jstests/replsets/election_handoff_flip.js
@@ -15,11 +15,7 @@ rst.startSet();
// Make sure there are no election timeouts firing for the duration of the test. This helps
// ensure that the test will only pass if the election handoff succeeds.
-const config = rst.getReplSetConfig();
-config.settings = {
- "electionTimeoutMillis": 12 * 60 * 60 * 1000
-};
-rst.initiate(config);
+rst.initiateWithHighElectionTimeout();
ElectionHandoffTest.testElectionHandoff(rst, 0, 1);
sleep(ElectionHandoffTest.stepDownPeriodSecs * 1000);
diff --git a/jstests/replsets/election_handoff_via_signal.js b/jstests/replsets/election_handoff_via_signal.js
index bca8d4b4991..58df0cc852e 100644
--- a/jstests/replsets/election_handoff_via_signal.js
+++ b/jstests/replsets/election_handoff_via_signal.js
@@ -15,11 +15,7 @@ rst.startSet();
// Make sure there are no election timeouts firing for the duration of the test. This helps
// ensure that the test will only pass if the election handoff succeeds.
-const config = rst.getReplSetConfig();
-config.settings = {
- "electionTimeoutMillis": 12 * 60 * 60 * 1000
-};
-rst.initiate(config);
+rst.initiateWithHighElectionTimeout();
ElectionHandoffTest.testElectionHandoff(rst, 0, 1, {stepDownBySignal: true});
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index aafca514fd2..0f686fd0e40 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -2179,8 +2179,7 @@ void ReplicationCoordinatorImpl::_handleTimePassing(
// For election protocol v1, call _startElectSelfIfEligibleV1 to avoid race
// against other elections caused by events like election timeout, replSetStepUp etc.
- _startElectSelfIfEligibleV1(
- TopologyCoordinator::StartElectionReason::kSingleNodePromptElection);
+ _startElectSelfIfEligibleV1(StartElectionReasonEnum::kSingleNodePromptElection);
}
bool ReplicationCoordinatorImpl::isMasterForReportingPurposes() {
@@ -2546,8 +2545,7 @@ Status ReplicationCoordinatorImpl::processReplSetFreeze(int secs, BSONObjBuilder
result.getValue()) {
// For election protocol v1, call _startElectSelfIfEligibleV1 to avoid race
// against other elections caused by events like election timeout, replSetStepUp etc.
- _startElectSelfIfEligibleV1(
- TopologyCoordinator::StartElectionReason::kSingleNodePromptElection);
+ _startElectSelfIfEligibleV1(StartElectionReasonEnum::kSingleNodePromptElection);
}
return Status::OK();
@@ -3000,7 +2998,7 @@ void ReplicationCoordinatorImpl::_performPostMemberStateUpdateAction(
case kActionStartSingleNodeElection:
// In protocol version 1, single node replset will run an election instead of
// kActionWinElection as in protocol version 0.
- _startElectSelfV1(TopologyCoordinator::StartElectionReason::kElectionTimeout);
+ _startElectSelfV1(StartElectionReasonEnum::kElectionTimeout);
break;
default:
severe() << "Unknown post member state update action " << static_cast<int>(action);
@@ -4065,8 +4063,8 @@ CallbackFn ReplicationCoordinatorImpl::_wrapAsCallbackFn(const stdx::function<vo
Status ReplicationCoordinatorImpl::stepUpIfEligible(bool skipDryRun) {
- auto reason = skipDryRun ? TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun
- : TopologyCoordinator::StartElectionReason::kStepUpRequest;
+ auto reason = skipDryRun ? StartElectionReasonEnum::kStepUpRequestSkipDryRun
+ : StartElectionReasonEnum::kStepUpRequest;
_startElectSelfIfEligibleV1(reason);
EventHandle finishEvent;
diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h
index 8d6753f6f58..ad0273da3b2 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.h
+++ b/src/mongo/db/repl/replication_coordinator_impl.h
@@ -1023,8 +1023,8 @@ private:
* _startVoteRequester_inlock()
* _onVoteRequestComplete()
*/
- void _startElectSelfV1_inlock(TopologyCoordinator::StartElectionReason reason);
- void _startElectSelfV1(TopologyCoordinator::StartElectionReason reason);
+ void _startElectSelfV1_inlock(StartElectionReasonEnum reason);
+ void _startElectSelfV1(StartElectionReasonEnum reason);
/**
* Callback called when the dryRun VoteRequester has completed; checks the results and
@@ -1032,15 +1032,13 @@ private:
* "originalTerm" was the term during which the dry run began, if the term has since
* changed, do not run for election.
*/
- void _processDryRunResult(long long originalTerm,
- TopologyCoordinator::StartElectionReason reason);
+ void _processDryRunResult(long long originalTerm, StartElectionReasonEnum reason);
/**
* Begins executing a real election. This is called either a successful dry run, or when the
* dry run was skipped (which may be specified for a ReplSetStepUp).
*/
- void _startRealElection_inlock(long long originalTerm,
- TopologyCoordinator::StartElectionReason reason);
+ void _startRealElection_inlock(long long originalTerm, StartElectionReasonEnum reason);
/**
* Writes the last vote in persistent storage after completing dry run successfully.
@@ -1048,13 +1046,12 @@ private:
*/
void _writeLastVoteForMyElection(LastVote lastVote,
const executor::TaskExecutor::CallbackArgs& cbData,
- TopologyCoordinator::StartElectionReason reason);
+ StartElectionReasonEnum reason);
/**
* Starts VoteRequester to run the real election when last vote write has completed.
*/
- void _startVoteRequester_inlock(long long newTerm,
- TopologyCoordinator::StartElectionReason reason);
+ void _startVoteRequester_inlock(long long newTerm, StartElectionReasonEnum reason);
/**
* Callback called when the VoteRequester has completed; checks the results and
@@ -1062,8 +1059,7 @@ private:
* "originalTerm" was the term during which the election began, if the term has since
* changed, do not step up as primary.
*/
- void _onVoteRequestComplete(long long originalTerm,
- TopologyCoordinator::StartElectionReason reason);
+ void _onVoteRequestComplete(long long originalTerm, StartElectionReasonEnum reason);
/**
* Removes 'host' from the sync source blacklist. If 'host' isn't found, it's simply
@@ -1281,7 +1277,7 @@ private:
/**
* Callback which starts an election if this node is electable and using protocolVersion 1.
*/
- void _startElectSelfIfEligibleV1(TopologyCoordinator::StartElectionReason reason);
+ void _startElectSelfIfEligibleV1(StartElectionReasonEnum reason);
/**
* Schedules work to be run no sooner than 'when' and returns handle to callback.
diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp
index b1464c0de9b..04f191ad81c 100644
--- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp
@@ -92,15 +92,12 @@ public:
}
};
-
-void ReplicationCoordinatorImpl::_startElectSelfV1(
- TopologyCoordinator::StartElectionReason reason) {
+void ReplicationCoordinatorImpl::_startElectSelfV1(StartElectionReasonEnum reason) {
stdx::lock_guard<stdx::mutex> lk(_mutex);
_startElectSelfV1_inlock(reason);
}
-void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(
- TopologyCoordinator::StartElectionReason reason) {
+void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(StartElectionReasonEnum reason) {
invariant(!_voteRequester);
switch (_rsConfigState) {
@@ -146,10 +143,7 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(
long long term = _topCoord->getTerm();
int primaryIndex = -1;
- Date_t now = _replExecutor->now();
- ReplicationMetrics::get(getServiceContext()).setElectionCandidateMetrics(now);
-
- if (reason == TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun) {
+ if (reason == StartElectionReasonEnum::kStepUpRequestSkipDryRun) {
long long newTerm = term + 1;
log() << "skipping dry run and running for election in term " << newTerm;
_startRealElection_inlock(newTerm, reason);
@@ -161,7 +155,7 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(
_voteRequester.reset(new VoteRequester);
// Only set primaryIndex if the primary's vote is required during the dry run.
- if (reason == TopologyCoordinator::StartElectionReason::kCatchupTakeover) {
+ if (reason == StartElectionReasonEnum::kCatchupTakeover) {
primaryIndex = _topCoord->getCurrentPrimaryIndex();
}
StatusWith<executor::TaskExecutor::EventHandle> nextPhaseEvh =
@@ -185,8 +179,8 @@ void ReplicationCoordinatorImpl::_startElectSelfV1_inlock(
lossGuard.dismiss();
}
-void ReplicationCoordinatorImpl::_processDryRunResult(
- long long originalTerm, TopologyCoordinator::StartElectionReason reason) {
+void ReplicationCoordinatorImpl::_processDryRunResult(long long originalTerm,
+ StartElectionReasonEnum reason) {
stdx::lock_guard<stdx::mutex> lk(_mutex);
LoseElectionDryRunGuardV1 lossGuard(this);
@@ -221,8 +215,30 @@ void ReplicationCoordinatorImpl::_processDryRunResult(
lossGuard.dismiss();
}
-void ReplicationCoordinatorImpl::_startRealElection_inlock(
- long long newTerm, TopologyCoordinator::StartElectionReason reason) {
+void ReplicationCoordinatorImpl::_startRealElection_inlock(long long newTerm,
+ StartElectionReasonEnum reason) {
+
+ const Date_t now = _replExecutor->now();
+ const OpTime lastCommittedOpTime = _topCoord->getLastCommittedOpTime();
+ const OpTime lastSeenOpTime = _topCoord->latestKnownOpTime();
+ const int numVotesNeeded = _rsConfig.getMajorityVoteCount();
+ const double priorityAtElection = _rsConfig.getMemberAt(_selfIndex).getPriority();
+ const Milliseconds electionTimeoutMillis = _rsConfig.getElectionTimeoutPeriod();
+ const int priorPrimaryIndex = _topCoord->getCurrentPrimaryIndex();
+ const boost::optional<int> priorPrimaryMemberId = (priorPrimaryIndex == -1)
+ ? boost::none
+ : boost::make_optional(_rsConfig.getMemberAt(priorPrimaryIndex).getId().getData());
+
+ ReplicationMetrics::get(getServiceContext())
+ .setElectionCandidateMetrics(reason,
+ now,
+ newTerm,
+ lastCommittedOpTime,
+ lastSeenOpTime,
+ numVotesNeeded,
+ priorityAtElection,
+ electionTimeoutMillis,
+ priorPrimaryMemberId);
ReplicationMetrics::get(getServiceContext()).incrementNumElectionsCalledForReason(reason);
LoseElectionDryRunGuardV1 lossGuard(this);
@@ -254,7 +270,7 @@ void ReplicationCoordinatorImpl::_startRealElection_inlock(
void ReplicationCoordinatorImpl::_writeLastVoteForMyElection(
LastVote lastVote,
const executor::TaskExecutor::CallbackArgs& cbData,
- TopologyCoordinator::StartElectionReason reason) {
+ StartElectionReasonEnum reason) {
// storeLocalLastVoteDocument can call back in to the replication coordinator,
// so _mutex must be unlocked here. However, we cannot return until we
// lock it because we want to lose the election on cancel or error and
@@ -292,8 +308,8 @@ void ReplicationCoordinatorImpl::_writeLastVoteForMyElection(
lossGuard.dismiss();
}
-void ReplicationCoordinatorImpl::_startVoteRequester_inlock(
- long long newTerm, TopologyCoordinator::StartElectionReason reason) {
+void ReplicationCoordinatorImpl::_startVoteRequester_inlock(long long newTerm,
+ StartElectionReasonEnum reason) {
const auto lastOpTime = _getMyLastAppliedOpTime_inlock();
_voteRequester.reset(new VoteRequester);
@@ -313,8 +329,8 @@ void ReplicationCoordinatorImpl::_startVoteRequester_inlock(
MONGO_FAIL_POINT_DEFINE(electionHangsBeforeUpdateMemberState);
-void ReplicationCoordinatorImpl::_onVoteRequestComplete(
- long long newTerm, TopologyCoordinator::StartElectionReason reason) {
+void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long newTerm,
+ StartElectionReasonEnum reason) {
stdx::lock_guard<stdx::mutex> lk(_mutex);
LoseElectionGuardV1 lossGuard(this);
diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp
index f54883200ea..ed6c0febe18 100644
--- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp
@@ -374,9 +374,9 @@ TEST_F(ReplCoordTest, ElectionFailsWhenInsufficientVotesAreReceivedDuringDryRun)
<< false << "reason"
<< "don't like him much")));
voteRequests++;
- // Check that the node's election candidate metrics are set once it has called an
- // election.
- ASSERT_BSONOBJ_NE(
+
+ // Check that the node's election candidate metrics are not set if a dry run fails.
+ ASSERT_BSONOBJ_EQ(
BSONObj(),
ReplicationMetrics::get(getServiceContext()).getElectionCandidateMetricsBSON());
} else {
@@ -883,7 +883,7 @@ public:
}
void performSuccessfulTakeover(Date_t takeoverTime,
- TopologyCoordinator::StartElectionReason reason,
+ StartElectionReasonEnum reason,
const LastVote& lastVoteExpected) {
startCapturingLogMessages();
simulateSuccessfulV1ElectionAt(takeoverTime);
@@ -898,7 +898,7 @@ public:
ASSERT_EQ(lastVoteExpected.getCandidateIndex(), lastVote.getValue().getCandidateIndex());
ASSERT_EQ(lastVoteExpected.getTerm(), lastVote.getValue().getTerm());
- if (reason == TopologyCoordinator::StartElectionReason::kPriorityTakeover) {
+ if (reason == StartElectionReasonEnum::kPriorityTakeover) {
ASSERT_EQUALS(1,
countLogLinesContaining("Starting an election for a priority takeover"));
}
@@ -1448,9 +1448,8 @@ TEST_F(TakeoverTest, SuccessfulCatchupTakeover) {
ASSERT_EQUALS(1, countLogLinesContaining("Starting an election for a catchup takeover"));
LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0);
- performSuccessfulTakeover(catchupTakeoverTime,
- TopologyCoordinator::StartElectionReason::kCatchupTakeover,
- lastVoteExpected);
+ performSuccessfulTakeover(
+ catchupTakeoverTime, StartElectionReasonEnum::kCatchupTakeover, lastVoteExpected);
// Check that the numCatchUpTakeoversCalled and the numCatchUpTakeoversSuccessful election
// metrics have been incremented, and that none of the metrics that track the number of
@@ -1707,9 +1706,8 @@ TEST_F(TakeoverTest, PrimaryCatchesUpBeforeHighPriorityNodeCatchupTakeover) {
config, now + longElectionTimeout, HostAndPort("node2", 12345), currentOptime);
LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0);
- performSuccessfulTakeover(priorityTakeoverTime,
- TopologyCoordinator::StartElectionReason::kPriorityTakeover,
- lastVoteExpected);
+ performSuccessfulTakeover(
+ priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected);
}
TEST_F(TakeoverTest, SchedulesPriorityTakeoverIfNodeHasHigherPriorityThanCurrentPrimary) {
@@ -1801,9 +1799,8 @@ TEST_F(TakeoverTest, SuccessfulPriorityTakeover) {
config, now + halfElectionTimeout, HostAndPort("node2", 12345), myOptime);
LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0);
- performSuccessfulTakeover(priorityTakeoverTime,
- TopologyCoordinator::StartElectionReason::kPriorityTakeover,
- lastVoteExpected);
+ performSuccessfulTakeover(
+ priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected);
// Check that the numPriorityTakeoversCalled and the numPriorityTakeoversSuccessful election
// metrics have been incremented, and that none of the metrics that track the number of
@@ -1892,9 +1889,8 @@ TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedSameSecond) {
Date_t() + Seconds(closeEnoughOpTime.getSecs()));
LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0);
- performSuccessfulTakeover(priorityTakeoverTime,
- TopologyCoordinator::StartElectionReason::kPriorityTakeover,
- lastVoteExpected);
+ performSuccessfulTakeover(
+ priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected);
}
TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedDifferentSecond) {
@@ -1967,9 +1963,8 @@ TEST_F(TakeoverTest, DontCallForPriorityTakeoverWhenLaggedDifferentSecond) {
Date_t() + Seconds(closeEnoughOpTime.getSecs()));
LastVote lastVoteExpected = LastVote(replCoord->getTerm() + 1, 0);
- performSuccessfulTakeover(priorityTakeoverTime,
- TopologyCoordinator::StartElectionReason::kPriorityTakeover,
- lastVoteExpected);
+ performSuccessfulTakeover(
+ priorityTakeoverTime, StartElectionReasonEnum::kPriorityTakeover, lastVoteExpected);
}
TEST_F(ReplCoordTest, NodeCancelsElectionUponReceivingANewConfigDuringDryRun) {
diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp
index 6fa02d27a39..faec4c34d41 100644
--- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp
@@ -304,8 +304,7 @@ stdx::unique_lock<stdx::mutex> ReplicationCoordinatorImpl::_handleHeartbeatRespo
LOG_FOR_ELECTION(0) << "Scheduling priority takeover at " << _priorityTakeoverWhen;
_priorityTakeoverCbh = _scheduleWorkAt(
_priorityTakeoverWhen, [=](const mongo::executor::TaskExecutor::CallbackArgs&) {
- _startElectSelfIfEligibleV1(
- TopologyCoordinator::StartElectionReason::kPriorityTakeover);
+ _startElectSelfIfEligibleV1(StartElectionReasonEnum::kPriorityTakeover);
});
}
break;
@@ -318,8 +317,7 @@ stdx::unique_lock<stdx::mutex> ReplicationCoordinatorImpl::_handleHeartbeatRespo
LOG_FOR_ELECTION(0) << "Scheduling catchup takeover at " << _catchupTakeoverWhen;
_catchupTakeoverCbh = _scheduleWorkAt(
_catchupTakeoverWhen, [=](const mongo::executor::TaskExecutor::CallbackArgs&) {
- _startElectSelfIfEligibleV1(
- TopologyCoordinator::StartElectionReason::kCatchupTakeover);
+ _startElectSelfIfEligibleV1(StartElectionReasonEnum::kCatchupTakeover);
});
}
break;
@@ -875,15 +873,14 @@ void ReplicationCoordinatorImpl::_cancelAndRescheduleElectionTimeout_inlock() {
_handleElectionTimeoutWhen = when;
_handleElectionTimeoutCbh =
_scheduleWorkAt(when, [=](const mongo::executor::TaskExecutor::CallbackArgs&) {
- _startElectSelfIfEligibleV1(TopologyCoordinator::StartElectionReason::kElectionTimeout);
+ _startElectSelfIfEligibleV1(StartElectionReasonEnum::kElectionTimeout);
});
}
-void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1(
- TopologyCoordinator::StartElectionReason reason) {
+void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1(StartElectionReasonEnum reason) {
stdx::lock_guard<stdx::mutex> lock(_mutex);
// If it is not a single node replica set, no need to start an election after stepdown timeout.
- if (reason == TopologyCoordinator::StartElectionReason::kSingleNodePromptElection &&
+ if (reason == StartElectionReasonEnum::kSingleNodePromptElection &&
_rsConfig.getNumMembers() != 1) {
return;
}
@@ -903,52 +900,56 @@ void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1(
const auto status = _topCoord->becomeCandidateIfElectable(_replExecutor->now(), reason);
if (!status.isOK()) {
switch (reason) {
- case TopologyCoordinator::StartElectionReason::kElectionTimeout:
+ case StartElectionReasonEnum::kElectionTimeout:
LOG_FOR_ELECTION(0)
<< "Not starting an election, since we are not electable due to: "
<< status.reason();
break;
- case TopologyCoordinator::StartElectionReason::kPriorityTakeover:
+ case StartElectionReasonEnum::kPriorityTakeover:
LOG_FOR_ELECTION(0) << "Not starting an election for a priority takeover, "
<< "since we are not electable due to: " << status.reason();
break;
- case TopologyCoordinator::StartElectionReason::kStepUpRequest:
- case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun:
+ case StartElectionReasonEnum::kStepUpRequest:
+ case StartElectionReasonEnum::kStepUpRequestSkipDryRun:
LOG_FOR_ELECTION(0) << "Not starting an election for a replSetStepUp request, "
<< "since we are not electable due to: " << status.reason();
break;
- case TopologyCoordinator::StartElectionReason::kCatchupTakeover:
+ case StartElectionReasonEnum::kCatchupTakeover:
LOG_FOR_ELECTION(0) << "Not starting an election for a catchup takeover, "
<< "since we are not electable due to: " << status.reason();
break;
- case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection:
+ case StartElectionReasonEnum::kSingleNodePromptElection:
LOG_FOR_ELECTION(0)
<< "Not starting an election for a single node replica set prompt election, "
<< "since we are not electable due to: " << status.reason();
break;
+ default:
+ MONGO_UNREACHABLE;
}
return;
}
switch (reason) {
- case TopologyCoordinator::StartElectionReason::kElectionTimeout:
+ case StartElectionReasonEnum::kElectionTimeout:
LOG_FOR_ELECTION(0) << "Starting an election, since we've seen no PRIMARY in the past "
<< _rsConfig.getElectionTimeoutPeriod();
break;
- case TopologyCoordinator::StartElectionReason::kPriorityTakeover:
+ case StartElectionReasonEnum::kPriorityTakeover:
LOG_FOR_ELECTION(0) << "Starting an election for a priority takeover";
break;
- case TopologyCoordinator::StartElectionReason::kStepUpRequest:
- case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun:
+ case StartElectionReasonEnum::kStepUpRequest:
+ case StartElectionReasonEnum::kStepUpRequestSkipDryRun:
LOG_FOR_ELECTION(0) << "Starting an election due to step up request";
break;
- case TopologyCoordinator::StartElectionReason::kCatchupTakeover:
+ case StartElectionReasonEnum::kCatchupTakeover:
LOG_FOR_ELECTION(0) << "Starting an election for a catchup takeover";
break;
- case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection:
+ case StartElectionReasonEnum::kSingleNodePromptElection:
LOG_FOR_ELECTION(0)
<< "Starting an election due to single node replica set prompt election";
break;
+ default:
+ MONGO_UNREACHABLE;
}
_startElectSelfV1_inlock(reason);
diff --git a/src/mongo/db/repl/replication_metrics.cpp b/src/mongo/db/repl/replication_metrics.cpp
index 55508674562..0794c43fd0c 100644
--- a/src/mongo/db/repl/replication_metrics.cpp
+++ b/src/mongo/db/repl/replication_metrics.cpp
@@ -56,32 +56,31 @@ ReplicationMetrics::ReplicationMetrics()
ReplicationMetrics::~ReplicationMetrics() {}
-void ReplicationMetrics::incrementNumElectionsCalledForReason(
- TopologyCoordinator::StartElectionReason reason) {
+void ReplicationMetrics::incrementNumElectionsCalledForReason(StartElectionReasonEnum reason) {
stdx::lock_guard<stdx::mutex> lk(_mutex);
switch (reason) {
- case TopologyCoordinator::StartElectionReason::kStepUpRequest:
- case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: {
+ case StartElectionReasonEnum::kStepUpRequest:
+ case StartElectionReasonEnum::kStepUpRequestSkipDryRun: {
ElectionReasonCounter& stepUpCmd = _electionMetrics.getStepUpCmd();
stepUpCmd.incrementCalled();
break;
}
- case TopologyCoordinator::StartElectionReason::kPriorityTakeover: {
+ case StartElectionReasonEnum::kPriorityTakeover: {
ElectionReasonCounter& priorityTakeover = _electionMetrics.getPriorityTakeover();
priorityTakeover.incrementCalled();
break;
}
- case TopologyCoordinator::StartElectionReason::kCatchupTakeover: {
+ case StartElectionReasonEnum::kCatchupTakeover: {
ElectionReasonCounter& catchUpTakeover = _electionMetrics.getCatchUpTakeover();
catchUpTakeover.incrementCalled();
break;
}
- case TopologyCoordinator::StartElectionReason::kElectionTimeout: {
+ case StartElectionReasonEnum::kElectionTimeout: {
ElectionReasonCounter& electionTimeout = _electionMetrics.getElectionTimeout();
electionTimeout.incrementCalled();
break;
}
- case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: {
+ case StartElectionReasonEnum::kSingleNodePromptElection: {
ElectionReasonCounter& freezeTimeout = _electionMetrics.getFreezeTimeout();
freezeTimeout.incrementCalled();
break;
@@ -89,32 +88,31 @@ void ReplicationMetrics::incrementNumElectionsCalledForReason(
}
}
-void ReplicationMetrics::incrementNumElectionsSuccessfulForReason(
- TopologyCoordinator::StartElectionReason reason) {
+void ReplicationMetrics::incrementNumElectionsSuccessfulForReason(StartElectionReasonEnum reason) {
stdx::lock_guard<stdx::mutex> lk(_mutex);
switch (reason) {
- case TopologyCoordinator::StartElectionReason::kStepUpRequest:
- case TopologyCoordinator::StartElectionReason::kStepUpRequestSkipDryRun: {
+ case StartElectionReasonEnum::kStepUpRequest:
+ case StartElectionReasonEnum::kStepUpRequestSkipDryRun: {
ElectionReasonCounter& stepUpCmd = _electionMetrics.getStepUpCmd();
stepUpCmd.incrementSuccessful();
break;
}
- case TopologyCoordinator::StartElectionReason::kPriorityTakeover: {
+ case StartElectionReasonEnum::kPriorityTakeover: {
ElectionReasonCounter& priorityTakeover = _electionMetrics.getPriorityTakeover();
priorityTakeover.incrementSuccessful();
break;
}
- case TopologyCoordinator::StartElectionReason::kCatchupTakeover: {
+ case StartElectionReasonEnum::kCatchupTakeover: {
ElectionReasonCounter& catchUpTakeover = _electionMetrics.getCatchUpTakeover();
catchUpTakeover.incrementSuccessful();
break;
}
- case TopologyCoordinator::StartElectionReason::kElectionTimeout: {
+ case StartElectionReasonEnum::kElectionTimeout: {
ElectionReasonCounter& electionTimeout = _electionMetrics.getElectionTimeout();
electionTimeout.incrementSuccessful();
break;
}
- case TopologyCoordinator::StartElectionReason::kSingleNodePromptElection: {
+ case StartElectionReasonEnum::kSingleNodePromptElection: {
ElectionReasonCounter& freezeTimeout = _electionMetrics.getFreezeTimeout();
freezeTimeout.incrementSuccessful();
break;
@@ -263,10 +261,30 @@ long ReplicationMetrics::getNumCatchUpsFailedWithReplSetAbortPrimaryCatchUpCmd_f
return _electionMetrics.getNumCatchUpsFailedWithReplSetAbortPrimaryCatchUpCmd();
}
-void ReplicationMetrics::setElectionCandidateMetrics(Date_t lastElectionDate) {
+void ReplicationMetrics::setElectionCandidateMetrics(
+ const StartElectionReasonEnum reason,
+ const Date_t lastElectionDate,
+ const long long termAtElection,
+ const OpTime lastCommittedOpTime,
+ const OpTime lastSeenOpTime,
+ const int numVotesNeeded,
+ const double priorityAtElection,
+ const Milliseconds electionTimeout,
+ const boost::optional<int> priorPrimaryMemberId) {
+
stdx::lock_guard<stdx::mutex> lk(_mutex);
- _electionCandidateMetrics.setLastElectionDate(lastElectionDate);
+
_nodeIsCandidateOrPrimary = true;
+ _electionCandidateMetrics.setLastElectionReason(reason);
+ _electionCandidateMetrics.setLastElectionDate(lastElectionDate);
+ _electionCandidateMetrics.setTermAtElection(termAtElection);
+ _electionCandidateMetrics.setLastCommittedOpTimeAtElection(lastCommittedOpTime);
+ _electionCandidateMetrics.setLastSeenOpTimeAtElection(lastSeenOpTime);
+ _electionCandidateMetrics.setNumVotesNeeded(numVotesNeeded);
+ _electionCandidateMetrics.setPriorityAtElection(priorityAtElection);
+ long long electionTimeoutMillis = durationCount<Milliseconds>(electionTimeout);
+ _electionCandidateMetrics.setElectionTimeoutMillis(electionTimeoutMillis);
+ _electionCandidateMetrics.setPriorPrimaryMemberId(priorPrimaryMemberId);
}
void ReplicationMetrics::setTargetCatchupOpTime(OpTime opTime) {
diff --git a/src/mongo/db/repl/replication_metrics.h b/src/mongo/db/repl/replication_metrics.h
index 816b1fc39bc..0d8c025e18b 100644
--- a/src/mongo/db/repl/replication_metrics.h
+++ b/src/mongo/db/repl/replication_metrics.h
@@ -49,8 +49,8 @@ public:
~ReplicationMetrics();
// Election metrics
- void incrementNumElectionsCalledForReason(TopologyCoordinator::StartElectionReason reason);
- void incrementNumElectionsSuccessfulForReason(TopologyCoordinator::StartElectionReason reason);
+ void incrementNumElectionsCalledForReason(StartElectionReasonEnum reason);
+ void incrementNumElectionsSuccessfulForReason(StartElectionReasonEnum reason);
void incrementNumStepDownsCausedByHigherTerm();
void incrementNumCatchUps();
void incrementNumCatchUpsConcludedForReason(
@@ -81,7 +81,15 @@ public:
// All the election candidate metrics that should be set when a node calls an election are set
// in this one function, so that the 'electionCandidateMetrics' section of replSetStatus shows a
// consistent state.
- void setElectionCandidateMetrics(Date_t lastElectionDate);
+ void setElectionCandidateMetrics(const StartElectionReasonEnum reason,
+ const Date_t lastElectionDate,
+ const long long termAtElection,
+ const OpTime lastCommittedOpTime,
+ const OpTime lastSeenOpTime,
+ const int numVotesNeeded,
+ const double priorityAtElection,
+ const Milliseconds electionTimeoutMillis,
+ const boost::optional<int> priorPrimary);
void setTargetCatchupOpTime(OpTime opTime);
void setNumCatchUpOps(int numCatchUpOps);
void setNewTermStartDate(Date_t newTermStartDate);
diff --git a/src/mongo/db/repl/replication_metrics.idl b/src/mongo/db/repl/replication_metrics.idl
index 6c5746cef92..e214c2b0dc8 100644
--- a/src/mongo/db/repl/replication_metrics.idl
+++ b/src/mongo/db/repl/replication_metrics.idl
@@ -40,6 +40,18 @@ imports:
- "mongo/idl/basic_types.idl"
- "mongo/db/repl/replication_types.idl"
+enums:
+ StartElectionReason:
+ description: "Reason why a node called for an election"
+ type: string
+ values:
+ kElectionTimeout: "electionTimeout"
+ kPriorityTakeover: "priorityTakeover"
+ kStepUpRequest: "stepUpRequest"
+ kStepUpRequestSkipDryRun: "stepUpRequestSkipDryRun"
+ kCatchupTakeover: "catchupTakeover"
+ kSingleNodePromptElection: "singleNodePromptElection"
+
types:
ElectionReasonCounter:
bson_serialization_type: any
@@ -126,10 +138,35 @@ structs:
candidate"
strict: true
fields:
+ lastElectionReason:
+ description: "Reason why the node called for the election"
+ type: StartElectionReason
lastElectionDate:
- description: "Time the node called the dry run election, or the actual election if
- it skipped dry run"
+ description: "Time the node called for the election"
type: date
+ termAtElection:
+ description: "New term at the time of election"
+ type: long
+ lastCommittedOpTimeAtElection:
+ description: "Last OpTime the node committed before calling the election"
+ type: optime
+ lastSeenOpTimeAtElection:
+ description: "Last OpTime the node saw before calling the election"
+ type: optime
+ numVotesNeeded:
+ description: "Number of votes the node needed to win the election"
+ type: int
+ priorityAtElection:
+ description: "Priority of the node at the time of the election"
+ type: double
+ electionTimeoutMillis:
+ description: "Timeout duration before a node initiates a new election"
+ type: long
+ priorPrimaryMemberId:
+ description: "MemberId of the prior primary, according to the node. Not set if
+ there was no prior primary"
+ type: int
+ optional: true
targetCatchupOpTime:
description: "The node's target opTime for catchup"
type: optime
diff --git a/src/mongo/db/repl/topology_coordinator.cpp b/src/mongo/db/repl/topology_coordinator.cpp
index 9d4e1235ba6..8ee1a117226 100644
--- a/src/mongo/db/repl/topology_coordinator.cpp
+++ b/src/mongo/db/repl/topology_coordinator.cpp
@@ -48,12 +48,8 @@
#include "mongo/db/mongod_options.h"
#include "mongo/db/operation_context.h"
#include "mongo/db/repl/heartbeat_response_action.h"
-#include "mongo/db/repl/is_master_response.h"
#include "mongo/db/repl/isself.h"
#include "mongo/db/repl/member_data.h"
-#include "mongo/db/repl/repl_set_heartbeat_args_v1.h"
-#include "mongo/db/repl/repl_set_heartbeat_response.h"
-#include "mongo/db/repl/repl_set_request_votes_args.h"
#include "mongo/db/repl/rslog.h"
#include "mongo/rpc/metadata/oplog_query_metadata.h"
#include "mongo/rpc/metadata/repl_set_metadata.h"
@@ -922,7 +918,8 @@ bool TopologyCoordinator::setMemberAsDown(Date_t now, const int memberIndex) {
MemberData& hbData = _memberData.at(memberIndex);
hbData.setDownValues(now, "no response within election timeout period");
- if (CannotSeeMajority & _getMyUnelectableReason(now, StartElectionReason::kElectionTimeout)) {
+ if (CannotSeeMajority &
+ _getMyUnelectableReason(now, StartElectionReasonEnum::kElectionTimeout)) {
return true;
}
@@ -1213,7 +1210,7 @@ int TopologyCoordinator::_findHealthyPrimaryOfEqualOrGreaterPriority(
}
bool TopologyCoordinator::_amIFreshEnoughForPriorityTakeover() const {
- const OpTime latestKnownOpTime = _latestKnownOpTime();
+ const OpTime ourLatestKnownOpTime = latestKnownOpTime();
// Rules are:
// - If the terms don't match, we don't call for priority takeover.
@@ -1227,23 +1224,24 @@ bool TopologyCoordinator::_amIFreshEnoughForPriorityTakeover() const {
// passes the timestamp component of the last oplog entry.
const OpTime ourLastOpApplied = getMyLastAppliedOpTime();
- if (ourLastOpApplied.getTerm() != latestKnownOpTime.getTerm()) {
+ if (ourLastOpApplied.getTerm() != ourLatestKnownOpTime.getTerm()) {
return false;
}
- if (ourLastOpApplied.getTimestamp().getSecs() != latestKnownOpTime.getTimestamp().getSecs()) {
+ if (ourLastOpApplied.getTimestamp().getSecs() !=
+ ourLatestKnownOpTime.getTimestamp().getSecs()) {
return ourLastOpApplied.getTimestamp().getSecs() +
gPriorityTakeoverFreshnessWindowSeconds >=
- latestKnownOpTime.getTimestamp().getSecs();
+ ourLatestKnownOpTime.getTimestamp().getSecs();
} else {
return ourLastOpApplied.getTimestamp().getInc() + 1000 >=
- latestKnownOpTime.getTimestamp().getInc();
+ ourLatestKnownOpTime.getTimestamp().getInc();
}
}
bool TopologyCoordinator::_amIFreshEnoughForCatchupTakeover() const {
- const OpTime latestKnownOpTime = _latestKnownOpTime();
+ const OpTime ourLatestKnownOpTime = latestKnownOpTime();
// Rules are:
// - We must have the freshest optime of all the up nodes.
@@ -1255,7 +1253,7 @@ bool TopologyCoordinator::_amIFreshEnoughForCatchupTakeover() const {
// There is no point to a catchup takeover if we aren't the freshest node because
// another node would immediately perform another catchup takeover when we become primary.
const OpTime ourLastOpApplied = getMyLastAppliedOpTime();
- if (ourLastOpApplied < latestKnownOpTime) {
+ if (ourLastOpApplied < ourLatestKnownOpTime) {
return false;
}
@@ -1284,34 +1282,6 @@ bool TopologyCoordinator::_iAmPrimary() const {
return false;
}
-OpTime TopologyCoordinator::_latestKnownOpTime() const {
- OpTime latest = getMyLastAppliedOpTime();
- for (std::vector<MemberData>::const_iterator it = _memberData.begin(); it != _memberData.end();
- ++it) {
- // Ignore self
- // TODO(russotto): Simplify when heartbeat and spanning tree times are combined.
- if (it->isSelf()) {
- continue;
- }
- // Ignore down members
- if (!it->up()) {
- continue;
- }
- // Ignore removed nodes (not in config, so not valid).
- if (it->getState().removed()) {
- continue;
- }
-
- OpTime optime = it->getHeartbeatAppliedOpTime();
-
- if (optime > latest) {
- latest = optime;
- }
- }
-
- return latest;
-}
-
bool TopologyCoordinator::prepareForUnconditionalStepDown() {
if (_leaderMode == LeaderMode::kSteppingDown) {
// Can only be processing one required stepdown at a time.
@@ -1983,7 +1953,7 @@ TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getUnelectableR
}
TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getMyUnelectableReason(
- const Date_t now, StartElectionReason reason) const {
+ const Date_t now, StartElectionReasonEnum reason) const {
UnelectableReasonMask result = None;
const OpTime lastApplied = getMyLastAppliedOpTime();
if (lastApplied.isNull()) {
@@ -2011,11 +1981,13 @@ TopologyCoordinator::UnelectableReasonMask TopologyCoordinator::_getMyUnelectabl
result |= NotSecondary;
}
- if (reason == StartElectionReason::kPriorityTakeover && !_amIFreshEnoughForPriorityTakeover()) {
+ if (reason == StartElectionReasonEnum::kPriorityTakeover &&
+ !_amIFreshEnoughForPriorityTakeover()) {
result |= NotCloseEnoughToLatestForPriorityTakeover;
}
- if (reason == StartElectionReason::kCatchupTakeover && !_amIFreshEnoughForCatchupTakeover()) {
+ if (reason == StartElectionReasonEnum::kCatchupTakeover &&
+ !_amIFreshEnoughForCatchupTakeover()) {
result |= NotFreshEnoughForCatchupTakeover;
}
return result;
@@ -2781,7 +2753,7 @@ void TopologyCoordinator::setPrimaryIndex(long long primaryIndex) {
}
Status TopologyCoordinator::becomeCandidateIfElectable(const Date_t now,
- StartElectionReason reason) {
+ StartElectionReasonEnum reason) {
if (_role == Role::kLeader) {
return {ErrorCodes::NodeNotElectable, "Not standing for election again; already primary"};
}
@@ -2814,6 +2786,33 @@ void TopologyCoordinator::restartHeartbeats() {
}
}
+OpTime TopologyCoordinator::latestKnownOpTime() const {
+ OpTime latest = getMyLastAppliedOpTime();
+ for (std::vector<MemberData>::const_iterator it = _memberData.begin(); it != _memberData.end();
+ ++it) {
+ // Ignore self
+ if (it->isSelf()) {
+ continue;
+ }
+ // Ignore down members
+ if (!it->up()) {
+ continue;
+ }
+ // Ignore removed nodes (not in config, so not valid).
+ if (it->getState().removed()) {
+ continue;
+ }
+
+ OpTime optime = it->getHeartbeatAppliedOpTime();
+
+ if (optime > latest) {
+ latest = optime;
+ }
+ }
+
+ return latest;
+}
+
boost::optional<OpTime> TopologyCoordinator::latestKnownOpTimeSinceHeartbeatRestart() const {
// The smallest OpTime in PV1.
OpTime latest(Timestamp(0, 0), 0);
diff --git a/src/mongo/db/repl/topology_coordinator.h b/src/mongo/db/repl/topology_coordinator.h
index e29453292a4..e23f820fef3 100644
--- a/src/mongo/db/repl/topology_coordinator.h
+++ b/src/mongo/db/repl/topology_coordinator.h
@@ -32,9 +32,13 @@
#include <iosfwd>
#include <string>
+#include "mongo/db/repl/is_master_response.h"
#include "mongo/db/repl/last_vote.h"
+#include "mongo/db/repl/repl_set_heartbeat_args_v1.h"
#include "mongo/db/repl/repl_set_heartbeat_response.h"
+#include "mongo/db/repl/repl_set_request_votes_args.h"
#include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/db/repl/replication_metrics_gen.h"
#include "mongo/db/repl/split_horizon.h"
#include "mongo/db/repl/update_position_args.h"
#include "mongo/db/server_options.h"
@@ -650,19 +654,10 @@ public:
*/
int getCurrentPrimaryIndex() const;
- enum StartElectionReason {
- kElectionTimeout,
- kPriorityTakeover,
- kStepUpRequest,
- kStepUpRequestSkipDryRun,
- kCatchupTakeover,
- kSingleNodePromptElection
- };
-
/**
* Transitions to the candidate role if the node is electable.
*/
- Status becomeCandidateIfElectable(const Date_t now, StartElectionReason reason);
+ Status becomeCandidateIfElectable(const Date_t now, StartElectionReasonEnum reason);
/**
* Updates the storage engine read committed support in the TopologyCoordinator options after
@@ -675,6 +670,9 @@ public:
*/
void restartHeartbeats();
+ // Scans through all members that are 'up' and returns the latest known optime.
+ OpTime latestKnownOpTime() const;
+
/**
* Scans through all members that are 'up' and return the latest known optime, if we have
* received (successful or failed) heartbeats from all nodes since heartbeat restart.
@@ -814,7 +812,7 @@ private:
// Returns reason why "self" member is unelectable
UnelectableReasonMask _getMyUnelectableReason(const Date_t now,
- StartElectionReason reason) const;
+ StartElectionReasonEnum reason) const;
// Returns reason why memberIndex is unelectable
UnelectableReasonMask _getUnelectableReason(int memberIndex) const;
@@ -825,9 +823,6 @@ private:
// Return true if we are currently primary
bool _iAmPrimary() const;
- // Scans through all members that are 'up' and return the latest known optime.
- OpTime _latestKnownOpTime() const;
-
// Helper shortcut to self config
const MemberConfig& _selfConfig() const;
diff --git a/src/mongo/db/repl/topology_coordinator_v1_test.cpp b/src/mongo/db/repl/topology_coordinator_v1_test.cpp
index 9dd69ddd6cb..04d0a152219 100644
--- a/src/mongo/db/repl/topology_coordinator_v1_test.cpp
+++ b/src/mongo/db/repl/topology_coordinator_v1_test.cpp
@@ -2486,7 +2486,7 @@ TEST_F(TopoCoordTest, NodeReturnsArbiterWhenGetMemberStateRunsAgainstArbiter) {
TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileRemovedFromTheConfig) {
const auto status = getTopoCoord().becomeCandidateIfElectable(
- now()++, TopologyCoordinator::StartElectionReason::kElectionTimeout);
+ now()++, StartElectionReasonEnum::kElectionTimeout);
ASSERT_NOT_OK(status);
ASSERT_STRING_CONTAINS(status.reason(), "not a member of a valid replica set config");
}
@@ -3690,8 +3690,8 @@ TEST_F(TopoCoordTest, FreshestNodeDoesCatchupTakeover) {
StatusWith<ReplSetHeartbeatResponse>(hbResp));
getTopoCoord().updateTerm(1, Date_t());
- ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(
- Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover));
+ ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(Date_t(),
+ StartElectionReasonEnum::kCatchupTakeover));
}
TEST_F(TopoCoordTest, StaleNodeDoesntDoCatchupTakeover) {
@@ -3742,7 +3742,7 @@ TEST_F(TopoCoordTest, StaleNodeDoesntDoCatchupTakeover) {
getTopoCoord().updateTerm(1, Date_t());
Status result = getTopoCoord().becomeCandidateIfElectable(
- Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover);
+ Date_t(), StartElectionReasonEnum::kCatchupTakeover);
ASSERT_NOT_OK(result);
ASSERT_STRING_CONTAINS(result.reason(),
"member is either not the most up-to-date member or not ahead of the "
@@ -3794,7 +3794,7 @@ TEST_F(TopoCoordTest, NodeDoesntDoCatchupTakeoverHeartbeatSaysPrimaryCaughtUp) {
getTopoCoord().updateTerm(1, Date_t());
Status result = getTopoCoord().becomeCandidateIfElectable(
- Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover);
+ Date_t(), StartElectionReasonEnum::kCatchupTakeover);
ASSERT_NOT_OK(result);
ASSERT_STRING_CONTAINS(result.reason(),
"member is either not the most up-to-date member or not ahead of the "
@@ -3851,7 +3851,7 @@ TEST_F(TopoCoordTest, NodeDoesntDoCatchupTakeoverIfTermNumbersSayPrimaryCaughtUp
getTopoCoord().updateTerm(1, Date_t());
Status result = getTopoCoord().becomeCandidateIfElectable(
- Date_t(), TopologyCoordinator::StartElectionReason::kCatchupTakeover);
+ Date_t(), StartElectionReasonEnum::kCatchupTakeover);
ASSERT_NOT_OK(result);
ASSERT_STRING_CONTAINS(result.reason(),
"member is either not the most up-to-date member or not ahead of the "
@@ -5005,8 +5005,8 @@ TEST_F(HeartbeatResponseTestV1,
ASSERT_NO_ACTION(nextAction.getAction());
ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
// We are electable now.
- ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(
- now(), TopologyCoordinator::StartElectionReason::kElectionTimeout));
+ ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(),
+ StartElectionReasonEnum::kElectionTimeout));
ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
}
@@ -5031,8 +5031,8 @@ TEST_F(HeartbeatResponseTestV1, ScheduleElectionWhenPrimaryIsMarkedDownAndWeAreE
ASSERT_EQUALS(-1, getCurrentPrimaryIndex());
ASSERT_TRUE(TopologyCoordinator::Role::kFollower == getTopoCoord().getRole());
// We are electable now.
- ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(
- now(), TopologyCoordinator::StartElectionReason::kElectionTimeout));
+ ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(),
+ StartElectionReasonEnum::kElectionTimeout));
ASSERT_TRUE(TopologyCoordinator::Role::kCandidate == getTopoCoord().getRole());
}