summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--jstests/replsets/election_candidate_and_participant_metrics.js12
-rw-r--r--jstests/replsets/election_participant_new_term_metrics.js109
-rw-r--r--src/mongo/db/pipeline/document_source_change_stream_test.cpp8
-rw-r--r--src/mongo/db/repl/oplog_applier_impl.cpp11
-rw-r--r--src/mongo/db/repl/replication_coordinator.h10
-rw-r--r--src/mongo/db/repl/replication_coordinator_external_state_impl.cpp6
-rw-r--r--src/mongo/db/repl/replication_coordinator_impl.cpp9
-rw-r--r--src/mongo/db/repl/replication_metrics.cpp27
-rw-r--r--src/mongo/db/repl/replication_metrics.h5
-rw-r--r--src/mongo/db/repl/replication_metrics.idl11
-rw-r--r--src/mongo/db/repl/topology_coordinator.cpp19
11 files changed, 200 insertions, 27 deletions
diff --git a/jstests/replsets/election_candidate_and_participant_metrics.js b/jstests/replsets/election_candidate_and_participant_metrics.js
index d75d058846e..029db2fc883 100644
--- a/jstests/replsets/election_candidate_and_participant_metrics.js
+++ b/jstests/replsets/election_candidate_and_participant_metrics.js
@@ -7,8 +7,7 @@
(function() {
"use strict";
-load("jstests/libs/check_log.js");
-load("jstests/replsets/libs/election_metrics.js");
+
load("jstests/replsets/libs/election_handoff.js");
const testName = jsTestName();
@@ -204,8 +203,8 @@ assert.eq(newPrimaryElectionParticipantMetrics.priorityAtElection, 1);
// Since the election participant metrics are only set in the real election, set up a failpoint that
// tells a voting node to vote yes in the dry run election and no in the real election.
-assert.commandWorked(
- originalPrimary.adminCommand({configureFailPoint: "forceVoteInElection", mode: "alwaysOn"}));
+assert.commandWorked(originalPrimary.adminCommand(
+ {configureFailPoint: "voteYesInDryRunButNoInRealElection", mode: "alwaysOn"}));
// Attempt to step up the new primary a second time. Due to the failpoint, the current primary
// should vote no, and as a result the election should fail.
@@ -222,8 +221,9 @@ originalPrimaryElectionParticipantMetrics =
assert.eq(originalPrimaryElectionParticipantMetrics.votedForCandidate, false);
assert.eq(originalPrimaryElectionParticipantMetrics.electionTerm, 4);
assert.eq(originalPrimaryElectionParticipantMetrics.electionCandidateMemberId, 1);
-assert.eq(originalPrimaryElectionParticipantMetrics.voteReason,
- "forced to vote no due to failpoint forceVoteInElection set");
+assert.eq(
+ originalPrimaryElectionParticipantMetrics.voteReason,
+ "forced to vote no in real election due to failpoint voteYesInDryRunButNoInRealElection set");
assert.eq(originalPrimaryElectionParticipantMetrics.priorityAtElection, 1);
rst.stopSet();
diff --git a/jstests/replsets/election_participant_new_term_metrics.js b/jstests/replsets/election_participant_new_term_metrics.js
new file mode 100644
index 00000000000..aa76bdf4546
--- /dev/null
+++ b/jstests/replsets/election_participant_new_term_metrics.js
@@ -0,0 +1,109 @@
+/**
+ * This test checks that the 'newTermStartDate' and 'newTermAppliedDate' metrics in
+ * 'electionParticipantMetrics' are set and updated correctly. We test this with a three node
+ * replica set by successfully stepping up one of the secondaries, then failing to step up the
+ * original primary. We check that the metrics are appropriately set or unset after each election.
+ */
+
+(function() {
+"use strict";
+
+const testName = jsTestName();
+const rst = ReplSetTest({name: testName, nodes: [{}, {}, {rsConfig: {priority: 0}}]});
+rst.startSet();
+
+// Make sure there are no election timeouts firing for the duration of the test. This helps
+// ensure that the test will only pass if the election succeeds.
+rst.initiateWithHighElectionTimeout();
+
+const originalPrimary = rst.getPrimary();
+const newPrimary = rst.getSecondaries()[0];
+const testNode = rst.getSecondaries()[1];
+
+// Set up a failpoint that forces the original primary to vote no in this election. This guarantees
+// that 'testNode' will be a participant in this election, since its vote will be needed for the new
+// primary to win.
+assert.commandWorked(
+ originalPrimary.adminCommand({configureFailPoint: "voteNoInElection", mode: "alwaysOn"}));
+
+// Step up the new primary.
+assert.commandWorked(newPrimary.adminCommand({replSetStepUp: 1}));
+rst.awaitNodesAgreeOnPrimary();
+assert.eq(newPrimary, rst.getPrimary());
+
+// Since the new term oplog entry needs to be replicated onto testNode for the metrics to be set, we
+// must await replication before checking the metrics.
+rst.awaitReplication();
+
+let testNodeReplSetGetStatus = assert.commandWorked(testNode.adminCommand({replSetGetStatus: 1}));
+let testNodeElectionParticipantMetrics = testNodeReplSetGetStatus.electionParticipantMetrics;
+
+const originalNewTermStartDate = testNodeElectionParticipantMetrics.newTermStartDate;
+const originalNewTermAppliedDate = testNodeElectionParticipantMetrics.newTermAppliedDate;
+
+// These fields should be set, since testNode has received and applied the new term oplog entry
+// after the election.
+assert(originalNewTermStartDate,
+ () => "Response should have an 'electionParticipantMetrics.newTermStartDate' field: " +
+ tojson(testNodeElectionParticipantMetrics));
+assert(originalNewTermAppliedDate,
+ () => "Response should have an 'electionParticipantMetrics.newTermAppliedDate' field: " +
+ tojson(testNodeElectionParticipantMetrics));
+
+// Set up a failpoint that forces newPrimary and testNode to vote no in the election, in addition to
+// the new primary above. This will cause the dry run to fail for the original primary.
+assert.commandWorked(
+ newPrimary.adminCommand({configureFailPoint: "voteNoInElection", mode: "alwaysOn"}));
+assert.commandWorked(
+ testNode.adminCommand({configureFailPoint: "voteNoInElection", mode: "alwaysOn"}));
+
+// Attempt to step up the original primary.
+assert.commandFailedWithCode(originalPrimary.adminCommand({replSetStepUp: 1}),
+ ErrorCodes.CommandFailed);
+
+testNodeReplSetGetStatus = assert.commandWorked(testNode.adminCommand({replSetGetStatus: 1}));
+testNodeElectionParticipantMetrics = testNodeReplSetGetStatus.electionParticipantMetrics;
+
+// The 'newTermStartDate' and 'newTermAppliedDate' fields should not be cleared, since the term is
+// not incremented when a dry run election is initiated.
+assert(testNodeElectionParticipantMetrics.newTermStartDate,
+ () => "Response should have an 'electionParticipantMetrics.newTermStartDate' field: " +
+ tojson(testNodeElectionParticipantMetrics));
+assert(testNodeElectionParticipantMetrics.newTermAppliedDate,
+ () => "Response should have an 'electionParticipantMetrics.newTermAppliedDate' field: " +
+ tojson(testNodeElectionParticipantMetrics));
+
+// The fields should store the same dates, since a new term oplog entry was not received.
+assert.eq(originalNewTermStartDate, testNodeElectionParticipantMetrics.newTermStartDate);
+assert.eq(originalNewTermAppliedDate, testNodeElectionParticipantMetrics.newTermAppliedDate);
+
+// Clear the previous failpoint and set up a new one that forces the two current secondaries to vote
+// yes for the candidate in the dry run election and no in the real election. This will cause the
+// real election to fail.
+assert.commandWorked(
+ newPrimary.adminCommand({configureFailPoint: "voteNoInElection", mode: "off"}));
+assert.commandWorked(newPrimary.adminCommand(
+ {configureFailPoint: "voteYesInDryRunButNoInRealElection", mode: "alwaysOn"}));
+assert.commandWorked(testNode.adminCommand({configureFailPoint: "voteNoInElection", mode: "off"}));
+assert.commandWorked(testNode.adminCommand(
+ {configureFailPoint: "voteYesInDryRunButNoInRealElection", mode: "alwaysOn"}));
+
+// Attempt to step up the original primary.
+assert.commandFailedWithCode(originalPrimary.adminCommand({replSetStepUp: 1}),
+ ErrorCodes.CommandFailed);
+
+testNodeReplSetGetStatus = assert.commandWorked(testNode.adminCommand({replSetGetStatus: 1}));
+testNodeElectionParticipantMetrics = testNodeReplSetGetStatus.electionParticipantMetrics;
+
+// Since the election succeeded in the dry run, a new term was encountered by the secondary.
+// However, because the real election failed, there was no new term oplog entry, so these fields
+// should not be set.
+assert(!testNodeElectionParticipantMetrics.newTermStartDate,
+ () => "Response should not have an 'electionParticipantMetrics.newTermStartDate' field: " +
+ tojson(testNodeElectionParticipantMetrics));
+assert(!testNodeElectionParticipantMetrics.newTermAppliedDate,
+ () => "Response should not have an 'electionParticipantMetrics.newTermAppliedDate' field: " +
+ tojson(testNodeElectionParticipantMetrics));
+
+rst.stopSet();
+})(); \ No newline at end of file
diff --git a/src/mongo/db/pipeline/document_source_change_stream_test.cpp b/src/mongo/db/pipeline/document_source_change_stream_test.cpp
index 1ce22080077..b918a53ca43 100644
--- a/src/mongo/db/pipeline/document_source_change_stream_test.cpp
+++ b/src/mongo/db/pipeline/document_source_change_stream_test.cpp
@@ -1423,8 +1423,8 @@ TEST_F(ChangeStreamStageTest, MatchFiltersCreateCollection) {
TEST_F(ChangeStreamStageTest, MatchFiltersNoOp) {
auto noOp = makeOplogEntry(OpTypeEnum::kNoop, // op type
{}, // namespace
- BSON("msg"
- << "new primary")); // o
+ BSON(repl::ReplicationCoordinator::newPrimaryMsgField
+ << repl::ReplicationCoordinator::newPrimaryMsg)); // o
checkTransformation(noOp, boost::none);
}
@@ -2024,8 +2024,8 @@ TEST_F(ChangeStreamStageDBTest, RenameFromUserToSystemCollectionShouldIncludeNot
TEST_F(ChangeStreamStageDBTest, MatchFiltersNoOp) {
OplogEntry noOp = makeOplogEntry(OpTypeEnum::kNoop,
NamespaceString(),
- BSON("msg"
- << "new primary"));
+ BSON(repl::ReplicationCoordinator::newPrimaryMsgField
+ << repl::ReplicationCoordinator::newPrimaryMsg));
checkTransformation(noOp, boost::none);
}
diff --git a/src/mongo/db/repl/oplog_applier_impl.cpp b/src/mongo/db/repl/oplog_applier_impl.cpp
index 97b9d37797f..5dc25227e47 100644
--- a/src/mongo/db/repl/oplog_applier_impl.cpp
+++ b/src/mongo/db/repl/oplog_applier_impl.cpp
@@ -43,6 +43,7 @@
#include "mongo/db/logical_session_id.h"
#include "mongo/db/repl/apply_ops.h"
#include "mongo/db/repl/insert_group.h"
+#include "mongo/db/repl/replication_metrics.h"
#include "mongo/db/repl/transaction_oplog_application.h"
#include "mongo/db/stats/timer_stats.h"
#include "mongo/platform/basic.h"
@@ -972,6 +973,16 @@ Status applyOplogEntryBatch(OperationContext* opCtx,
if (opType == OpTypeEnum::kNoop) {
incrementOpsAppliedStats();
+
+ auto opObj = op.getObject();
+ if (opObj.hasField(ReplicationCoordinator::newPrimaryMsgField) &&
+ opObj.getField(ReplicationCoordinator::newPrimaryMsgField).str() ==
+ ReplicationCoordinator::newPrimaryMsg) {
+
+ ReplicationMetrics::get(opCtx).setParticipantNewTermDates(op.getWallClockTime(),
+ applyStartTime);
+ }
+
return Status::OK();
} else if (OplogEntry::isCrudOpType(opType)) {
return finishApply(writeConflictRetry(opCtx, "applyOplogEntryBatch_CRUD", nss.ns(), [&] {
diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h
index 63d51768d53..8bc865775b0 100644
--- a/src/mongo/db/repl/replication_coordinator.h
+++ b/src/mongo/db/repl/replication_coordinator.h
@@ -930,6 +930,16 @@ public:
*/
virtual void finishRecoveryIfEligible(OperationContext* opCtx) = 0;
+ /**
+ * Field name of the newPrimaryMsg within the 'o' field in the new term oplog entry.
+ */
+ inline static constexpr StringData newPrimaryMsgField = "msg"_sd;
+
+ /**
+ * Message string passed in the new term oplog entry after a primary has stepped up.
+ */
+ inline static constexpr StringData newPrimaryMsg = "new primary"_sd;
+
protected:
ReplicationCoordinator();
};
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
index 5be36ed3881..e7489542f40 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
@@ -450,8 +450,8 @@ OpTime ReplicationCoordinatorExternalStateImpl::onTransitionToPrimary(OperationC
WriteUnitOfWork wuow(opCtx);
opCtx->getClient()->getServiceContext()->getOpObserver()->onOpMessage(
opCtx,
- BSON("msg"
- << "new primary"));
+ BSON(ReplicationCoordinator::newPrimaryMsgField
+ << ReplicationCoordinator::newPrimaryMsg));
wuow.commit();
});
const auto loadLastOpTimeAndWallTimeResult = loadLastOpTimeAndWallTime(opCtx);
@@ -459,7 +459,7 @@ OpTime ReplicationCoordinatorExternalStateImpl::onTransitionToPrimary(OperationC
auto opTimeToReturn = loadLastOpTimeAndWallTimeResult.getValue().opTime;
auto newTermStartDate = loadLastOpTimeAndWallTimeResult.getValue().wallTime;
- ReplicationMetrics::get(opCtx).setNewTermStartDate(newTermStartDate);
+ ReplicationMetrics::get(opCtx).setCandidateNewTermStartDate(newTermStartDate);
auto replCoord = ReplicationCoordinator::get(opCtx);
replCoord->createWMajorityWriteAvailabilityDateWaiter(opTimeToReturn);
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index 8925e576199..a15e1944f92 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -3757,7 +3757,9 @@ Status ReplicationCoordinatorImpl::processReplSetRequestVotes(
const int candidateIndex = args.getCandidateIndex();
LastVote lastVote{args.getTerm(), candidateIndex};
- if (response->getVoteGranted()) {
+ const bool votedForCandidate = response->getVoteGranted();
+
+ if (votedForCandidate) {
Status status = _externalState->storeLocalLastVoteDocument(opCtx, lastVote);
if (!status.isOK()) {
error() << "replSetRequestVotes failed to store LastVote document; " << status;
@@ -3767,7 +3769,6 @@ Status ReplicationCoordinatorImpl::processReplSetRequestVotes(
// If the vote was not granted to the candidate, we still want to track metrics around the
// node's participation in the election.
- const bool votedForCandidate = response->getVoteGranted();
const long long electionTerm = args.getTerm();
const Date_t lastVoteDate = _replExecutor->now();
const int electionCandidateMemberId =
@@ -3926,6 +3927,10 @@ EventHandle ReplicationCoordinatorImpl::_updateTerm_inlock(
auto now = _replExecutor->now();
TopologyCoordinator::UpdateTermResult localUpdateTermResult = _topCoord->updateTerm(term, now);
if (localUpdateTermResult == TopologyCoordinator::UpdateTermResult::kUpdatedTerm) {
+ // When the node discovers a new term, the new term date metrics are now out-of-date, so we
+ // clear them.
+ ReplicationMetrics::get(getServiceContext()).clearParticipantNewTermDates();
+
_termShadow.store(term);
_cancelPriorityTakeover_inlock();
_cancelAndRescheduleElectionTimeout_inlock();
diff --git a/src/mongo/db/repl/replication_metrics.cpp b/src/mongo/db/repl/replication_metrics.cpp
index 2eef9206b55..addb4f7027d 100644
--- a/src/mongo/db/repl/replication_metrics.cpp
+++ b/src/mongo/db/repl/replication_metrics.cpp
@@ -300,7 +300,7 @@ void ReplicationMetrics::setNumCatchUpOps(long numCatchUpOps) {
_updateAverageCatchUpOps(lk);
}
-void ReplicationMetrics::setNewTermStartDate(Date_t newTermStartDate) {
+void ReplicationMetrics::setCandidateNewTermStartDate(Date_t newTermStartDate) {
stdx::lock_guard<Latch> lk(_mutex);
_electionCandidateMetrics.setNewTermStartDate(newTermStartDate);
}
@@ -328,6 +328,15 @@ BSONObj ReplicationMetrics::getElectionCandidateMetricsBSON() {
return BSONObj();
}
+void ReplicationMetrics::clearElectionCandidateMetrics() {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _electionCandidateMetrics.setTargetCatchupOpTime(boost::none);
+ _electionCandidateMetrics.setNumCatchUpOps(boost::none);
+ _electionCandidateMetrics.setNewTermStartDate(boost::none);
+ _electionCandidateMetrics.setWMajorityWriteAvailabilityDate(boost::none);
+ _nodeIsCandidateOrPrimary = false;
+}
+
void ReplicationMetrics::setElectionParticipantMetrics(const bool votedForCandidate,
const long long electionTerm,
const Date_t lastVoteDate,
@@ -357,13 +366,17 @@ BSONObj ReplicationMetrics::getElectionParticipantMetricsBSON() {
return BSONObj();
}
-void ReplicationMetrics::clearElectionCandidateMetrics() {
+void ReplicationMetrics::setParticipantNewTermDates(Date_t newTermStartDate,
+ Date_t newTermAppliedDate) {
stdx::lock_guard<Latch> lk(_mutex);
- _electionCandidateMetrics.setTargetCatchupOpTime(boost::none);
- _electionCandidateMetrics.setNumCatchUpOps(boost::none);
- _electionCandidateMetrics.setNewTermStartDate(boost::none);
- _electionCandidateMetrics.setWMajorityWriteAvailabilityDate(boost::none);
- _nodeIsCandidateOrPrimary = false;
+ _electionParticipantMetrics.setNewTermStartDate(newTermStartDate);
+ _electionParticipantMetrics.setNewTermAppliedDate(newTermAppliedDate);
+}
+
+void ReplicationMetrics::clearParticipantNewTermDates() {
+ stdx::lock_guard<Latch> lk(_mutex);
+ _electionParticipantMetrics.setNewTermStartDate(boost::none);
+ _electionParticipantMetrics.setNewTermAppliedDate(boost::none);
}
void ReplicationMetrics::_updateAverageCatchUpOps(WithLock lk) {
diff --git a/src/mongo/db/repl/replication_metrics.h b/src/mongo/db/repl/replication_metrics.h
index 24ed57fe5a7..108510bbcd8 100644
--- a/src/mongo/db/repl/replication_metrics.h
+++ b/src/mongo/db/repl/replication_metrics.h
@@ -92,7 +92,7 @@ public:
const boost::optional<int> priorPrimary);
void setTargetCatchupOpTime(OpTime opTime);
void setNumCatchUpOps(long numCatchUpOps);
- void setNewTermStartDate(Date_t newTermStartDate);
+ void setCandidateNewTermStartDate(Date_t newTermStartDate);
void setWMajorityWriteAvailabilityDate(Date_t wMajorityWriteAvailabilityDate);
boost::optional<OpTime> getTargetCatchupOpTime_forTesting();
@@ -116,6 +116,9 @@ public:
const double priorityAtElection);
BSONObj getElectionParticipantMetricsBSON();
+ void setParticipantNewTermDates(Date_t newTermStartDate, Date_t newTermAppliedDate);
+ void clearParticipantNewTermDates();
+
private:
class ElectionMetricsSSS;
diff --git a/src/mongo/db/repl/replication_metrics.idl b/src/mongo/db/repl/replication_metrics.idl
index 853ca5d2d5c..4fb062411ed 100644
--- a/src/mongo/db/repl/replication_metrics.idl
+++ b/src/mongo/db/repl/replication_metrics.idl
@@ -208,8 +208,17 @@ structs:
description: "Latest applied OpTime at the time of voting"
type: optime
maxAppliedOpTimeInSet:
- description: "Highest applied time of any node in this replica set, as currently known by this node"
+ description: "Highest applied time of any node in this replica set, as currently
+ known by this node"
type: optime
priorityAtElection:
description: "The node's priority at the time of the election"
type: double
+ newTermStartDate:
+ description: "Time the new term oplog entry was written by the primary"
+ type: date
+ optional: true
+ newTermAppliedDate:
+ description: "Time this node applied the new term oplog entry"
+ type: date
+ optional: true
diff --git a/src/mongo/db/repl/topology_coordinator.cpp b/src/mongo/db/repl/topology_coordinator.cpp
index f254d311986..1de492a3eef 100644
--- a/src/mongo/db/repl/topology_coordinator.cpp
+++ b/src/mongo/db/repl/topology_coordinator.cpp
@@ -65,7 +65,8 @@ namespace mongo {
namespace repl {
MONGO_FAIL_POINT_DEFINE(forceSyncSourceCandidate);
-MONGO_FAIL_POINT_DEFINE(forceVoteInElection);
+MONGO_FAIL_POINT_DEFINE(voteNoInElection);
+MONGO_FAIL_POINT_DEFINE(voteYesInDryRunButNoInRealElection);
// If this fail point is enabled, TopologyCoordinator::shouldChangeSyncSource() will ignore
// the option TopologyCoordinator::Options::maxSyncSourceLagSecs. The sync source will not be
@@ -2694,13 +2695,25 @@ void TopologyCoordinator::processReplSetRequestVotes(const ReplSetRequestVotesAr
ReplSetRequestVotesResponse* response) {
response->setTerm(_term);
- if (MONGO_unlikely(forceVoteInElection.shouldFail())) {
+ if (MONGO_unlikely(voteNoInElection.shouldFail())) {
+ log() << "failpoint voteNoInElection enabled";
+ response->setVoteGranted(false);
+ response->setReason(str::stream() << "forced to vote no during dry run election due to "
+ "failpoint voteNoInElection set");
+ return;
+ }
+
+ if (MONGO_unlikely(voteYesInDryRunButNoInRealElection.shouldFail())) {
+ log() << "failpoint voteYesInDryRunButNoInRealElection enabled";
if (args.isADryRun()) {
response->setVoteGranted(true);
+ response->setReason(str::stream() << "forced to vote yes in dry run due to failpoint "
+ "voteYesInDryRunButNoInRealElection set");
} else {
response->setVoteGranted(false);
response->setReason(str::stream()
- << "forced to vote no due to failpoint forceVoteInElection set");
+ << "forced to vote no in real election due to failpoint "
+ "voteYesInDryRunButNoInRealElection set");
}
return;
}
View Lorry Controller Status