diff options
author | Xuerui Fa <xuerui.fa@mongodb.com> | 2020-03-13 16:31:02 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-03-17 18:45:56 +0000 |
commit | 1909cd159e4b0b3678513c8e3f83286051148b99 (patch) | |
tree | 8e6dd733dd61787028083b10d7fc7da2ebfc16cc | |
parent | ab8166fcd2855cc71e895101f5d7e6450b9ae8ed (diff) | |
download | mongo-1909cd159e4b0b3678513c8e3f83286051148b99.tar.gz |
SERVER-46195: Hang secondary before validate command in rollback_after_enabling_majority_reads.js
-rw-r--r-- | jstests/replsets/rollback_after_enabling_majority_reads.js | 32 | ||||
-rw-r--r-- | src/mongo/db/repl/sync_tail.cpp | 8 |
2 files changed, 37 insertions, 3 deletions
diff --git a/jstests/replsets/rollback_after_enabling_majority_reads.js b/jstests/replsets/rollback_after_enabling_majority_reads.js index 0a3b11ddf53..6a9b0c354e6 100644 --- a/jstests/replsets/rollback_after_enabling_majority_reads.js +++ b/jstests/replsets/rollback_after_enabling_majority_reads.js @@ -52,13 +52,27 @@ let allowedExitCode = 14; try { - rollbackTest.restartNode(0, 15, {enableMajorityReadConcern: "false"}, allowedExitCode); + rollbackTest.restartNode(0, + 15, + { + enableMajorityReadConcern: "false", + setParameter: 'failpoint.hangAfterTransitionToSecondary=' + + tojson({mode: 'alwaysOn'}) + }, + allowedExitCode); } catch (e) { if ((e.name === "StopError") && (e.returnCode === -6)) { // Retry once if we get error -6 instead of 14. This can happen if we get an fassert // while already in the middle of crashing from the expected fassert above. allowedExitCode = 0; - rollbackTest.restartNode(0, 15, {enableMajorityReadConcern: "false"}, allowedExitCode); + rollbackTest.restartNode(0, + 15, + { + enableMajorityReadConcern: "false", + setParameter: 'failpoint.hangAfterTransitionToSecondary=' + + tojson({mode: 'alwaysOn'}) + }, + allowedExitCode); } else { throw e; } @@ -67,7 +81,16 @@ // Fix counts for "local.startup_log", since they are corrupted by this rollback. // transitionToSteadyStateOperations() checks collection counts. replTest.waitForState(rollbackNode, ReplSetTest.State.SECONDARY); + + // Hang the node to prevent any state transition from occurring before the validate command + // executes. + checkLog.contains(rollbackNode, "fail point hangAfterTransitionToSecondary enabled."); assert.commandWorked(rollbackNode.getDB("local").runCommand({validate: "startup_log"})); + + // Turn off the fail point to allow the node to proceed. + assert.commandWorked(rollbackNode.adminCommand( + {configureFailPoint: 'hangAfterTransitionToSecondary', mode: 'off'})); + rollbackTest.transitionToSteadyStateOperations(); assert.commandWorked(rollbackTest.getPrimary().getDB(dbName)[collName].insert( @@ -78,7 +101,10 @@ jsTest.log( "Test that rollback succeeds with enableMajorityReadConcern=true once we have set a stable timestamp."); - rollbackTest.restartNode(0, 15, {enableMajorityReadConcern: "true"}); + + // Since 'restartNode' carries previous startup params, we set 'setParameter' to an empty obj to + // avoid turning on 'hangAfterTransitionToSecondary' again. + rollbackTest.restartNode(0, 15, {enableMajorityReadConcern: "true", setParameter: {}}); // Make sure node 0 is the primary. let node = replTest.nodes[0]; diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp index 7a9e4b46d57..38c8b07256c 100644 --- a/src/mongo/db/repl/sync_tail.cpp +++ b/src/mongo/db/repl/sync_tail.cpp @@ -98,6 +98,7 @@ namespace { MONGO_FAIL_POINT_DEFINE(pauseBatchApplicationBeforeCompletion); MONGO_FAIL_POINT_DEFINE(hangAfterRecordingOpApplicationStartTime); +MONGO_FAIL_POINT_DEFINE(hangAfterTransitionToSecondary); /** * This variable determines the number of writer threads SyncTail will have. It can be overridden @@ -1012,6 +1013,13 @@ void SyncTail::_oplogApplication(OplogBuffer* oplogBuffer, // Transition to SECONDARY state, if possible. tryToGoLiveAsASecondary(&opCtx, replCoord, minValid); + if (MONGO_unlikely(hangAfterTransitionToSecondary.shouldFail()) && + replCoord->getMemberState().secondary()) { + log() << "fail point hangAfterTransitionToSecondary enabled. Blocking until fail point " + "is disabled."; + MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangAfterTransitionToSecondary); + } + // Blocks up to a second waiting for a batch to be ready to apply. If one doesn't become // ready in time, we'll loop again so we can do the above checks periodically. OpQueue ops = batcher->getNextBatch(Seconds(1)); |