diff options
38 files changed, 103 insertions, 313 deletions
diff --git a/jstests/libs/override_methods/continuous_stepdown.js b/jstests/libs/override_methods/continuous_stepdown.js index d9ea61240bd..9fd8668a663 100644 --- a/jstests/libs/override_methods/continuous_stepdown.js +++ b/jstests/libs/override_methods/continuous_stepdown.js @@ -69,9 +69,7 @@ let ContinuousStepdown; try { // The config primary may unexpectedly step down during startup if under heavy - // load and too slowly processing heartbeats. When it steps down, it closes all of - // its connections. ReplSetTest will therefore retry discovery of the replica set's - // config. + // load and too slowly processing heartbeats. const replSet = new ReplSetTest(seedNode); let primary = replSet.getPrimary(); @@ -79,15 +77,8 @@ let ContinuousStepdown; while (stopCounter.getCount() > 0) { print("*** Stepping down " + primary); - assert.throws(function() { - let result = primary.adminCommand( - {replSetStepDown: options.stepdownDurationSecs, force: true}); - print("replSetStepDown command did not throw and returned: " + - tojson(result)); - - // The call to replSetStepDown should never succeed. - assert.commandWorked(result); - }); + assert.commandWorked(primary.adminCommand( + {replSetStepDown: options.stepdownDurationSecs, force: true})); // Wait for primary to get elected and allow the test to make some progress // before attempting another stepdown. diff --git a/jstests/noPassthrough/auto_retry_on_network_error.js b/jstests/noPassthrough/auto_retry_on_network_error.js index 5667b1c0a55..57e36a3b036 100644 --- a/jstests/noPassthrough/auto_retry_on_network_error.js +++ b/jstests/noPassthrough/auto_retry_on_network_error.js @@ -16,29 +16,26 @@ load("jstests/libs/override_methods/auto_retry_on_network_error.js"); load("jstests/replsets/rslib.js"); - function stepDownPrimary(rst) { - // Since we expect the mongo shell's connection to get severed as a result of running the - // "replSetStepDown" command, we temporarily disable the retry on network error behavior. - TestData.skipRetryOnNetworkError = true; - try { - const primary = rst.getPrimary(); - const error = assert.throws(function() { - const res = primary.adminCommand({replSetStepDown: 1, force: true}); - print("replSetStepDown did not throw exception but returned: " + tojson(res)); - }); - assert(isNetworkError(error), - "replSetStepDown did not disconnect client; failed with " + tojson(error)); - - // We use the reconnect() function to run a command against the former primary that - // acquires the global lock to ensure that it has finished stepping down and has - // therefore closed all of its client connections. This ensures commands sent on other - // connections to the former primary trigger a network error rather than potentially - // returning a "not master" error while the server is in the midst of closing client - // connections. - reconnect(primary); - } finally { - TestData.skipRetryOnNetworkError = false; - } + function getThreadName(db) { + let myUri = db.adminCommand({whatsmyuri: 1}).you; + return db.getSiblingDB("admin") + .aggregate([{$currentOp: {localOps: true}}, {$match: {client: myUri}}]) + .toArray()[0] + .desc; + } + + function failNextCommand(db, command) { + let threadName = getThreadName(db); + + assert.commandWorked(db.adminCommand({ + configureFailPoint: "failCommand", + mode: {times: 1}, + data: { + closeConnection: true, + failCommands: [command], + threadName: threadName, + } + })); } const rst = new ReplSetTest({nodes: 1}); @@ -55,22 +52,22 @@ // allow automatic re-targeting of the primary on NotMaster errors. const db = new Mongo(rst.getURL()).startSession({retryWrites: true}).getDatabase(dbName); - // Commands with no stepdowns should work as normal. + // Commands with no disconnections should work as normal. assert.commandWorked(db.runCommand({ping: 1})); assert.commandWorked(db.runCommandWithMetadata({ping: 1}, {}).commandReply); // Read commands are automatically retried on network errors. - stepDownPrimary(rst); + failNextCommand(db, "find"); assert.commandWorked(db.runCommand({find: collName})); - stepDownPrimary(rst); + failNextCommand(db, "find"); assert.commandWorked(db.runCommandWithMetadata({find: collName}, {}).commandReply); // Retryable write commands that can be retried succeed. - stepDownPrimary(rst); + failNextCommand(db, "insert"); assert.writeOK(db[collName].insert({x: 1})); - stepDownPrimary(rst); + failNextCommand(db, "insert"); assert.commandWorked(db.runCommandWithMetadata({ insert: collName, documents: [{x: 2}, {x: 3}], @@ -82,7 +79,7 @@ // Retryable write commands that cannot be retried (i.e. no transaction number, no session id, // or are unordered) throw. - stepDownPrimary(rst); + failNextCommand(db, "insert"); assert.throws(function() { db.runCommand({insert: collName, documents: [{x: 1}, {x: 2}], ordered: false}); }); @@ -91,7 +88,7 @@ // the primary, so the connection to it can be closed. assert.commandWorked(db.runCommandWithMetadata({ping: 1}, {}).commandReply); - stepDownPrimary(rst); + failNextCommand(db, "insert"); assert.throws(function() { db.runCommandWithMetadata({insert: collName, documents: [{x: 1}, {x: 2}], ordered: false}, {}); @@ -100,13 +97,13 @@ // getMore commands can't be retried because we won't know whether the cursor was advanced or // not. let cursorId = assert.commandWorked(db.runCommand({find: collName, batchSize: 0})).cursor.id; - stepDownPrimary(rst); + failNextCommand(db, "getMore"); assert.throws(function() { db.runCommand({getMore: cursorId, collection: collName}); }); cursorId = assert.commandWorked(db.runCommand({find: collName, batchSize: 0})).cursor.id; - stepDownPrimary(rst); + failNextCommand(db, "getMore"); assert.throws(function() { db.runCommandWithMetadata({getMore: cursorId, collection: collName}, {}); }); diff --git a/jstests/noPassthrough/change_stream_failover.js b/jstests/noPassthrough/change_stream_failover.js index b5c81ab4f7c..8168c7722de 100644 --- a/jstests/noPassthrough/change_stream_failover.js +++ b/jstests/noPassthrough/change_stream_failover.js @@ -44,9 +44,7 @@ assert.docEq(firstChange.fullDocument, {_id: 0}); // Make the primary step down - assert.throws(function() { - primaryDB.adminCommand({replSetStepDown: 30}); - }); + assert.commandWorked(primaryDB.adminCommand({replSetStepDown: 30})); // Now wait for another primary to be elected. const newPrimary = rst.getPrimary(); diff --git a/jstests/noPassthrough/replica_set_connection_error_codes.js b/jstests/noPassthrough/replica_set_connection_error_codes.js index 3e726a77296..7deebdfcc27 100644 --- a/jstests/noPassthrough/replica_set_connection_error_codes.js +++ b/jstests/noPassthrough/replica_set_connection_error_codes.js @@ -33,14 +33,9 @@ const[secondary1, secondary2] = rst.getSecondaries(); function stepDownPrimary(rst) { - const awaitShell = startParallelShell(function() { - const error = assert.throws(function() { - const res = db.adminCommand({replSetStepDown: 60, force: true}); - print("replSetStepDown did not throw exception but returned: " + tojson(res)); - }); - assert(isNetworkError(error), - "replSetStepDown did not disconnect client; failed with " + tojson(error)); - }, directConn.port); + const awaitShell = startParallelShell( + () => assert.commandWorked(db.adminCommand({replSetStepDown: 60, force: true})), + directConn.port); // We wait for the primary to transition to the SECONDARY state to ensure we're waiting // until after the parallel shell has started the replSetStepDown command and the server is diff --git a/jstests/noPassthrough/replica_set_connection_stepdown.js b/jstests/noPassthrough/replica_set_connection_stepdown.js index c6c5625ba4a..ab11d72d465 100644 --- a/jstests/noPassthrough/replica_set_connection_stepdown.js +++ b/jstests/noPassthrough/replica_set_connection_stepdown.js @@ -16,14 +16,9 @@ "expected " + rsConn.host + " to be a replica set connection string"); function stepDownPrimary(rst) { - const awaitShell = startParallelShell(function() { - const error = assert.throws(function() { - const res = db.adminCommand({replSetStepDown: 60, force: true}); - print("replSetStepDown did not throw exception but returned: " + tojson(res)); - }); - assert(isNetworkError(error), - "replSetStepDown did not disconnect client; failed with " + tojson(error)); - }, directConn.port); + const awaitShell = startParallelShell( + () => assert.commandWorked(db.adminCommand({replSetStepDown: 60, force: true})), + directConn.port); // We wait for the primary to transition to the SECONDARY state to ensure we're waiting // until after the parallel shell has started the replSetStepDown command and the server is diff --git a/jstests/noPassthrough/snapshot_cursor_shutdown_stepdown.js b/jstests/noPassthrough/snapshot_cursor_shutdown_stepdown.js index ec2cbfd5254..f73eb952043 100644 --- a/jstests/noPassthrough/snapshot_cursor_shutdown_stepdown.js +++ b/jstests/noPassthrough/snapshot_cursor_shutdown_stepdown.js @@ -81,9 +81,8 @@ // replSetStepDown. // function replSetStepDown(replSetTest) { - assert.throws(function() { - replSetTest.getPrimary().adminCommand({replSetStepDown: 60, force: true}); - }); + assert.commandWorked( + replSetTest.getPrimary().adminCommand({replSetStepDown: 60, force: true})); } testStepdown(replSetStepDown); diff --git a/jstests/noPassthrough/stepdown_during_getmore_then_legacy_killop.js b/jstests/noPassthrough/stepdown_during_getmore_then_legacy_killop.js deleted file mode 100644 index 7211a5c2402..00000000000 --- a/jstests/noPassthrough/stepdown_during_getmore_then_legacy_killop.js +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Test the situation where - * - A stepdown happens at the end of a getMore operation, after it has checked for interrupt the - last time. The command will fail because the connection with the client will be cut as part of - the stepdown, but the cursor will still exist. - * - Kill the cursor using OP_KILL_CURSORS. - * - * This is intended to reproduce SERVER-37838. - * @tags: [requires_replication] - */ -(function() { - "use strict"; - - load("jstests/libs/fixture_helpers.js"); // For waitUntilServerHangsOnFailPoint(). - - // This test runs manual getMores using different connections, which will not inherit the - // implicit session of the cursor establishing command. - TestData.disableImplicitSessions = true; - - const kName = "stepdown_during_getmore_then_legacy_killop"; - const rst = new ReplSetTest({name: kName, nodes: 2}); - rst.startSet(); - rst.initiate(); - - const primary = rst.getPrimary(); - const testDB = primary.getDB("test"); - const coll = testDB[kName]; - - for (let i = 0; i < 10; i++) { - assert.commandWorked(coll.insert({x: 1})); - } - rst.awaitReplication(); - - // Start an aggregation. - const csCmdRes = assert.commandWorked( - coll.runCommand({aggregate: coll.getName(), pipeline: [], cursor: {batchSize: 2}})); - - const cursorId = csCmdRes.cursor.id; - - // Start a getMore that waits on a failpoint which comes after any checks for interrupt. This - // ensures that once we kill the operation (by causing a stepdown), it will still succeed, - // since it will have no opportunity to notice it's been killed. - const kFailPointName = "waitBeforeUnpinningOrDeletingCursorAfterGetMoreBatch"; - assert.commandWorked( - primary.adminCommand({configureFailPoint: kFailPointName, mode: 'alwaysOn'})); - - let code = `const collName = "${coll.getName()}";`; - code += `const cursorId = ${cursorId};`; - function runGetMore() { - const getMoreCmd = {getMore: cursorId, collection: collName, batchSize: 2}; - - const exn = assert.throws(() => db[collName].runCommand(getMoreCmd)); - assert.gte(exn.message.indexOf("network error"), 0); - } - code += `(${runGetMore.toString()})();`; - const getMoreJoiner = startParallelShell(code, primary.port); - - // Be sure the server is hanging on the failpoint. - assert.soon(function() { - const filter = {"msg": kFailPointName, "cursor.cursorId": cursorId}; - const ops = primary.getDB("admin") - .aggregate([{$currentOp: {allUsers: true}}, {$match: filter}]) - .toArray(); - return ops.length == 1; - }); - - // Now step down the node. This will cause the getMore operation to be marked as killed, though - // it will never check for interrupt because it's hanging on a failpoint that's past any - // interrupt checks. When the operation is done using its cursor, it will realize that its been - // killed and mark the _cursor_ as having been killed, so that further attempts to use it will - // fail. - assert.throws(function() { - primary.adminCommand({replSetStepDown: ReplSetTest.kDefaultTimeoutMS, force: true}); - }); - - assert.commandWorked(primary.adminCommand({configureFailPoint: kFailPointName, mode: 'off'})); - getMoreJoiner(); - - // Close the cursor with an OP_KILL_CURSORS. - (function() { - testDB.getMongo().forceReadMode("legacy"); - - const curs = new DBCommandCursor(testDB, csCmdRes); - curs.close(); - - testDB.getMongo().forceReadMode("commands"); - })(); - - rst.stopSet(); -})(); diff --git a/jstests/noPassthrough/stepdown_query.js b/jstests/noPassthrough/stepdown_query.js index 2db51e71558..6351493bbb4 100644 --- a/jstests/noPassthrough/stepdown_query.js +++ b/jstests/noPassthrough/stepdown_query.js @@ -37,9 +37,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; assert.eq(0, cursor.objsLeftInBatch()); var primary = rst.getPrimary(); var secondary = rst.getSecondary(); - assert.throws(function() { - primary.getDB("admin").runCommand({replSetStepDown: 60, force: true}); - }); + assert.commandWorked(primary.getDB("admin").runCommand({replSetStepDown: 60, force: true})); rst.waitForState(primary, ReplSetTest.State.SECONDARY); if (waitForPrimary) { rst.waitForState(secondary, ReplSetTest.State.PRIMARY); diff --git a/jstests/replsets/change_stream_stepdown.js b/jstests/replsets/change_stream_stepdown.js index 02cf3f68cad..dbd785c934a 100644 --- a/jstests/replsets/change_stream_stepdown.js +++ b/jstests/replsets/change_stream_stepdown.js @@ -7,11 +7,8 @@ load("jstests/libs/write_concern_util.js"); // for [stop|restart]ServerReplication. const name = "change_stream_speculative_majority"; - const replTest = new ReplSetTest({ - name: name, - nodes: [{setParameter: {closeConnectionsOnStepdown: false}}, {}], - nodeOptions: {enableMajorityReadConcern: 'false'} - }); + const replTest = new ReplSetTest( + {name: name, nodes: [{}, {}], nodeOptions: {enableMajorityReadConcern: 'false'}}); replTest.startSet(); replTest.initiate(); diff --git a/jstests/replsets/disconnect_on_legacy_write_to_secondary.js b/jstests/replsets/disconnect_on_legacy_write_to_secondary.js index d3456898945..b230c4f7f27 100644 --- a/jstests/replsets/disconnect_on_legacy_write_to_secondary.js +++ b/jstests/replsets/disconnect_on_legacy_write_to_secondary.js @@ -6,8 +6,7 @@ load("jstests/libs/check_log.js"); - const rst = new ReplSetTest( - {nodes: [{setParameter: {closeConnectionsOnStepdown: false}}, {rsConfig: {priority: 0}}]}); + const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]}); rst.startSet(); rst.initiate(); diff --git a/jstests/replsets/initial_sync_during_stepdown.js b/jstests/replsets/initial_sync_during_stepdown.js index af86780813c..5b8f9ad3b11 100644 --- a/jstests/replsets/initial_sync_during_stepdown.js +++ b/jstests/replsets/initial_sync_during_stepdown.js @@ -11,8 +11,7 @@ const dbName = testName; const collName = "testcoll"; - const rst = new ReplSetTest( - {nodes: [{setParameter: {closeConnectionsOnStepdown: false}}, {rsConfig: {priority: 0}}]}); + const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]}); rst.startSet(); rst.initiate(); diff --git a/jstests/replsets/interrupted_batch_insert.js b/jstests/replsets/interrupted_batch_insert.js index 81d925f5226..f6d354fb883 100644 --- a/jstests/replsets/interrupted_batch_insert.js +++ b/jstests/replsets/interrupted_batch_insert.js @@ -57,11 +57,10 @@ return {idx: i}; }); var coll = new Mongo(host).getCollection(collName); - assert.throws( - () => coll.insert(docsToInsert, - {writeConcern: {w: "majority", wtimeout: 5000}, ordered: true}), - [], - "network error"); + assert.commandFailedWithCode( + coll.insert(docsToInsert, + {writeConcern: {w: "majority", wtimeout: 5000}, ordered: true}), + ErrorCodes.InterruptedDueToStepDown); }, primary.host, collName, 10 * batchSize); worker.start(); diff --git a/jstests/replsets/last_error_reported_after_stepdown.js b/jstests/replsets/last_error_reported_after_stepdown.js index bb94bb87269..b595f5c1b55 100644 --- a/jstests/replsets/last_error_reported_after_stepdown.js +++ b/jstests/replsets/last_error_reported_after_stepdown.js @@ -7,8 +7,7 @@ load("jstests/libs/check_log.js"); - const rst = new ReplSetTest( - {nodes: [{setParameter: {closeConnectionsOnStepdown: false}}, {rsConfig: {priority: 0}}]}); + const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]}); rst.startSet(); rst.initiate(); diff --git a/jstests/replsets/last_vote.js b/jstests/replsets/last_vote.js index 12f13493f08..6ff7198b3d3 100644 --- a/jstests/replsets/last_vote.js +++ b/jstests/replsets/last_vote.js @@ -72,9 +72,7 @@ assertNodeHasLastVote(node, term, primary); }); } - assert.throws(function() { - primary.adminCommand({replSetStepDown: 60 * 10, force: true}); - }); + assert.commandWorked(primary.adminCommand({replSetStepDown: 60 * 10, force: true})); // Make sure a new primary has been established. rst.waitForState(primary, ReplSetTest.State.SECONDARY); diff --git a/jstests/replsets/linearizable_read_concern.js b/jstests/replsets/linearizable_read_concern.js index ba08690be06..ce98eb7cf34 100644 --- a/jstests/replsets/linearizable_read_concern.js +++ b/jstests/replsets/linearizable_read_concern.js @@ -122,11 +122,8 @@ load('jstests/libs/write_concern_util.js'); }); assert.eq(primary, replTest.getPrimary(), "Primary unexpectedly changed mid test."); jsTestLog("Making Primary step down"); - var stepDownException = assert.throws(function() { - var result = primary.adminCommand( - {"replSetStepDown": 100, secondaryCatchUpPeriodSecs: 0, "force": true}); - print('replSetStepDown did not throw exception but returned: ' + tojson(result)); - }); + assert.commandWorked(primary.adminCommand( + {"replSetStepDown": 100, secondaryCatchUpPeriodSecs: 0, "force": true})); parallelShell(); replTest.stopSet(); }()); diff --git a/jstests/replsets/no_disconnect_on_stepdown.js b/jstests/replsets/no_disconnect_on_stepdown.js index 8a73a07d0f0..05f0195a40b 100644 --- a/jstests/replsets/no_disconnect_on_stepdown.js +++ b/jstests/replsets/no_disconnect_on_stepdown.js @@ -6,8 +6,7 @@ load("jstests/libs/curop_helpers.js"); - const rst = new ReplSetTest( - {nodes: [{setParameter: {closeConnectionsOnStepdown: false}}, {rsConfig: {priority: 0}}]}); + const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]}); rst.startSet(); rst.initiate(); diff --git a/jstests/replsets/prepared_transaction_on_failover.js b/jstests/replsets/prepared_transaction_on_failover.js index faec21bc52b..18e9fbefdc2 100644 --- a/jstests/replsets/prepared_transaction_on_failover.js +++ b/jstests/replsets/prepared_transaction_on_failover.js @@ -81,9 +81,7 @@ function stepDownViaCommand() { jsTestLog("Stepping down primary via command"); - assert.throws(function() { - replTest.getPrimary().adminCommand({replSetStepDown: 10}); - }); + assert.commandWorked(replTest.getPrimary().adminCommand({replSetStepDown: 10})); } testTransactionsWithFailover(stepDownViaCommand); diff --git a/jstests/replsets/priority_takeover_one_node_higher_priority.js b/jstests/replsets/priority_takeover_one_node_higher_priority.js index b4385cc2376..c1d0a7e5dc3 100644 --- a/jstests/replsets/priority_takeover_one_node_higher_priority.js +++ b/jstests/replsets/priority_takeover_one_node_higher_priority.js @@ -27,12 +27,7 @@ // Primary should step down long enough for election to occur on secondary. var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - var stepDownException = assert.throws(function() { - var result = primary.adminCommand({replSetStepDown: replSet.kDefaultTimeoutMS / 1000}); - print('replSetStepDown did not throw exception but returned: ' + tojson(result)); - }); - assert(isNetworkError(stepDownException), - 'replSetStepDown did not disconnect client; failed with ' + tojson(stepDownException)); + assert.commandWorked(primary.adminCommand({replSetStepDown: replSet.kDefaultTimeoutMS / 1000})); // Step down primary and wait for node 1 to be promoted to primary. replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY); diff --git a/jstests/replsets/priority_takeover_two_nodes_equal_priority.js b/jstests/replsets/priority_takeover_two_nodes_equal_priority.js index e05805b1e96..2a7a8b610df 100644 --- a/jstests/replsets/priority_takeover_two_nodes_equal_priority.js +++ b/jstests/replsets/priority_takeover_two_nodes_equal_priority.js @@ -38,10 +38,8 @@ load('jstests/replsets/rslib.js'); ); jsTestLog("Stepping down the current primary."); - assert.throws(function() { - assert.commandWorked( - primary.adminCommand({replSetStepDown: 10 * 60, secondaryCatchUpPeriodSecs: 10 * 60})); - }); + assert.commandWorked( + primary.adminCommand({replSetStepDown: 10 * 60, secondaryCatchUpPeriodSecs: 10 * 60})); // Make sure the primary has stepped down. assert.neq(primary, replTest.getPrimary()); diff --git a/jstests/replsets/read_operations_during_step_down.js b/jstests/replsets/read_operations_during_step_down.js index 8120a41cc07..b03adf7a731 100644 --- a/jstests/replsets/read_operations_during_step_down.js +++ b/jstests/replsets/read_operations_during_step_down.js @@ -14,10 +14,7 @@ load("jstests/libs/curop_helpers.js"); // for waitForCurOpByFailPoint(). const dbName = "test"; const collName = "coll"; - var rst = new ReplSetTest({ - name: testName, - nodes: [{setParameter: {closeConnectionsOnStepdown: false}}, {rsConfig: {priority: 0}}] - }); + var rst = new ReplSetTest({name: testName, nodes: [{}, {rsConfig: {priority: 0}}]}); rst.startSet(); rst.initiate(); diff --git a/jstests/replsets/request_primary_stepdown.js b/jstests/replsets/request_primary_stepdown.js index c06706526aa..8ea0f78688c 100644 --- a/jstests/replsets/request_primary_stepdown.js +++ b/jstests/replsets/request_primary_stepdown.js @@ -23,12 +23,8 @@ clearRawMongoProgramOutput(); // Primary should step down long enough for election to occur on secondary. - var stepDownException = assert.throws(function() { - var result = primary.adminCommand({replSetStepDown: 70, secondaryCatchUpPeriodSecs: 60}); - print('replSetStepDown did not throw exception but returned: ' + tojson(result)); - }); - assert(isNetworkError(stepDownException), - 'replSetStepDown did not disconnect client; failed with ' + tojson(stepDownException)); + assert.commandWorked( + primary.adminCommand({replSetStepDown: 70, secondaryCatchUpPeriodSecs: 60})); // Wait for node 1 to be promoted to primary after node 0 stepped down. replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY, 60 * 1000); diff --git a/jstests/replsets/retryable_commit_transaction_after_failover.js b/jstests/replsets/retryable_commit_transaction_after_failover.js index b996b6800f7..0431d193875 100644 --- a/jstests/replsets/retryable_commit_transaction_after_failover.js +++ b/jstests/replsets/retryable_commit_transaction_after_failover.js @@ -58,11 +58,14 @@ })); jsTest.log("Step up the secondary"); + const oldPrimary = rst.getPrimary(); const oldSecondary = rst.getSecondary(); rst.stepUp(oldSecondary); // Wait until the other node becomes primary. assert.eq(oldSecondary, rst.getPrimary()); // Reconnect the connection to the new primary. + sessionDb.getMongo()._markNodeAsFailed( + oldPrimary.host, ErrorCodes.NotMaster, "Notice that primary is not master"); reconnect(sessionDb); jsTest.log("commitTransaction command is retryable after failover"); diff --git a/jstests/replsets/retryable_writes_failover.js b/jstests/replsets/retryable_writes_failover.js index 0c254de15dc..30eb069906d 100644 --- a/jstests/replsets/retryable_writes_failover.js +++ b/jstests/replsets/retryable_writes_failover.js @@ -13,13 +13,8 @@ } function stepDownPrimary(replTest) { - let exception = assert.throws(function() { - let res = assert.commandWorked( - replTest.getPrimary().adminCommand({replSetStepDown: 10, force: true})); - print("replSetStepDown did not throw exception but returned: " + tojson(res)); - }); - assert(isNetworkError(exception), - "replSetStepDown did not disconnect client; failed with " + tojson(exception)); + assert.commandWorked( + replTest.getPrimary().adminCommand({replSetStepDown: 10, force: true})); } const replTest = new ReplSetTest({nodes: 3}); diff --git a/jstests/replsets/stepdown.js b/jstests/replsets/stepdown.js index 52f8aef23a9..ecba409427f 100644 --- a/jstests/replsets/stepdown.js +++ b/jstests/replsets/stepdown.js @@ -67,15 +67,9 @@ try { 'Step down ' + master + ' expected error: ' + tojson(assert.commandFailed(master.getDB("admin").runCommand({replSetStepDown: 10})))); - // The server will disconnect the client on a successful forced stepdown so we use the - // presence of an exception to confirm the forced stepdown result. jsTestLog('Do stepdown of primary ' + master + ' that should work'); - var exceptionFromForcedStepDown = assert.throws(function() { - master.getDB("admin").runCommand( - {replSetStepDown: ReplSetTest.kDefaultTimeoutMS, force: true}); - }); - jsTestLog('Forced stepdown ' + master + ' expected failure: ' + - tojson(exceptionFromForcedStepDown)); + assert.commandWorked( + master.adminCommand({replSetStepDown: ReplSetTest.kDefaultTimeoutMS, force: true})); jsTestLog('Checking isMaster on ' + master); var r2 = assert.commandWorked(master.getDB("admin").runCommand({ismaster: 1})); diff --git a/jstests/replsets/stepdown3.js b/jstests/replsets/stepdown3.js index d125d7664f1..f40caabe242 100644 --- a/jstests/replsets/stepdown3.js +++ b/jstests/replsets/stepdown3.js @@ -32,26 +32,17 @@ // step down the primary asyncronously print("stepdown"); - var command = "sleep(4000); tojson(db.adminCommand( { replSetStepDown : 60, force : 1 } ));"; + var command = + "sleep(4000); assert.commandWorked(db.adminCommand( { replSetStepDown : 60, force : 1 } ));"; var awaitShell = startParallelShell(command, master.port); - print("getlasterror; should assert or return an error, depending on timing"); - var gleFunction = function() { - var result = - master.getDB("test").runCommand({getLastError: 1, w: 2, wtimeout: 10 * 60 * 1000}); - if (result.errmsg === "not master" || result.code == ErrorCodes.NotMaster || - result.code == ErrorCodes.InterruptedDueToStepDown) { - throw new Error("satisfy assert.throws()"); - } - print("failed to throw exception; GLE returned: "); - printjson(result); - }; - var result = assert.throws(gleFunction); + print("getlasterror; should return an error"); + let result = master.getDB("test").runCommand({getLastError: 1, w: 2, wtimeout: 10 * 60 * 1000}); + assert(ErrorCodes.isNotMasterError(result.code)); print("result of gle:"); printjson(result); - var exitCode = awaitShell({checkExitSuccess: false}); - assert.neq(0, exitCode, "expected replSetStepDown to close the shell's connection"); + awaitShell(); // unlock and shut down printjson(locked.getDB("admin").fsyncUnlock()); diff --git a/jstests/replsets/stepdown_long_wait_time.js b/jstests/replsets/stepdown_long_wait_time.js index 2b0c0def919..1d0aab19ac3 100644 --- a/jstests/replsets/stepdown_long_wait_time.js +++ b/jstests/replsets/stepdown_long_wait_time.js @@ -36,7 +36,8 @@ assert.writeOK(primary.getDB(name).foo.insert({x: 1}, options)); var stepDownCmd = function() { - db.getSiblingDB('admin').runCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60}); + assert.commandWorked( + db.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60})); }; var stepDowner = startParallelShell(stepDownCmd, primary.port); @@ -67,8 +68,7 @@ jsTestLog("Wait for PRIMARY " + primary.host + " to completely step down."); replSet.waitForState(primary, ReplSetTest.State.SECONDARY); - var exitCode = stepDowner({checkExitSuccess: false}); - assert.neq(0, exitCode, "expected replSetStepDown to close the shell's connection"); + var exitCode = stepDowner(); jsTestLog("Wait for SECONDARY " + secondary.host + " to become PRIMARY"); replSet.waitForState(secondary, ReplSetTest.State.PRIMARY); diff --git a/jstests/replsets/stepdown_needs_electable_secondary.js b/jstests/replsets/stepdown_needs_electable_secondary.js index 3adb0d2b43b..799a2e69a9b 100644 --- a/jstests/replsets/stepdown_needs_electable_secondary.js +++ b/jstests/replsets/stepdown_needs_electable_secondary.js @@ -46,15 +46,14 @@ function assertStepDownFailsWithExceededTimeLimit(node) { assert.commandFailedWithCode( - node.getDB("admin").runCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}), + node.adminCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}), ErrorCodes.ExceededTimeLimit, "step down did not fail with 'ExceededTimeLimit'"); } function assertStepDownSucceeds(node) { - assert.throws(function() { - node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60}); - }); + assert.commandWorked( + node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60})); } var primary = replTest.getPrimary(); diff --git a/jstests/replsets/stepdown_needs_majority.js b/jstests/replsets/stepdown_needs_majority.js index ae6c909d9fe..efc874fde3a 100644 --- a/jstests/replsets/stepdown_needs_majority.js +++ b/jstests/replsets/stepdown_needs_majority.js @@ -24,15 +24,14 @@ function assertStepDownFailsWithExceededTimeLimit(node) { assert.commandFailedWithCode( - node.getDB("admin").runCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}), + node.adminCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}), ErrorCodes.ExceededTimeLimit, "step down did not fail with 'ExceededTimeLimit'"); } function assertStepDownSucceeds(node) { - assert.throws(function() { - node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60}); - }); + assert.commandWorked( + node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60})); } function nodeIdStr(repltest, node) { diff --git a/jstests/replsets/transactions_during_step_down.js b/jstests/replsets/transactions_during_step_down.js index 7e8bb234f64..6453c9d4c01 100644 --- a/jstests/replsets/transactions_during_step_down.js +++ b/jstests/replsets/transactions_during_step_down.js @@ -14,8 +14,7 @@ const dbName = testName; const collName = "testcoll"; - const rst = new ReplSetTest( - {nodes: [{setParameter: {closeConnectionsOnStepdown: false}}, {rsConfig: {priority: 0}}]}); + const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]}); rst.startSet(); rst.initiate(); diff --git a/jstests/replsets/write_concern_after_stepdown.js b/jstests/replsets/write_concern_after_stepdown.js index f20e35e6e8c..40d2df98a36 100644 --- a/jstests/replsets/write_concern_after_stepdown.js +++ b/jstests/replsets/write_concern_after_stepdown.js @@ -88,17 +88,8 @@ nodes[2].acceptConnectionsFrom(nodes[0]); // Allow the old primary to finish stepping down so that shutdown can finish. - var res = null; - try { - res = nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'off'}); - } catch (e) { - // Expected - once we disable the fail point the stepdown will proceed and it's racy whether - // the stepdown closes all connections before or after the configureFailPoint command - // returns - } - if (res) { - assert.commandWorked(res); - } + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'off'})); joinMajorityWriter(); diff --git a/jstests/replsets/write_concern_after_stepdown_and_stepup.js b/jstests/replsets/write_concern_after_stepdown_and_stepup.js index e2667d4f66a..2908e5e3717 100644 --- a/jstests/replsets/write_concern_after_stepdown_and_stepup.js +++ b/jstests/replsets/write_concern_after_stepdown_and_stepup.js @@ -34,9 +34,7 @@ function stepUp(node) { var primary = rst.getPrimary(); if (primary != node) { - assert.throws(function() { - primary.adminCommand({replSetStepDown: 60 * 5}); - }); + assert.commandWorked(primary.adminCommand({replSetStepDown: 60 * 5})); } waitForPrimary(node); } diff --git a/jstests/sharding/change_stream_shard_failover.js b/jstests/sharding/change_stream_shard_failover.js index 27a141bcfed..f4b3007bd30 100644 --- a/jstests/sharding/change_stream_shard_failover.js +++ b/jstests/sharding/change_stream_shard_failover.js @@ -74,11 +74,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; // Make one of the primaries step down. const oldPrimary = st.rs0.getPrimary(); - const stepDownError = assert.throws(function() { - oldPrimary.adminCommand({replSetStepDown: 300, force: true}); - }); - assert(isNetworkError(stepDownError), - "replSetStepDown did not disconnect client; failed with " + tojson(stepDownError)); + assert.commandWorked(oldPrimary.adminCommand({replSetStepDown: 300, force: true})); st.rs0.awaitNodesAgreeOnPrimary(); const newPrimary = st.rs0.getPrimary(); diff --git a/jstests/sharding/cursor_valid_after_shard_stepdown.js b/jstests/sharding/cursor_valid_after_shard_stepdown.js index 01e7ee57c30..b717d73cac8 100644 --- a/jstests/sharding/cursor_valid_after_shard_stepdown.js +++ b/jstests/sharding/cursor_valid_after_shard_stepdown.js @@ -28,9 +28,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; assert.neq(Timestamp(0, 0), shardVersionBeforeStepdown); // Stepdown the primary of the shard and ensure that that cursor can still be read - assert.throws(function() { - assert.commandWorked(st.rs0.getPrimary().adminCommand({replSetStepDown: 60, force: 1})); - }); + assert.commandWorked(st.rs0.getPrimary().adminCommand({replSetStepDown: 60, force: 1})); var getMoreCursor = assert.commandWorked(db.runCommand({getMore: findCursor.id, collection: 'TestColl'})) diff --git a/jstests/sharding/movechunk_interrupt_at_primary_stepdown.js b/jstests/sharding/movechunk_interrupt_at_primary_stepdown.js index 70d0edd6258..7158d12c719 100644 --- a/jstests/sharding/movechunk_interrupt_at_primary_stepdown.js +++ b/jstests/sharding/movechunk_interrupt_at_primary_stepdown.js @@ -37,10 +37,7 @@ load('./jstests/libs/chunk_manipulation_util.js'); // Stepdown the primary in order to force the balancer to stop. Use a timeout of 5 seconds for // both step down operations, because mongos will retry to find the CSRS primary for up to 20 // seconds and we have two successive ones. - assert.throws(function() { - assert.commandWorked( - st.configRS.getPrimary().adminCommand({replSetStepDown: 5, force: true})); - }); + assert.commandWorked(st.configRS.getPrimary().adminCommand({replSetStepDown: 5, force: true})); // Ensure a new primary is found promptly st.configRS.getPrimary(30000); @@ -56,10 +53,7 @@ load('./jstests/libs/chunk_manipulation_util.js'); // At this point, the balancer is in recovery mode. Ensure that stepdown can be done again and // the recovery mode interrupted. - assert.throws(function() { - assert.commandWorked( - st.configRS.getPrimary().adminCommand({replSetStepDown: 5, force: true})); - }); + assert.commandWorked(st.configRS.getPrimary().adminCommand({replSetStepDown: 5, force: true})); // Ensure a new primary is found promptly st.configRS.getPrimary(30000); diff --git a/jstests/sharding/rs_stepdown_and_pooling.js b/jstests/sharding/rs_stepdown_and_pooling.js index 0e34f19636e..e298c61ff17 100644 --- a/jstests/sharding/rs_stepdown_and_pooling.js +++ b/jstests/sharding/rs_stepdown_and_pooling.js @@ -52,17 +52,8 @@ jsTest.log("Stepdown primary and then step back up..."); var stepDown = function(node, timeSecs) { - var result = null; - try { - result = node.getDB("admin").runCommand({replSetStepDown: timeSecs, force: true}); - // Should not get here - } catch (e) { - printjson(e); - } - - if (result != null) - printjson(result); - assert.eq(null, result); + assert.commandWorked( + node.getDB("admin").runCommand({replSetStepDown: timeSecs, force: true})); }; stepDown(primary, 0); @@ -108,4 +99,4 @@ jsTest.log("DONE!"); st.stop(); -}());
\ No newline at end of file +}()); diff --git a/jstests/sharding/txn_two_phase_commit_failover.js b/jstests/sharding/txn_two_phase_commit_failover.js index 85b6880fc97..47e637b0240 100644 --- a/jstests/sharding/txn_two_phase_commit_failover.js +++ b/jstests/sharding/txn_two_phase_commit_failover.js @@ -174,12 +174,8 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; failpointData.numTimesShouldBeHit); // Induce the coordinator primary to step down. - const stepDownResult = assert.throws(function() { - coordPrimary.adminCommand({replSetStepDown: stepDownSecs, force: true}); - }); - assert(isNetworkError(stepDownResult), - 'Expected exception from stepping down coordinator primary ' + - coordPrimary.host + ': ' + tojson(stepDownResult)); + assert.commandWorked( + coordPrimary.adminCommand({replSetStepDown: stepDownSecs, force: true})); assert.commandWorked(coordPrimary.adminCommand({ configureFailPoint: failpointData.failpoint, mode: "off", diff --git a/jstests/sharding/txn_two_phase_commit_wait_for_majority_commit_after_stepup.js b/jstests/sharding/txn_two_phase_commit_wait_for_majority_commit_after_stepup.js index b15a7f9f8f9..9d20c7cd8dd 100644 --- a/jstests/sharding/txn_two_phase_commit_wait_for_majority_commit_after_stepup.js +++ b/jstests/sharding/txn_two_phase_commit_wait_for_majority_commit_after_stepup.js @@ -104,14 +104,11 @@ stopServerReplication([coordPrimary, coordSecondary]); // Induce the coordinator primary to step down. - const stepDownResult = assert.throws(function() { - // The amount of time the node has to wait before becoming primary again. - const stepDownSecs = 1; - coordPrimary.adminCommand({replSetStepDown: stepDownSecs, force: true}); - }); - assert(isNetworkError(stepDownResult), - 'Expected exception from stepping down coordinator primary ' + coordPrimary.host + ': ' + - tojson(stepDownResult)); + + // The amount of time the node has to wait before becoming primary again. + const stepDownSecs = 1; + assert.commandWorked(coordPrimary.adminCommand({replSetStepDown: stepDownSecs, force: true})); + assert.commandWorked(coordPrimary.adminCommand({ configureFailPoint: "hangBeforeWritingDecision", mode: "off", diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 3becf53450a..0c717af02bf 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -98,8 +98,6 @@ MONGO_FAIL_POINT_DEFINE(stepdownHangBeforePerformingPostMemberStateUpdateActions MONGO_FAIL_POINT_DEFINE(transitionToPrimaryHangBeforeTakingGlobalExclusiveLock); MONGO_FAIL_POINT_DEFINE(holdStableTimestampAtSpecificTimestamp); -MONGO_EXPORT_SERVER_PARAMETER(closeConnectionsOnStepdown, bool, true); - using CallbackArgs = executor::TaskExecutor::CallbackArgs; using CallbackFn = executor::TaskExecutor::CallbackFn; using CallbackHandle = executor::TaskExecutor::CallbackHandle; @@ -1770,7 +1768,7 @@ void ReplicationCoordinatorImpl::_killUserOperationsOnStepDown( if (toKill && toKill->getOpID() != stepDownOpCtx->getOpID()) { const GlobalLockAcquisitionTracker& globalLockTracker = GlobalLockAcquisitionTracker::get(toKill); - if (closeConnectionsOnStepdown.load() || globalLockTracker.getGlobalWriteLocked() || + if (globalLockTracker.getGlobalWriteLocked() || globalLockTracker.getGlobalSharedLockTaken()) { serviceCtx->killOperation(lk, toKill, ErrorCodes::InterruptedDueToStepDown); } @@ -2810,9 +2808,6 @@ void ReplicationCoordinatorImpl::_performPostMemberStateUpdateAction( _externalState->closeConnections(); /* FALLTHROUGH */ case kActionSteppedDown: - if (closeConnectionsOnStepdown.load()) { - _externalState->closeConnections(); - } _externalState->shardingOnStepDownHook(); _externalState->stopNoopWriter(); break; |