SERVER-38511 Avoid killing read operations on stepdown, gated by new server parameter “closeConnectionsOnStepdown”.

2 files changed, 135 insertions, 1 deletions

diff --git a/jstests/replsets/no_disconnect_on_stepdown.js b/jstests/replsets/no_disconnect_on_stepdown.js
index d21f9785d6a..284a8d39483 100644
--- a/jstests/replsets/no_disconnect_on_stepdown.js
+++ b/jstests/replsets/no_disconnect_on_stepdown.js
@@ -43,7 +43,7 @@
         assert.commandWorked(
             primaryAdmin.adminCommand({configureFailPoint: failpoint, mode: "alwaysOn"}));
 
-        errorCode = errorCode || ErrorCodes.InterruptedDueToStepDown;
+        errorCode = errorCode || ErrorCodes.PrimarySteppedDown;
         const writeCommand = `db.getMongo().forceWriteMode("commands");
                               assert.commandFailedWithCode(${operation}, ${errorCode});
                               assert.commandWorked(db.adminCommand({ping:1}));`;
@@ -85,6 +85,7 @@
         failpoint: "hangBeforeLinearizableReadConcern",
         operation: "db.runCommand({find: '" + collname +
             "', filter: {'_id': 'findme'}, readConcern: {level: 'linearizable'}})",
+        errorCode: ErrorCodes.NotMaster
     });
     rst.stopSet();
 })();
diff --git a/jstests/replsets/read_operations_during_step_down.js b/jstests/replsets/read_operations_during_step_down.js
new file mode 100644
index 00000000000..4859df5f064
--- /dev/null
+++ b/jstests/replsets/read_operations_during_step_down.js
@@ -0,0 +1,133 @@
+/*
+ * Test that the read operations are not killed and their connections are also not
+ * closed during step down.
+ */
+load('jstests/replsets/rslib.js');
+load('jstests/libs/parallelTester.js');
+
+(function() {
+
+    "use strict";
+
+    const testName = "readOpsDuringStepDown";
+    const dbName = "test";
+    const collName = "coll";
+
+    var rst = new ReplSetTest({
+        name: testName,
+        nodes: [{setParameter: {closeConnectionsOnStepdown: false}}, {rsConfig: {priority: 0}}]
+    });
+    rst.startSet();
+    rst.initiate();
+
+    const primary = rst.getPrimary();
+    const primaryDB = primary.getDB(dbName);
+    const primaryAdminDB = primary.getDB("admin");
+    const primaryColl = primaryDB[collName];
+
+    TestData.dbName = dbName;
+    TestData.collName = collName;
+
+    var waitForOpToReachFailPoint = (failPointMsg) => {
+        // Wait until we know the failpoint "failPointMsg" has been
+        // reached for the namespace specified in primaryColl.
+        assert.soon(
+            () => {
+                const res =
+                    primaryAdminDB
+                        .aggregate([
+                            {$currentOp: {}},
+                            {
+                              $match: {
+                                  $and: [{ns: primaryColl.getFullName()}, {"msg": failPointMsg}]
+                              }
+                            }
+                        ])
+                        .toArray();
+                if (res.length === 1) {
+                    return true;
+                }
+                return false;
+            },
+            () => {
+                return "Failed to find operation in $currentOp output: " +
+                    tojson(primaryAdminDB
+                               .aggregate(
+                                   [{$currentOp: {}}, {$match: {ns: primaryColl.getFullName()}}])
+                               .toArray());
+            });
+    };
+
+    jsTestLog("1. Do a document write");
+    assert.writeOK(
+        primaryColl.insert({_id: 0}, {"writeConcern": {"w": "majority"}}));
+    rst.awaitReplication();
+
+    // Open a cursor on primary.
+    const cursorIdToBeReadAfterStepDown =
+        assert.commandWorked(primaryDB.runCommand({"find": collName, batchSize: 0})).cursor.id;
+
+    jsTestLog("2. Start blocking getMore cmd before step down");
+    const joinGetMoreThread = startParallelShell(() => {
+        // Open another cursor on primary before step down.
+        primaryDB = db.getSiblingDB(TestData.dbName);
+        const cursorIdToBeReadDuringStepDown =
+            assert.commandWorked(primaryDB.runCommand({"find": TestData.collName, batchSize: 0}))
+                .cursor.id;
+
+        // Enable the fail point for get more cmd.
+        assert.commandWorked(db.adminCommand(
+            {configureFailPoint: "waitAfterPinningCursorBeforeGetMoreBatch", mode: "alwaysOn"}));
+
+        getMoreRes = assert.commandWorked(primaryDB.runCommand(
+            {"getMore": cursorIdToBeReadDuringStepDown, collection: TestData.collName}));
+        assert.docEq([{_id: 0}], getMoreRes.cursor.nextBatch);
+    }, primary.port);
+
+    // Wait for getmore cmd to reach the fail point.
+    waitForOpToReachFailPoint("waitAfterPinningCursorBeforeGetMoreBatch");
+
+    jsTestLog("2. Start blocking find cmd before step down");
+    const joinFindThread = startParallelShell(() => {
+        // Enable the fail point for find cmd.
+        assert.commandWorked(
+            db.adminCommand({configureFailPoint: "waitInFindBeforeMakingBatch", mode: "alwaysOn"}));
+
+        var findRes = assert.commandWorked(
+            db.getSiblingDB(TestData.dbName).runCommand({"find": TestData.collName}));
+        assert.docEq([{_id: 0}], findRes.cursor.firstBatch);
+
+    }, primary.port);
+
+    // Wait for find cmd to reach the fail point.
+    waitForOpToReachFailPoint("waitInFindBeforeMakingBatch");
+
+    jsTestLog("3. Make primary step down");
+    const joinStepDownThread = startParallelShell(() => {
+        assert.commandWorked(db.adminCommand({"replSetStepDown": 100, "force": true}));
+    }, primary.port);
+
+    // Wait untill the step down has started to kill user operations.
+    checkLog.contains(primary, "Starting to kill user operations");
+
+    jsTestLog("4. Disable fail points");
+    assert.commandWorked(primaryAdminDB.runCommand(
+        {configureFailPoint: "waitInFindBeforeMakingBatch", mode: "off"}));
+    assert.commandWorked(primaryAdminDB.runCommand(
+        {configureFailPoint: "waitAfterPinningCursorBeforeGetMoreBatch", mode: "off"}));
+
+    // Wait for threads to join.
+    joinGetMoreThread();
+    joinFindThread();
+    joinStepDownThread();
+
+    // Wait untill the old primary transitioned to SECONDARY state.
+    waitForState(primary, ReplSetTest.State.SECONDARY);
+
+    jsTestLog("5. Start get more cmd after step down");
+    var getMoreRes = assert.commandWorked(
+        primaryDB.runCommand({"getMore": cursorIdToBeReadAfterStepDown, collection: collName}));
+    assert.docEq([{_id: 0}], getMoreRes.cursor.nextBatch);
+
+    rst.stopSet();
+})();