SERVER-38696 Add additional metrics and logging for new step down sequence.

1 files changed, 30 insertions, 7 deletions

diff --git a/jstests/replsets/read_operations_during_step_down.js b/jstests/replsets/read_operations_during_step_down.js
index b03adf7a731..4f9507ff902 100644
--- a/jstests/replsets/read_operations_during_step_down.js
+++ b/jstests/replsets/read_operations_during_step_down.js
@@ -2,7 +2,7 @@
  * Test that the read operations are not killed and their connections are also not
  * closed during step down.
  */
-load('jstests/replsets/rslib.js');
+load("jstests/libs/check_log.js");
 load('jstests/libs/parallelTester.js');
 load("jstests/libs/curop_helpers.js");  // for waitForCurOpByFailPoint().
 
@@ -76,27 +76,50 @@ load("jstests/libs/curop_helpers.js");  // for waitForCurOpByFailPoint().
         assert.commandWorked(db.adminCommand({"replSetStepDown": 100, "force": true}));
     }, primary.port);
 
-    // Wait untill the step down has started to kill user operations.
+    // Wait until the step down has started to kill user operations.
     checkLog.contains(primary, "Starting to kill user operations");
 
+    // Enable "waitAfterReadCommandFinishesExecution" fail point to make sure the find and get more
+    // commands on database 'test' does not complete before step down.
+    assert.commandWorked(primaryAdmin.runCommand({
+        configureFailPoint: "waitAfterReadCommandFinishesExecution",
+        data: {db: dbName},
+        mode: "alwaysOn"
+    }));
+
     jsTestLog("4. Disable fail points");
     assert.commandWorked(
         primaryAdmin.runCommand({configureFailPoint: "waitInFindBeforeMakingBatch", mode: "off"}));
     assert.commandWorked(primaryAdmin.runCommand(
         {configureFailPoint: "waitAfterPinningCursorBeforeGetMoreBatch", mode: "off"}));
 
-    // Wait for threads to join.
-    joinGetMoreThread();
-    joinFindThread();
+    // Wait until the primary transitioned to SECONDARY state.
     joinStepDownThread();
+    rst.waitForState(primary, ReplSetTest.State.SECONDARY);
+
+    // We don't want to check if we have reached "waitAfterReadCommandFinishesExecution" fail point
+    // because we already know that the primary has stepped down successfully. This implies that
+    // the find and get more commands are still running even after the node stepped down.
+    assert.commandWorked(primaryAdmin.runCommand(
+        {configureFailPoint: "waitAfterReadCommandFinishesExecution", mode: "off"}));
 
-    // Wait untill the old primary transitioned to SECONDARY state.
-    waitForState(primary, ReplSetTest.State.SECONDARY);
+    // Wait for find & getmore thread to join.
+    joinGetMoreThread();
+    joinFindThread();
 
     jsTestLog("5. Start get more cmd after step down");
     var getMoreRes = assert.commandWorked(
         primaryDB.runCommand({"getMore": cursorIdToBeReadAfterStepDown, collection: collName}));
     assert.docEq([{_id: 0}], getMoreRes.cursor.nextBatch);
 
+    // Validate that no operations got killed on step down and no network disconnection happened due
+    // to failed unacknowledged operations.
+    let replMetrics =
+        assert.commandWorked(primaryAdmin.adminCommand({serverStatus: 1})).metrics.repl;
+    assert.eq(replMetrics.stepDown.userOperationsKilled, 0);
+    // Should account for find and getmore commands issued before step down.
+    assert.gte(replMetrics.stepDown.userOperationsRunning, 2);
+    assert.eq(replMetrics.network.notMasterUnacknowledgedWrites, 0);
+
     rst.stopSet();
 })();