SERVER-41510 Track the number of failed stepDowns in serverStatus

1 files changed, 89 insertions, 5 deletions

diff --git a/jstests/replsets/stepdown.js b/jstests/replsets/stepdown.js
index 7ff4142314f..934d9b9a2f4 100644
--- a/jstests/replsets/stepdown.js
+++ b/jstests/replsets/stepdown.js
@@ -2,6 +2,9 @@
  * Check that on a loss of primary, another node doesn't assume primary if it is stale. We force a
  * stepDown to test this.
  *
+ * This test also checks that the serverStatus command metrics replSetStepDown and
+ * replSetStepDownWithForce are incremented correctly.
+ *
  * This test requires the fsync command to force a secondary to be stale.
  * @tags: [requires_fsync]
  */
@@ -61,7 +64,7 @@ try {
         assert.writeOK(master.getDB("foo").bar.insert({x: i}));
     }
 
-    const intitialServerStatus = assert.commandWorked(master.adminCommand({serverStatus: 1}));
+    let intitialServerStatus = assert.commandWorked(master.adminCommand({serverStatus: 1}));
 
     jsTestLog('Do stepdown of primary ' + master + ' that should not work');
 
@@ -70,39 +73,120 @@ try {
         'Step down ' + master + ' expected error: ' +
         tojson(assert.commandFailed(master.getDB("admin").runCommand({replSetStepDown: 10}))));
 
-    // Check that the 'total' field of 'replSetStepDown' has been incremented in serverStatus and
-    // that it has not been incremented for 'replSetStepDownWithForce'.
+    // Check that the 'total' and 'failed' fields of 'replSetStepDown' have been incremented in
+    // serverStatus and that they have not been incremented for 'replSetStepDownWithForce'.
     let newServerStatus = assert.commandWorked(master.adminCommand({serverStatus: 1}));
     verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDown,
                              newServerStatus.metrics.commands.replSetStepDown,
                              "total",
                              1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDown,
+                             newServerStatus.metrics.commands.replSetStepDown,
+                             "failed",
+                             1);
     verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDownWithForce,
                              newServerStatus.metrics.commands.replSetStepDownWithForce,
                              "total",
                              0);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDownWithForce,
+                             newServerStatus.metrics.commands.replSetStepDownWithForce,
+                             "failed",
+                             0);
+
+    // This section checks that the metrics are incremented accurately when the command fails due to
+    // an error occurring before stepDown is called in the replication coordinator, such as due to
+    // bad values or type mismatches in the arguments, or checkReplEnabledForCommand returning a bad
+    // status. The stepdown period being negative is one example of such an error, but success in
+    // this case gives us confidence that the behavior in the other cases is the same.
+
+    // Stepdown should fail because the stepdown period is negative
+    jsTestLog('Do stepdown of primary ' + master + ' that should not work');
+    assert.commandFailedWithCode(
+        master.getDB("admin").runCommand({replSetStepDown: -1, force: true}), ErrorCodes.BadValue);
+
+    // Check that the 'total' and 'failed' fields of 'replSetStepDown' and
+    // 'replSetStepDownWithForce' have been incremented in serverStatus.
+    intitialServerStatus = newServerStatus;
+    newServerStatus = assert.commandWorked(master.adminCommand({serverStatus: 1}));
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDown,
+                             newServerStatus.metrics.commands.replSetStepDown,
+                             "total",
+                             1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDown,
+                             newServerStatus.metrics.commands.replSetStepDown,
+                             "failed",
+                             1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDownWithForce,
+                             newServerStatus.metrics.commands.replSetStepDownWithForce,
+                             "total",
+                             1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDownWithForce,
+                             newServerStatus.metrics.commands.replSetStepDownWithForce,
+                             "failed",
+                             1);
 
     jsTestLog('Do stepdown of primary ' + master + ' that should work');
     assert.commandWorked(
         master.adminCommand({replSetStepDown: ReplSetTest.kDefaultTimeoutMS, force: true}));
 
     // Check that the 'total' fields of 'replSetStepDown' and 'replSetStepDownWithForce' have been
-    // incremented in serverStatus.
+    // incremented in serverStatus and that their 'failed' fields have not been incremented.
+    intitialServerStatus = newServerStatus;
     newServerStatus = assert.commandWorked(master.adminCommand({serverStatus: 1}));
     verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDown,
                              newServerStatus.metrics.commands.replSetStepDown,
                              "total",
-                             2);
+                             1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDown,
+                             newServerStatus.metrics.commands.replSetStepDown,
+                             "failed",
+                             0);
     verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDownWithForce,
                              newServerStatus.metrics.commands.replSetStepDownWithForce,
                              "total",
                              1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDownWithForce,
+                             newServerStatus.metrics.commands.replSetStepDownWithForce,
+                             "failed",
+                             0);
 
     jsTestLog('Checking isMaster on ' + master);
     var r2 = assert.commandWorked(master.getDB("admin").runCommand({ismaster: 1}));
     jsTestLog('Result from running isMaster on ' + master + ': ' + tojson(r2));
     assert.eq(r2.ismaster, false);
     assert.eq(r2.secondary, true);
+
+    // This section checks that the metrics are incremented accurately when the command fails due to
+    // an error while stepping down. This is one reason the replSetStepDown command could fail once
+    // we call stepDown in the replication coordinator, but success in this case gives us confidence
+    // that the behavior in the other cases is the same.
+
+    // Stepdown should fail because the node is no longer primary
+    jsTestLog('Do stepdown of primary ' + master + ' that should not work');
+    assert.commandFailedWithCode(master.getDB("admin").runCommand(
+                                     {replSetStepDown: ReplSetTest.kDefaultTimeoutMS, force: true}),
+                                 ErrorCodes.NotMaster);
+
+    // Check that the 'total' and 'failed' fields of 'replSetStepDown' and
+    // 'replSetStepDownWithForce' have been incremented in serverStatus.
+    intitialServerStatus = newServerStatus;
+    newServerStatus = assert.commandWorked(master.adminCommand({serverStatus: 1}));
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDown,
+                             newServerStatus.metrics.commands.replSetStepDown,
+                             "total",
+                             1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDown,
+                             newServerStatus.metrics.commands.replSetStepDown,
+                             "failed",
+                             1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDownWithForce,
+                             newServerStatus.metrics.commands.replSetStepDownWithForce,
+                             "total",
+                             1);
+    verifyServerStatusChange(intitialServerStatus.metrics.commands.replSetStepDownWithForce,
+                             newServerStatus.metrics.commands.replSetStepDownWithForce,
+                             "failed",
+                             1);
 } catch (e) {
     throw e;
 } finally {