SERVER-27024 Added test for stepdown needs majority behavior

(cherry picked from commit 48741605f13b278f3c51bfc11660bd0a2dbb498b)

3 files changed, 166 insertions, 19 deletions

diff --git a/jstests/libs/write_concern_util.js b/jstests/libs/write_concern_util.js
new file mode 100644
index 00000000000..476a9c3e818
--- /dev/null
+++ b/jstests/libs/write_concern_util.js
@@ -0,0 +1,47 @@
+/**
+ * Utilities for testing writeConcern.
+ */
+// Stops replication at a server.
+function stopServerReplication(conn) {
+    var errMsg = 'Failed to enable rsSyncApplyStop failpoint.';
+    assert.commandWorked(
+        conn.getDB('admin').runCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'alwaysOn'}),
+        errMsg);
+}
+
+// Stops replication at all replicaset secondaries.
+function stopReplicationOnSecondaries(rs) {
+    var secondaries = rs.getSecondaries();
+    secondaries.forEach(stopServerReplication);
+}
+
+// Stops replication at all shard secondaries.
+function stopReplicationOnSecondariesOfAllShards(st) {
+    st._rsObjects.forEach(stopReplicationOnSecondaries);
+}
+
+// Restarts replication at a server.
+function restartServerReplication(conn) {
+    var errMsg = 'Failed to disable rsSyncApplyStop failpoint.';
+    assert.commandWorked(
+        conn.getDB('admin').runCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'off'}),
+        errMsg);
+}
+
+// Restarts replication at all nodes in a replicaset.
+function restartReplSetReplication(rs) {
+    rs.nodes.forEach(restartServerReplication);
+}
+
+// Restarts replication at all nodes in a sharded cluster.
+function restartReplicationOnAllShards(st) {
+    st._rsObjects.forEach(restartReplSetReplication);
+    restartReplSetReplication(st.configRS);
+}
+
+// Asserts that a writeConcernError was received.
+function assertWriteConcernError(res) {
+    assert(res.writeConcernError, "No writeConcernError received, got: " + tojson(res));
+    assert(res.writeConcernError.code);
+    assert(res.writeConcernError.errmsg);
+}
diff --git a/jstests/replsets/stepdown_needs_electable_secondary.js b/jstests/replsets/stepdown_needs_electable_secondary.js
index 88490ee803f..2e6a373d788 100644
--- a/jstests/replsets/stepdown_needs_electable_secondary.js
+++ b/jstests/replsets/stepdown_needs_electable_secondary.js
@@ -20,7 +20,12 @@
  *
  */
 (function() {
+    load("jstests/libs/write_concern_util.js");  // for stopReplicationOnSecondaries,
+                                                 // restartServerReplication,
+                                                 // restartReplSetReplication
+
     'use strict';
+
     var name = 'stepdown_needs_electable_secondary';
 
     var replTest = new ReplSetTest({name: name, nodes: 5});
@@ -38,20 +43,6 @@
         ]
     });
 
-    /* Disable all incoming writes to a node (secondary) */
-    function disableReplicationToNode(node) {
-        assert.commandWorked(node.getDB('admin').runCommand(
-                                 {configureFailPoint: 'rsSyncApplyStop', mode: 'alwaysOn'}),
-                             'Failed to enable rsSyncApplyStop failpoint.');
-    }
-
-    /* Re-enable all incoming writes to a node (secondary) */
-    function enableReplicationToNode(node) {
-        assert.commandWorked(
-            node.getDB('admin').runCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'off'}),
-            'Failed to disable rsSyncApplyStop failpoint.');
-    }
-
     function assertStepDownFailsWithExceededTimeLimit(node) {
         assert.commandFailedWithCode(
             node.getDB("admin").runCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}),
@@ -68,7 +59,7 @@
     var primary = replTest.getPrimary();
 
     jsTestLog("Blocking writes to all secondaries.");
-    replTest.liveNodes.slaves.forEach(disableReplicationToNode);
+    stopReplicationOnSecondaries(replTest);
 
     jsTestLog("Doing a write to primary.");
     var testDB = replTest.getPrimary().getDB('testdb');
@@ -95,7 +86,7 @@
     // stepDown should fail due to no caught up majority.
     jsTestLog("Re-enabling writes to unelectable secondary: node #" +
               replTest.getNodeId(secondaryB_unelectable) + ", " + secondaryB_unelectable);
-    enableReplicationToNode(secondaryB_unelectable);
+    restartServerReplication(secondaryB_unelectable);
 
     // Wait for this secondary to catch up by issuing a write that must be replicated to 2 nodes
     assert.writeOK(coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 2}}));
@@ -109,7 +100,7 @@
     // stepDown should fail due to caught up majority without electable node.
     jsTestLog("Re-enabling writes to unelectable secondary: node #" +
               replTest.getNodeId(secondaryC_unelectable) + ", " + secondaryC_unelectable);
-    enableReplicationToNode(secondaryC_unelectable);
+    restartServerReplication(secondaryC_unelectable);
 
     // Wait for this secondary to catch up by issuing a write that must be replicated to 3 nodes
     assert.writeOK(coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 3}}));
@@ -124,7 +115,7 @@
     // stepDown should succeed due to caught up majority containing an electable node.
     jsTestLog("Re-enabling writes to electable secondary: node #" +
               replTest.getNodeId(secondaryA_electable) + ", " + secondaryA_electable);
-    enableReplicationToNode(secondaryA_electable);
+    restartServerReplication(secondaryA_electable);
 
     // Wait for this secondary to catch up by issuing a write that must be replicated to 4 nodes
     assert.writeOK(coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 4}}));
@@ -140,7 +131,7 @@
     replTest.waitForState(primary, ReplSetTest.State.SECONDARY);
 
     // Disable all fail points for clean shutdown
-    replTest.liveNodes.slaves.forEach(enableReplicationToNode);
+    restartReplSetReplication(replTest);
     replTest.stopSet();
 
 }());
diff --git a/jstests/replsets/stepdown_needs_majority.js b/jstests/replsets/stepdown_needs_majority.js
new file mode 100644
index 00000000000..9b52ba58eb0
--- /dev/null
+++ b/jstests/replsets/stepdown_needs_majority.js
@@ -0,0 +1,109 @@
+/**
+ * Test to ensure that 'replSetStepDown' called on a primary will fail if an electable node is
+ * caught up to it but less than a majority of nodes are caught up to it. Additionally tests that
+ * step down will then succeed once a majority has caught up. Tests this with a 5 node replica set.
+ *
+ * 1.  Initiate a 5-node replica set
+ * 2.  Disable replication to all secondaries
+ * 3.  Execute a write on primary with writeConcern:1
+ * 4.  Try to step down primary and expect to fail
+ * 5.  Enable replication to one secondary (Secondary A)
+ * 6.  Await replication to Secondary A by executing primary write with writeConcern:2
+ * 7.  Try to step down primary and expect failure
+ * 8.  Enable replication to a different secondary (Secondary B)
+ * 9.  Await replication to Secondary B by executing primary write with writeConcern:3
+ * 10. Try to step down primary and expect success
+ *
+ */
+(function() {
+    load("jstests/libs/write_concern_util.js");  // for stopReplicationOnSecondaries, //
+                                                 // restartServerReplication,
+                                                 // restartReplSetReplication
+
+    'use strict';
+
+    function assertStepDownFailsWithExceededTimeLimit(node) {
+        assert.commandFailedWithCode(
+            node.getDB("admin").runCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}),
+            ErrorCodes.ExceededTimeLimit,
+            "step down did not fail with 'ExceededTimeLimit'");
+    }
+
+    function assertStepDownSucceeds(node) {
+        assert.throws(function() {
+            node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60});
+        });
+    }
+
+    function nodeIdStr(repltest, node) {
+        return "node #" + repltest.getNodeId(node) + ", " + node.host;
+    }
+
+    //
+    // Test setup
+    //
+    var name = 'stepdown_needs_majority';
+    var replTest = new ReplSetTest({name: name, nodes: 5});
+
+    replTest.startSet();
+    replTest.initiate();
+
+    var primary = replTest.getPrimary();
+    var testDB = primary.getDB('testdb');
+    var coll = testDB[name];
+    var dummy_doc = {
+        "dummy_key": "dummy_val"
+    };
+
+    //
+    // Block writes to all secondaries
+    //
+    jsTestLog("Blocking writes to all secondaries.");
+    stopReplicationOnSecondaries(replTest);
+
+    //
+    // Write to the primary and attempt stepdown
+    //
+    jsTestLog("Issuing a write to the primary(" + primary.host + ") with write_concern:1");
+    assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 1}}));
+
+    jsTestLog("Trying to step down primary with only 1 node out of 5 caught up.");
+    assertStepDownFailsWithExceededTimeLimit(primary);
+
+    //
+    // Re-enable writes to Secondary A and attempt stepdown
+    //
+    var secondaryA = replTest.getSecondaries()[0];
+    jsTestLog("Reenabling writes to one secondary (" + nodeIdStr(replTest, secondaryA) + ")");
+    restartServerReplication(secondaryA);
+
+    jsTestLog("Issuing a write to the primary with write_concern:2");
+    assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 2}}));
+
+    jsTestLog("Trying to step down primary with only 2 nodes out of 5 caught up.");
+    assertStepDownFailsWithExceededTimeLimit(primary);
+
+    //
+    // Re-enable writes to Secondary B and attempt stepdown
+    //
+    var secondaryB = replTest.getSecondaries()[1];
+    jsTestLog("Reenabling writes to another secondary (" + nodeIdStr(replTest, secondaryB) + ")");
+    restartServerReplication(secondaryB);
+
+    jsTestLog("Issuing a write to the primary with write_concern:3");
+    assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 3}}));
+
+    jsTestLog("Trying to step down primary with 3 nodes out of 5 caught up.");
+    assertStepDownSucceeds(primary);
+
+    jsTestLog("Waiting for PRIMARY(" + primary.host + ") to step down & become SECONDARY.");
+    replTest.waitForState(primary, ReplSetTest.State.SECONDARY);
+
+    //
+    // Disable failpoints and stop replica set
+    //
+    jsTestLog("Disabling all fail points to allow for clean shutdown");
+    restartReplSetReplication(replTest);
+    replTest.stopSet();
+
+}());