SERVER-29500 Call for catchup takeover election when catchup takeover timeout fires

2 files changed, 140 insertions, 0 deletions

diff --git a/jstests/replsets/catchup_takeover_one_high_priority.js b/jstests/replsets/catchup_takeover_one_high_priority.js
new file mode 100644
index 00000000000..ed6d7faee50
--- /dev/null
+++ b/jstests/replsets/catchup_takeover_one_high_priority.js
@@ -0,0 +1,84 @@
+// Test to ensure that catchup takeover runs even if it isn't the highest
+// priority node and that once the high priority node is caught up,
+// it becomes primary again.
+
+// 3-node replica set with one high priority node.
+// Start replica set. Make node 0 primary and stop the replication
+// for the high priority node as well as isolate it. Have the
+// primary write something so node 2 is more than 2 seconds behind.
+// Write something else to ensure the third node is also lagged.
+// Reconnect the high priority node to the other nodes and make
+// the lagged node (node 1) the next primary.
+// Confirm that the most up-to-date node becomes primary.
+// Let the highest priority node catchup and then confirm
+// that it becomes primary.
+
+(function() {
+    'use strict';
+
+    load('jstests/replsets/rslib.js');
+
+    var name = 'catchup_takeover_one_high_priority';
+    var replSet =
+        new ReplSetTest({name: name, nodes: [{}, {}, {rsConfig: {priority: 2}}], useBridge: true});
+    var nodes = replSet.startSet();
+    replSet.initiate();
+
+    // Wait until node 2 becomes primary.
+    replSet.waitForState(2, ReplSetTest.State.PRIMARY, replSet.kDefaultTimeoutMS);
+    jsTestLog('node 2 is now primary');
+
+    replSet.awaitReplication();
+
+    // Stop replication and disconnect node 2 so that it cannot do a priority takeover.
+    stopServerReplication(nodes[2]);
+    nodes[2].disconnect(nodes[1]);
+    nodes[2].disconnect(nodes[0]);
+
+    // Ensure that node 0 becomes primary.
+    assert.commandWorked(nodes[0].adminCommand({replSetStepUp: 1}));
+    replSet.awaitNodesAgreeOnPrimary(replSet.kDefaultTimeoutMS, nodes.slice(0, 2));
+    assert.eq(ReplSetTest.State.PRIMARY,
+              assert.commandWorked(nodes[0].adminCommand('replSetGetStatus')).myState,
+              nodes[0].host + " was not primary after step-up");
+    jsTestLog('node 0 is now primary');
+
+    // Sleep for a few seconds to ensure that node 2's optime is more than 2 seconds behind.
+    // This will ensure it can't do a priority takeover until it catches up.
+    sleep(3000);
+
+    var primary = replSet.getPrimary();
+    var writeConcern = {writeConcern: {w: 2, wtimeout: replSet.kDefaultTimeoutMS}};
+    assert.writeOK(primary.getDB(name).bar.insert({y: 100}, writeConcern));
+
+    // Write something so that node 0 is ahead of node 1.
+    stopServerReplication(nodes[1]);
+    writeConcern = {writeConcern: {w: 1, wtimeout: replSet.kDefaultTimeoutMS}};
+    assert.writeOK(primary.getDB(name).bar.insert({x: 100}, writeConcern));
+
+    nodes[2].reconnect(nodes[0]);
+    nodes[2].reconnect(nodes[1]);
+
+    // Step up a lagged node.
+    assert.commandWorked(nodes[1].adminCommand({replSetStepUp: 1}));
+    replSet.awaitNodesAgreeOnPrimary(replSet.kDefaultTimeoutMS, nodes);
+    assert.eq(ReplSetTest.State.PRIMARY,
+              assert.commandWorked(nodes[1].adminCommand('replSetGetStatus')).myState,
+              nodes[1].host + " was not primary after step-up");
+    jsTestLog('node 1 is now primary, but cannot accept writes');
+
+    // Confirm that the most up-to-date node becomes primary
+    // after the default catchup delay.
+    replSet.waitForState(0, ReplSetTest.State.PRIMARY, 60 * 1000);
+    jsTestLog('node 0 performed catchup takeover and is now primary');
+
+    // Let the nodes catchup.
+    restartServerReplication(nodes[1]);
+    restartServerReplication(nodes[2]);
+
+    // Confirm that the highest priority node becomes primary
+    // after catching up.
+    replSet.waitForState(2, ReplSetTest.State.PRIMARY, 30 * 1000);
+    jsTestLog('node 2 performed priority takeover and is now primary');
+
+})();
\ No newline at end of file
diff --git a/jstests/replsets/catchup_takeover_two_nodes_ahead.js b/jstests/replsets/catchup_takeover_two_nodes_ahead.js
new file mode 100644
index 00000000000..4be8fee9007
--- /dev/null
+++ b/jstests/replsets/catchup_takeover_two_nodes_ahead.js
@@ -0,0 +1,56 @@
+// Test to ensure that a catchup takeover happens when the primary is lagged.
+// Make sure that when two nodes are more caught up than the primary,
+// the most up-to-date node becomes the primary.
+
+// 5-node replica set
+// Start replica set. Ensure that node 0 becomes primary.
+// Stop the replication for some nodes and have the primary write something.
+// Stop replication for an up-to-date node and have the primary write something.
+// Now the primary is most-up-to-date and another node is more up-to-date than others.
+// Make a lagged node the next primary.
+// Confirm that the most up-to-date node becomes primary.
+
+(function() {
+    'use strict';
+
+    load('jstests/replsets/rslib.js');
+
+    var name = 'catchup_takeover_two_nodes_ahead';
+    var replSet = new ReplSetTest({name: name, nodes: 5});
+    var nodes = replSet.startSet();
+    replSet.initiate();
+
+    // Wait until all nodes get the "no-op" of "new primary" after initial sync.
+    waitUntilAllNodesCaughtUp(nodes);
+
+    // Write something so that nodes 0 and 1 are ahead.
+    stopServerReplication(nodes.slice(2, 5));
+    var primary = replSet.getPrimary();
+    var writeConcern = {writeConcern: {w: 2, wtimeout: replSet.kDefaultTimeoutMS}};
+    assert.writeOK(primary.getDB(name).bar.insert({x: 100}, writeConcern));
+
+    // Write something so that node 0 is ahead of node 1.
+    stopServerReplication(nodes[1]);
+    writeConcern = {writeConcern: {w: 1, wtimeout: replSet.kDefaultTimeoutMS}};
+    assert.writeOK(primary.getDB(name).bar.insert({y: 100}, writeConcern));
+
+    // Step up one of the lagged nodes.
+    assert.commandWorked(nodes[2].adminCommand({replSetStepUp: 1}));
+    replSet.awaitNodesAgreeOnPrimary(replSet.kDefaultTimeoutMS, nodes);
+    assert.eq(ReplSetTest.State.PRIMARY,
+              assert.commandWorked(nodes[2].adminCommand('replSetGetStatus')).myState,
+              nodes[2].host + " was not primary after step-up");
+    jsTestLog('node 2 is now primary, but cannot accept writes');
+
+    // Make sure that node 2 cannot write anything. Because it is lagged and replication
+    // has been stopped, it shouldn't be able to become master.
+    assert.writeErrorWithCode(nodes[2].getDB(name).bar.insert({z: 100}, writeConcern),
+                              ErrorCodes.NotMaster);
+
+    // Confirm that the most up-to-date node becomes primary
+    // after the default catchup delay.
+    replSet.waitForState(0, ReplSetTest.State.PRIMARY, 60 * 1000);
+
+    // Let the nodes catchup
+    restartServerReplication(nodes.slice(1, 5));
+})();
\ No newline at end of file