diff options
author | Samy Lanka <samy.lanka@10gen.com> | 2017-07-05 15:18:06 -0400 |
---|---|---|
committer | Samy Lanka <samy.lanka@10gen.com> | 2017-08-02 15:29:26 -0400 |
commit | 5f1ce8b6765a25d45ba5e35063db417b3069c8d6 (patch) | |
tree | de942dd067b10bb2ae70f3a5b0f5601cd7fa87ec /jstests/replsets | |
parent | 743119c879ab2e5d1d8ca05aadf6fe29b1526a94 (diff) | |
download | mongo-5f1ce8b6765a25d45ba5e35063db417b3069c8d6.tar.gz |
SERVER-29500 Call for catchup takeover election when catchup takeover timeout fires
Diffstat (limited to 'jstests/replsets')
-rw-r--r-- | jstests/replsets/catchup_takeover_one_high_priority.js | 84 | ||||
-rw-r--r-- | jstests/replsets/catchup_takeover_two_nodes_ahead.js | 56 |
2 files changed, 140 insertions, 0 deletions
diff --git a/jstests/replsets/catchup_takeover_one_high_priority.js b/jstests/replsets/catchup_takeover_one_high_priority.js new file mode 100644 index 00000000000..ed6d7faee50 --- /dev/null +++ b/jstests/replsets/catchup_takeover_one_high_priority.js @@ -0,0 +1,84 @@ +// Test to ensure that catchup takeover runs even if it isn't the highest +// priority node and that once the high priority node is caught up, +// it becomes primary again. + +// 3-node replica set with one high priority node. +// Start replica set. Make node 0 primary and stop the replication +// for the high priority node as well as isolate it. Have the +// primary write something so node 2 is more than 2 seconds behind. +// Write something else to ensure the third node is also lagged. +// Reconnect the high priority node to the other nodes and make +// the lagged node (node 1) the next primary. +// Confirm that the most up-to-date node becomes primary. +// Let the highest priority node catchup and then confirm +// that it becomes primary. + +(function() { + 'use strict'; + + load('jstests/replsets/rslib.js'); + + var name = 'catchup_takeover_one_high_priority'; + var replSet = + new ReplSetTest({name: name, nodes: [{}, {}, {rsConfig: {priority: 2}}], useBridge: true}); + var nodes = replSet.startSet(); + replSet.initiate(); + + // Wait until node 2 becomes primary. + replSet.waitForState(2, ReplSetTest.State.PRIMARY, replSet.kDefaultTimeoutMS); + jsTestLog('node 2 is now primary'); + + replSet.awaitReplication(); + + // Stop replication and disconnect node 2 so that it cannot do a priority takeover. + stopServerReplication(nodes[2]); + nodes[2].disconnect(nodes[1]); + nodes[2].disconnect(nodes[0]); + + // Ensure that node 0 becomes primary. + assert.commandWorked(nodes[0].adminCommand({replSetStepUp: 1})); + replSet.awaitNodesAgreeOnPrimary(replSet.kDefaultTimeoutMS, nodes.slice(0, 2)); + assert.eq(ReplSetTest.State.PRIMARY, + assert.commandWorked(nodes[0].adminCommand('replSetGetStatus')).myState, + nodes[0].host + " was not primary after step-up"); + jsTestLog('node 0 is now primary'); + + // Sleep for a few seconds to ensure that node 2's optime is more than 2 seconds behind. + // This will ensure it can't do a priority takeover until it catches up. + sleep(3000); + + var primary = replSet.getPrimary(); + var writeConcern = {writeConcern: {w: 2, wtimeout: replSet.kDefaultTimeoutMS}}; + assert.writeOK(primary.getDB(name).bar.insert({y: 100}, writeConcern)); + + // Write something so that node 0 is ahead of node 1. + stopServerReplication(nodes[1]); + writeConcern = {writeConcern: {w: 1, wtimeout: replSet.kDefaultTimeoutMS}}; + assert.writeOK(primary.getDB(name).bar.insert({x: 100}, writeConcern)); + + nodes[2].reconnect(nodes[0]); + nodes[2].reconnect(nodes[1]); + + // Step up a lagged node. + assert.commandWorked(nodes[1].adminCommand({replSetStepUp: 1})); + replSet.awaitNodesAgreeOnPrimary(replSet.kDefaultTimeoutMS, nodes); + assert.eq(ReplSetTest.State.PRIMARY, + assert.commandWorked(nodes[1].adminCommand('replSetGetStatus')).myState, + nodes[1].host + " was not primary after step-up"); + jsTestLog('node 1 is now primary, but cannot accept writes'); + + // Confirm that the most up-to-date node becomes primary + // after the default catchup delay. + replSet.waitForState(0, ReplSetTest.State.PRIMARY, 60 * 1000); + jsTestLog('node 0 performed catchup takeover and is now primary'); + + // Let the nodes catchup. + restartServerReplication(nodes[1]); + restartServerReplication(nodes[2]); + + // Confirm that the highest priority node becomes primary + // after catching up. + replSet.waitForState(2, ReplSetTest.State.PRIMARY, 30 * 1000); + jsTestLog('node 2 performed priority takeover and is now primary'); + +})();
\ No newline at end of file diff --git a/jstests/replsets/catchup_takeover_two_nodes_ahead.js b/jstests/replsets/catchup_takeover_two_nodes_ahead.js new file mode 100644 index 00000000000..4be8fee9007 --- /dev/null +++ b/jstests/replsets/catchup_takeover_two_nodes_ahead.js @@ -0,0 +1,56 @@ +// Test to ensure that a catchup takeover happens when the primary is lagged. +// Make sure that when two nodes are more caught up than the primary, +// the most up-to-date node becomes the primary. + +// 5-node replica set +// Start replica set. Ensure that node 0 becomes primary. +// Stop the replication for some nodes and have the primary write something. +// Stop replication for an up-to-date node and have the primary write something. +// Now the primary is most-up-to-date and another node is more up-to-date than others. +// Make a lagged node the next primary. +// Confirm that the most up-to-date node becomes primary. + +(function() { + 'use strict'; + + load('jstests/replsets/rslib.js'); + + var name = 'catchup_takeover_two_nodes_ahead'; + var replSet = new ReplSetTest({name: name, nodes: 5}); + var nodes = replSet.startSet(); + replSet.initiate(); + + // Wait until all nodes get the "no-op" of "new primary" after initial sync. + waitUntilAllNodesCaughtUp(nodes); + + // Write something so that nodes 0 and 1 are ahead. + stopServerReplication(nodes.slice(2, 5)); + var primary = replSet.getPrimary(); + var writeConcern = {writeConcern: {w: 2, wtimeout: replSet.kDefaultTimeoutMS}}; + assert.writeOK(primary.getDB(name).bar.insert({x: 100}, writeConcern)); + + // Write something so that node 0 is ahead of node 1. + stopServerReplication(nodes[1]); + writeConcern = {writeConcern: {w: 1, wtimeout: replSet.kDefaultTimeoutMS}}; + assert.writeOK(primary.getDB(name).bar.insert({y: 100}, writeConcern)); + + // Step up one of the lagged nodes. + assert.commandWorked(nodes[2].adminCommand({replSetStepUp: 1})); + replSet.awaitNodesAgreeOnPrimary(replSet.kDefaultTimeoutMS, nodes); + assert.eq(ReplSetTest.State.PRIMARY, + assert.commandWorked(nodes[2].adminCommand('replSetGetStatus')).myState, + nodes[2].host + " was not primary after step-up"); + jsTestLog('node 2 is now primary, but cannot accept writes'); + + // Make sure that node 2 cannot write anything. Because it is lagged and replication + // has been stopped, it shouldn't be able to become master. + assert.writeErrorWithCode(nodes[2].getDB(name).bar.insert({z: 100}, writeConcern), + ErrorCodes.NotMaster); + + // Confirm that the most up-to-date node becomes primary + // after the default catchup delay. + replSet.waitForState(0, ReplSetTest.State.PRIMARY, 60 * 1000); + + // Let the nodes catchup + restartServerReplication(nodes.slice(1, 5)); +})();
\ No newline at end of file |