1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
/**
* Test to ensure that replSetStepDown called on a primary will only succeed if a majority of nodes
* are caught up to it and that at least one node in this majority is electable. Tests this with a
* 5 node replica set.
*
* 1. Initiate a 5-node replica set
* 2. Disable replication to all secondaries
* 3. Execute some writes on primary
* 4. Try to step down primary and expect to fail
* 5. Enable replication to one unelectable secondary, secondary B
* 6. Await replication to secondary B by executing primary write with writeConcern:2
* 7. Try to step down primary and expect failure
* 8. Enable replication to a different unelectable secondary, secondary C
* 9. Await replication to secondary C by executing primary write with writeConcern:3
* 10. Try to step down primary and expect failure
* 11. Enable replication to an electable secondary, secondary A
* 12. Await replication to secondary A by executing primary write with writeConcern:4
* 13. Try to step down primary and expect success
* 14. Assert that original primary is now a secondary
*
*/
(function() {
'use strict';
load("jstests/libs/write_concern_util.js"); // for stopReplicationOnSecondaries,
// restartServerReplication,
// restartReplSetReplication
var name = 'stepdown_needs_electable_secondary';
var replTest = new ReplSetTest({
name: name,
nodes: 5,
nodeOptions: {
setParameter: {logComponentVerbosity: tojson({replication: 2}), numInitialSyncAttempts: 25},
}
});
var nodes = replTest.nodeList();
replTest.startSet();
replTest.initiate({
"_id": name,
"members": [
{"_id": 0, "host": nodes[0]},
{"_id": 1, "host": nodes[1]},
{"_id": 2, "host": nodes[2]},
{"_id": 3, "host": nodes[3], "priority": 0}, // unelectable
{"_id": 4, "host": nodes[4], "priority": 0} // unelectable
],
"settings": {"chainingAllowed": false}
});
function assertStepDownFailsWithExceededTimeLimit(node) {
assert.commandFailedWithCode(
node.adminCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}),
ErrorCodes.ExceededTimeLimit,
"step down did not fail with 'ExceededTimeLimit'");
}
function assertStepDownSucceeds(node) {
assert.commandWorked(node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60}));
}
var primary = replTest.getPrimary();
jsTestLog("Blocking writes to all secondaries.");
stopReplicationOnSecondaries(replTest);
jsTestLog("Doing a write to primary.");
var testDB = replTest.getPrimary().getDB('testdb');
var coll = testDB.stepdown_needs_electable_secondary;
var timeout = ReplSetTest.kDefaultTimeoutMS;
assert.commandWorked(
coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 1, wtimeout: timeout}}));
// Try to step down with only the primary caught up (1 node out of 5).
// stepDown should fail.
jsTestLog("Trying to step down primary with only 1 node out of 5 caught up.");
assertStepDownFailsWithExceededTimeLimit(primary);
// Get the two unelectable secondaries
var secondaryB_unelectable = replTest.nodes[3];
var secondaryC_unelectable = replTest.nodes[4];
// Get an electable secondary
var secondaryA_electable = replTest.getSecondaries().find(function(s) {
var nodeId = replTest.getNodeId(s);
return (nodeId !== 3 && nodeId !== 4); // nodes 3 and 4 are set to be unelectable
});
// Enable writes to Secondary B (unelectable). Await replication.
// (2 out of 5 nodes caught up, 0 electable)
// stepDown should fail due to no caught up majority.
jsTestLog("Re-enabling writes to unelectable secondary: node #" +
replTest.getNodeId(secondaryB_unelectable) + ", " + secondaryB_unelectable);
restartServerReplication(secondaryB_unelectable);
// Wait for this secondary to catch up by issuing a write that must be replicated to 2 nodes
assert.commandWorked(
coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 2, wtimeout: timeout}}));
// Try to step down and fail
jsTestLog("Trying to step down primary with only 2 nodes out of 5 caught up.");
assertStepDownFailsWithExceededTimeLimit(primary);
// Enable writes to Secondary C (unelectable). Await replication.
// (3 out of 5 nodes caught up, 0 electable)
// stepDown should fail due to caught up majority without electable node.
jsTestLog("Re-enabling writes to unelectable secondary: node #" +
replTest.getNodeId(secondaryC_unelectable) + ", " + secondaryC_unelectable);
restartServerReplication(secondaryC_unelectable);
// Wait for this secondary to catch up by issuing a write that must be replicated to 3 nodes
assert.commandWorked(
coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 3, wtimeout: timeout}}));
// Try to step down and fail
jsTestLog("Trying to step down primary with a caught up majority that " +
"doesn't contain an electable node.");
assertStepDownFailsWithExceededTimeLimit(primary);
// Enable writes to Secondary A (electable). Await replication.
// (4 out of 5 nodes caught up, 1 electable)
// stepDown should succeed due to caught up majority containing an electable node.
jsTestLog("Re-enabling writes to electable secondary: node #" +
replTest.getNodeId(secondaryA_electable) + ", " + secondaryA_electable);
restartServerReplication(secondaryA_electable);
// Wait for this secondary to catch up by issuing a write that must be replicated to 4 nodes
assert.commandWorked(
coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 4, wtimeout: timeout}}));
// Try to step down. We expect success, so catch the exception thrown by 'replSetStepDown'.
jsTestLog("Trying to step down primary with a caught up majority that " +
"does contain an electable node.");
assertStepDownSucceeds(primary);
// Make sure that original primary has transitioned to SECONDARY state
jsTestLog("Wait for PRIMARY " + primary.host + " to completely step down.");
replTest.waitForState(primary, ReplSetTest.State.SECONDARY);
// Disable all fail points for clean shutdown
restartReplSetReplication(replTest);
replTest.stopSet();
}());
|
