1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
// Ensure stepDown operations that are waiting for replication can be interrupted with killOp()
// 1. Start up a 3 node set (1 arbiter).
// 2. Stop replication on the SECONDARY using a fail point.
// 3. Do one write and then spin up a second shell which asks the PRIMARY to StepDown.
// 4. Once StepDown has begun, spin up a third shell which will attempt to do writes, which should
// block waiting for stepDown to release its lock, which it never will do because no secondaries
// are caught up.
// 5. Once a write is blocked, kill the stepDown operation
// 6. Writes should become unblocked and the primary should stay primary
(function() {
"use strict";
var name = "interruptStepDown";
var replSet = new ReplSetTest({name: name, nodes: 3});
var nodes = replSet.nodeList();
replSet.startSet();
replSet.initiate({
"_id": name,
"members": [
{"_id": 0, "host": nodes[0], "priority": 3},
{"_id": 1, "host": nodes[1]},
{"_id": 2, "host": nodes[2], "arbiterOnly": true}
]
});
replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000);
var secondary = replSet.getSecondary();
jsTestLog('Disable replication on the SECONDARY ' + secondary.host);
assert.commandWorked(secondary.getDB('admin').runCommand(
{configureFailPoint: 'rsSyncApplyStop', mode: 'alwaysOn'}),
'Failed to configure rsSyncApplyStop failpoint.');
replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000);
var primary = replSet.getPrimary();
assert.eq(primary.host, nodes[0], "primary assumed to be node 0");
// do a write then ask the PRIMARY to stepdown
jsTestLog("Initiating stepdown");
assert.writeOK(primary.getDB(name).foo.insert({myDoc: true, x: 1},
{writeConcern: {w: 1, wtimeout: 60000}}));
var stepDownCmd = function() {
var res = db.getSiblingDB('admin').runCommand(
{replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60});
assert.commandFailedWithCode(res, 11601 /*interrupted*/);
};
var stepDowner = startParallelShell(stepDownCmd, primary.port);
var stepDownOpID = -1;
jsTestLog("Looking for stepdown in currentOp() output");
assert.soon(function() {
var res = primary.getDB('admin').currentOp(true);
for (var index in res.inprog) {
var entry = res.inprog[index];
if (entry["query"] && entry["query"]["replSetStepDown"] === 60) {
stepDownOpID = entry.opid;
return true;
}
}
printjson(res);
return false;
}, "global shared lock not acquired");
jsTestLog("Ensuring writes block on the stepdown");
// Start repeatedly doing an update until one blocks waiting for the lock.
// If the test is successful this thread will be terminated when we remove the document
// being updated.
var updateCmd = function() {
while (true) {
var res =
db.getSiblingDB("interruptStepDown").foo.update({myDoc: true}, {$inc: {x: 1}});
assert.writeOK(res);
if (res.nModified == 0) {
quit(0);
} else {
printjson(res);
}
}
};
var writer = startParallelShell(updateCmd, primary.port);
assert.soon(function() {
var res = primary.getDB(name).currentOp();
for (var entry in res.inprog) {
if (res.inprog[entry]["waitingForLock"]) {
return true;
}
}
printjson(res);
return false;
}, "write never blocked on the global shared lock");
// kill the stepDown and ensure that that unblocks writes to the db
jsTestLog("Killing stepdown");
primary.getDB('admin').killOp(stepDownOpID);
var exitCode = stepDowner();
assert.eq(0, exitCode);
assert.writeOK(primary.getDB(name).foo.remove({}));
exitCode = writer();
assert.eq(0, exitCode);
})();
|
