1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
|
// Ensure stepDown operations that are waiting for replication can be interrupted with killOp()
// 1. Start up a 3 node set (1 arbiter).
// 2. Isolate the SECONDARY.
// 3. Do one write and then spin up a second shell which asks the PRIMARY to StepDown.
// 4. Once StepDown has begun, spin up a third shell which will attempt to do writes, which should
// block waiting for stepDown to release its lock, which it never will do because no secondaries
// are caught up.
// 5. Once a write is blocked, kill the stepDown operation
// 6. Writes should become unblocked and the primary should stay primary
(function () {
"use strict";
var name = "interruptStepDown";
var replSet = new ReplSetTest({name: name, nodes: 3});
var nodes = replSet.nodeList();
replSet.startSet();
replSet.initiate({"_id" : name,
"members" : [
{"_id" : 0, "host" : nodes[0], "priority" : 3},
{"_id" : 1, "host" : nodes[1]},
{"_id" : 2, "host" : nodes[2], "arbiterOnly" : true}]});
jsTestLog("Isolating primary");
replSet.bridge();
replSet.partition(1,0);
replSet.partition(1,2);
var primary = replSet.getPrimary();
assert.eq(primary.host, nodes[0], "primary assumed to be node 0");
// do a write then ask the PRIMARY to stepdown
jsTestLog("Initiating stepdown");
assert.writeOK(primary.getDB(name).foo.insert({myDoc: true, x: 1},
{writeConcern: {w: 1, wtimeout: 60000}}));
var stepDownCmd = function() {
var res = db.getSiblingDB('admin').runCommand({replSetStepDown: 60,
secondaryCatchUpPeriodSecs: 60});
assert.commandFailedWithCode(res, 11601 /*interrupted*/);
}
var stepDowner = startParallelShell(stepDownCmd, primary.port);
var stepDownOpID = -1;
jsTestLog("Looking for stepdown in currentOp() output");
assert.soon(function() {
var res = primary.getDB('admin').currentOp(true);
for (var index in res.inprog) {
var entry = res.inprog[index];
if (entry["query"] && entry["query"]["replSetStepDown"] === 60){
stepDownOpID = entry.opid;
return true;
}
}
printjson(res);
return false;
}, "global shared lock not acquired");
jsTestLog("Ensuring writes block on the stepdown");
// Start repeatedly doing an update until one blocks waiting for the lock.
// If the test is successful this thread will be terminated when we remove the document
// being updated.
var updateCmd = function() {
while(true) {
var res = db.getSiblingDB("interruptStepDown").foo.update({myDoc: true},
{$inc: {x: 1}});
assert.writeOK(res);
if (res.nModified == 0) {
quit(0);
}
else {
printjson(res);
}
}
};
var writer = startParallelShell(updateCmd, primary.port);
assert.soon(function() {
var res = primary.getDB(name).currentOp();
for (var entry in res.inprog) {
if (res.inprog[entry]["waitingForLock"]) {
return true;
}
}
printjson(res);
return false;
}, "write never blocked on the global shared lock");
// kill the stepDown and ensure that that unblocks writes to the db
jsTestLog("Killing stepdown");
primary.getDB('admin').killOp(stepDownOpID);
var exitCode = stepDowner();
assert.eq(0, exitCode);
assert.writeOK(primary.getDB(name).foo.remove({}));
exitCode = writer();
assert.eq(0, exitCode);
})();
|
