SERVER-15997 Make sure stepdown can be interrupted with killop

1 files changed, 96 insertions, 0 deletions

diff --git a/jstests/replsets/stepdown_killop.js b/jstests/replsets/stepdown_killop.js
new file mode 100644
index 00000000000..fd7975478d4
--- /dev/null
+++ b/jstests/replsets/stepdown_killop.js
@@ -0,0 +1,96 @@
+// Ensure stepDown operations that are waiting for replication can be interrupted with killOp()
+// 1. Start up a 3 node set (1 arbiter).
+// 2. Isolate the SECONDARY.
+// 3. Do one write and then spin up a second shell which asks the PRIMARY to StepDown.
+// 4. Once StepDown has begun, spin up a third shell which will attempt to do writes, which should
+//    block waiting for stepDown to release its lock, which it never will do because no secondaries
+//    are caught up.
+// 5. Once a write is blocked, kill the stepDown operation
+// 6. Writes should become unblocked and the primary should stay primary
+
+(function () {
+     "use strict";
+     var name = "interruptStepDown";
+     var replSet = new ReplSetTest({name: name, nodes: 3});
+     var nodes = replSet.nodeList();
+     replSet.startSet();
+     replSet.initiate({"_id" : name,
+                       "members" : [
+                           {"_id" : 0, "host" : nodes[0], "priority" : 3},
+                           {"_id" : 1, "host" : nodes[1]},
+                           {"_id" : 2, "host" : nodes[2], "arbiterOnly" : true}]});
+
+     jsTestLog("Isolating primary");
+     replSet.bridge();
+     replSet.partition(1,0);
+     replSet.partition(1,2);
+     var primary = replSet.getPrimary();
+     assert.eq(primary.host, nodes[0], "primary assumed to be node 0");
+
+     // do a write then ask the PRIMARY to stepdown
+     jsTestLog("Initiating stepdown");
+     assert.writeOK(primary.getDB(name).foo.insert({myDoc: true, x: 1},
+                                                   {writeConcern: {w: 1, wtimeout: 60000}}));
+     var stepDownCmd = function() {
+         var res = db.getSiblingDB('admin').runCommand({replSetStepDown: 60,
+                                                        secondaryCatchUpPeriodSecs: 60});
+         assert.commandFailedWithCode(res, 11601 /*interrupted*/);
+     }
+     var stepDowner = startParallelShell(stepDownCmd, primary.port);
+     var stepDownOpID = -1;
+
+     jsTestLog("Looking for stepdown in currentOp() output");
+     assert.soon(function() {
+                     var res = primary.getDB('admin').currentOp(true);
+                     for (var index in res.inprog) {
+                         var entry = res.inprog[index];
+                         if (entry["query"] && entry["query"]["replSetStepDown"] === 60){
+                             stepDownOpID = entry.opid;
+                             return true;
+                         }
+                     }
+                     printjson(res);
+                     return false;
+                 }, "global shared lock not acquired");
+
+     jsTestLog("Ensuring writes block on the stepdown");
+     // Start repeatedly doing an update until one blocks waiting for the lock.
+     // If the test is successful this thread will be terminated when we remove the document
+     // being updated.
+     var updateCmd = function() {
+         while(true) {
+             var res = db.getSiblingDB("interruptStepDown").foo.update({myDoc: true},
+                                                                       {$inc: {x: 1}});
+             assert.writeOK(res);
+             if (res.nModified == 0) {
+                 quit(0);
+             }
+             else {
+                 printjson(res);
+             }
+
+         }
+     };
+     var writer = startParallelShell(updateCmd, primary.port);
+     assert.soon(function() {
+                     var res = primary.getDB(name).currentOp();
+                     for (var entry in res.inprog) {
+                         if (res.inprog[entry]["waitingForLock"]) {
+                             return true;
+                         }
+                     }
+                     printjson(res);
+                     return false;
+                 }, "write never blocked on the global shared lock");
+
+     // kill the stepDown and ensure that that unblocks writes to the db
+     jsTestLog("Killing stepdown");
+     primary.getDB('admin').killOp(stepDownOpID);
+
+     var exitCode = stepDowner();
+     assert.eq(0, exitCode);
+
+     assert.writeOK(primary.getDB(name).foo.remove({}));
+     exitCode = writer();
+     assert.eq(0, exitCode);
+ })();