SERVER-41070 Add blockConnection mode to failCommand

3 files changed, 211 insertions, 2 deletions

diff --git a/jstests/core/failcommand_failpoint.js b/jstests/core/failcommand_failpoint.js
index 644d8060129..cd30696d145 100644
--- a/jstests/core/failcommand_failpoint.js
+++ b/jstests/core/failcommand_failpoint.js
@@ -19,6 +19,53 @@ const getThreadName = function() {
 
 let threadName = getThreadName();
 
+// Test idempotent configureFailPoint.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["ping"],
+        threadName: threadName,
+    }
+}));
+assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster);
+// Configure failCommand again and verify that it still works correctly.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["ping"],
+        threadName: threadName,
+    }
+}));
+assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster);
+assert.commandWorked(adminDB.runCommand({configureFailPoint: "failCommand", mode: "off"}));
+
+// Test switching command sets.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["ping"],
+        threadName: threadName,
+    }
+}));
+assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster);
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["isMaster"],
+        threadName: threadName,
+    }
+}));
+assert.commandWorked(testDB.runCommand({ping: 1}));
+assert.commandWorked(adminDB.runCommand({configureFailPoint: "failCommand", mode: "off"}));
+
 // Test failpoint with extraErrorInfo
 assert.commandWorked(adminDB.runCommand({
     configureFailPoint: "failCommand",
diff --git a/jstests/noPassthrough/failcommand_failpoint_not_parallel.js b/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
index 4cf36aed31b..f60ad7146d1 100644
--- a/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
+++ b/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
@@ -1,9 +1,12 @@
 (function() {
 "use strict";
 
+load("jstests/libs/parallelTester.js");
+
 const conn = MongoRunner.runMongod();
 assert.neq(null, conn);
-const db = conn.getDB("test_failcommand_noparallel");
+const kDbName = "test_failcommand_noparallel";
+const db = conn.getDB(kDbName);
 
 // Test times when closing connection.
 // Use distinct because it is rarely used by internal operations, making it less likely unrelated
@@ -21,5 +24,143 @@ assert.throws(() => db.runCommand({distinct: "c", key: "_id"}));
 assert.commandWorked(db.runCommand({distinct: "c", key: "_id"}));
 assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
 
+// Test the blockConnection patterns.
+jsTest.log("Test validation of blockConnection fields");
+{
+    // 'blockTimeMS' is required when 'blockConnection' is true.
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            failCommands: ["isMaster"],
+        }
+    }));
+    assert.commandFailedWithCode(db.runCommand({isMaster: 1}), ErrorCodes.InvalidOptions);
+
+    // 'blockTimeMS' must be non-negative.
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            blockTimeMS: -100,
+            failCommands: ["isMaster"],
+        }
+    }));
+    assert.commandFailedWithCode(db.runCommand({isMaster: 1}), ErrorCodes.InvalidOptions);
+
+    assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+}
+
+// Insert a test document.
+assert.commandWorked(db.runCommand({
+    insert: "c",
+    documents: [{_id: 'block_test', run_id: 0}],
+}));
+
+/**
+ * Returns the "run_id" of the test document.
+ */
+function checkRunId() {
+    const ret = db.runCommand({find: "c", filter: {_id: 'block_test'}});
+    assert.commandWorked(ret);
+
+    const doc = ret["cursor"]["firstBatch"][0];
+    return doc["run_id"];
+}
+
+/**
+ * Runs update to increment the "run_id" of the test document by one.
+ */
+function incrementRunId() {
+    assert.commandWorked(db.runCommand({
+        update: "c",
+        updates: [{q: {_id: 'block_test'}, u: {$inc: {run_id: 1}}}],
+    }));
+}
+
+/**
+ * Starts and returns a thread for updating the test document by incrementing the
+ * "run_id" by one.
+ */
+function startIncrementRunIdThread() {
+    const latch = new CountDownLatch(1);
+    let thread = new Thread(function(connStr, dbName, latch) {
+        jsTest.log("Sending update");
+
+        const client = new Mongo(connStr);
+        const db = client.getDB(dbName);
+        latch.countDown();
+        assert.commandWorked(db.runCommand({
+            update: "c",
+            updates: [{q: {_id: 'block_test'}, u: {$inc: {run_id: 1}}}],
+        }));
+
+        jsTest.log("Successfully applied update");
+    }, conn.name, kDbName, latch);
+    thread.start();
+    latch.await();
+    return thread;
+}
+
+assert.eq(checkRunId(), 0);
+const kLargeBlockTimeMS = 60 * 1000;
+
+jsTest.log("Test that only commands listed in failCommands block");
+{
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            blockTimeMS: kLargeBlockTimeMS,
+            failCommands: ["update"],
+        }
+    }));
+    let thread = startIncrementRunIdThread();
+
+    // Check that other commands get through.
+    assert.commandWorked(db.runCommand({isMaster: 1}));
+    assert.eq(checkRunId(), 0);
+
+    // Wait for the blocked update to get through.
+    thread.join();
+    assert.soon(() => {
+        return checkRunId() == 1;
+    });
+
+    assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+}
+
+jsTest.log("Test command changes");
+{
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            blockTimeMS: kLargeBlockTimeMS,
+            failCommands: ["update", "insert"],
+        }
+    }));
+    assert.eq(checkRunId(), 1);
+
+    // Drop update from the command list and verify that the update gets through.
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            blockTimeMS: kLargeBlockTimeMS,
+            failCommands: ["insert"],
+        }
+    }));
+    incrementRunId();
+    assert.eq(checkRunId(), 2);
+
+    assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+}
+
 MongoRunner.stopMongod(conn);
 }());
diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp
index 338b427e024..e5707b01d37 100644
--- a/src/mongo/db/commands.cpp
+++ b/src/mongo/db/commands.cpp
@@ -540,6 +540,7 @@ bool CommandHelpers::shouldActivateFailCommandFailPoint(const BSONObj& data,
 void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx,
                                                   const CommandInvocation* invocation) {
     bool closeConnection;
+    bool blockConnection;
     bool hasErrorCode;
     /**
      * Default value is used to suppress the uassert for `errorExtraInfo` if `errorCode` is not set.
@@ -570,6 +571,23 @@ void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx,
                 uasserted(50985, "Failing command due to 'failCommand' failpoint");
             }
 
+            if (blockConnection) {
+                long long blockTimeMS = 0;
+                uassert(ErrorCodes::InvalidOptions,
+                        "must specify 'blockTimeMS' when 'blockConnection' is true",
+                        data.hasField("blockTimeMS") &&
+                            bsonExtractIntegerField(data, "blockTimeMS", &blockTimeMS).isOK());
+                uassert(ErrorCodes::InvalidOptions,
+                        "'blockTimeMS' must be non-negative",
+                        blockTimeMS >= 0);
+
+                log() << "Blocking command '" << cmd->getName()
+                      << "' via 'failCommand' failpoint for " << blockTimeMS << " milliseconds";
+                opCtx->sleepFor(Milliseconds{blockTimeMS});
+                log() << "Unblocking command '" << cmd->getName()
+                      << "' via 'failCommand' failpoint";
+            }
+
             if (hasExtraInfo) {
                 log() << "Failing command '" << cmd->getName()
                       << "' via 'failCommand' failpoint. Action: returning error code " << errorCode
@@ -594,8 +612,11 @@ void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx,
             hasExtraInfo = data.hasField("errorExtraInfo") &&
                 bsonExtractTypedField(data, "errorExtraInfo", BSONType::Object, &errorExtraInfo)
                     .isOK();
+            blockConnection = data.hasField("blockConnection") &&
+                bsonExtractBooleanField(data, "blockConnection", &blockConnection).isOK() &&
+                blockConnection;
             return shouldActivateFailCommandFailPoint(data, invocation, opCtx->getClient()) &&
-                (closeConnection || hasErrorCode);
+                (closeConnection || blockConnection || hasErrorCode);
         });
 }