SERVER-41070 Add blockConnection mode to failCommand

(cherry picked from commit 86a1a3a10511f24f374478fbe41b19a669f15e70)

3 files changed, 209 insertions, 4 deletions

diff --git a/jstests/core/failcommand_failpoint.js b/jstests/core/failcommand_failpoint.js
index 4e94e052f8a..8c8b0571473 100644
--- a/jstests/core/failcommand_failpoint.js
+++ b/jstests/core/failcommand_failpoint.js
@@ -16,6 +16,53 @@ const getThreadName = function() {
 
 let threadName = getThreadName();
 
+// Test idempotent configureFailPoint.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["ping"],
+        threadName: threadName,
+    }
+}));
+assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster);
+// Configure failCommand again and verify that it still works correctly.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["ping"],
+        threadName: threadName,
+    }
+}));
+assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster);
+assert.commandWorked(adminDB.runCommand({configureFailPoint: "failCommand", mode: "off"}));
+
+// Test switching command sets.
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["ping"],
+        threadName: threadName,
+    }
+}));
+assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster);
+assert.commandWorked(adminDB.runCommand({
+    configureFailPoint: "failCommand",
+    mode: "alwaysOn",
+    data: {
+        errorCode: ErrorCodes.NotMaster,
+        failCommands: ["isMaster"],
+        threadName: threadName,
+    }
+}));
+assert.commandWorked(testDB.runCommand({ping: 1}));
+assert.commandWorked(adminDB.runCommand({configureFailPoint: "failCommand", mode: "off"}));
+
 // Test failing with a particular error code.
 assert.commandWorked(adminDB.runCommand({
     configureFailPoint: "failCommand",
diff --git a/jstests/noPassthrough/failcommand_failpoint_not_parallel.js b/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
index a9d776d0079..386e8c8f7e7 100644
--- a/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
+++ b/jstests/noPassthrough/failcommand_failpoint_not_parallel.js
@@ -1,9 +1,12 @@
 (function() {
 "use strict";
 
+load("jstests/libs/parallelTester.js");
+
 const conn = MongoRunner.runMongod();
 assert.neq(null, conn);
-const db = conn.getDB("test_failcommand_noparallel");
+const kDbName = "test_failcommand_noparallel";
+const db = conn.getDB(kDbName);
 
 // Test times when closing connection.
 assert.commandWorked(db.adminCommand({
@@ -19,5 +22,143 @@ assert.throws(() => db.runCommand({find: "c"}));
 assert.commandWorked(db.runCommand({find: "c"}));
 assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
 
+// Test the blockConnection patterns.
+jsTest.log("Test validation of blockConnection fields");
+{
+    // 'blockTimeMS' is required when 'blockConnection' is true.
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            failCommands: ["isMaster"],
+        }
+    }));
+    assert.commandFailedWithCode(db.runCommand({isMaster: 1}), ErrorCodes.InvalidOptions);
+
+    // 'blockTimeMS' must be non-negative.
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            blockTimeMS: -100,
+            failCommands: ["isMaster"],
+        }
+    }));
+    assert.commandFailedWithCode(db.runCommand({isMaster: 1}), ErrorCodes.InvalidOptions);
+
+    assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+}
+
+// Insert a test document.
+assert.commandWorked(db.runCommand({
+    insert: "c",
+    documents: [{_id: 'block_test', run_id: 0}],
+}));
+
+/**
+ * Returns the "run_id" of the test document.
+ */
+function checkRunId() {
+    const ret = db.runCommand({find: "c", filter: {_id: 'block_test'}});
+    assert.commandWorked(ret);
+
+    const doc = ret["cursor"]["firstBatch"][0];
+    return doc["run_id"];
+}
+
+/**
+ * Runs update to increment the "run_id" of the test document by one.
+ */
+function incrementRunId() {
+    assert.commandWorked(db.runCommand({
+        update: "c",
+        updates: [{q: {_id: 'block_test'}, u: {$inc: {run_id: 1}}}],
+    }));
+}
+
+/**
+ * Starts and returns a thread for updating the test document by incrementing the
+ * "run_id" by one.
+ */
+function startIncrementRunIdThread() {
+    const latch = new CountDownLatch(1);
+    let thread = new Thread(function(connStr, dbName, latch) {
+        jsTest.log("Sending update");
+
+        const client = new Mongo(connStr);
+        const db = client.getDB(dbName);
+        latch.countDown();
+        assert.commandWorked(db.runCommand({
+            update: "c",
+            updates: [{q: {_id: 'block_test'}, u: {$inc: {run_id: 1}}}],
+        }));
+
+        jsTest.log("Successfully applied update");
+    }, conn.name, kDbName, latch);
+    thread.start();
+    latch.await();
+    return thread;
+}
+
+assert.eq(checkRunId(), 0);
+const kLargeBlockTimeMS = 60 * 1000;
+
+jsTest.log("Test that only commands listed in failCommands block");
+{
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            blockTimeMS: kLargeBlockTimeMS,
+            failCommands: ["update"],
+        }
+    }));
+    let thread = startIncrementRunIdThread();
+
+    // Check that other commands get through.
+    assert.commandWorked(db.runCommand({isMaster: 1}));
+    assert.eq(checkRunId(), 0);
+
+    // Wait for the blocked update to get through.
+    thread.join();
+    assert.soon(() => {
+        return checkRunId() == 1;
+    });
+
+    assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+}
+
+jsTest.log("Test command changes");
+{
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            blockTimeMS: kLargeBlockTimeMS,
+            failCommands: ["update", "insert"],
+        }
+    }));
+    assert.eq(checkRunId(), 1);
+
+    // Drop update from the command list and verify that the update gets through.
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "failCommand",
+        mode: "alwaysOn",
+        data: {
+            blockConnection: true,
+            blockTimeMS: kLargeBlockTimeMS,
+            failCommands: ["insert"],
+        }
+    }));
+    incrementRunId();
+    assert.eq(checkRunId(), 2);
+
+    assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"}));
+}
+
 MongoRunner.stopMongod(conn);
 }());
diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp
index fc356126564..381d426b2ab 100644
--- a/src/mongo/db/commands.cpp
+++ b/src/mongo/db/commands.cpp
@@ -498,8 +498,8 @@ bool CommandHelpers::shouldActivateFailCommandFailPoint(const BSONObj& data,
 void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx,
                                                   StringData commandName,
                                                   const NamespaceString& nss) {
-    bool closeConnection, hasErrorCode;
-    long long errorCode;
+    bool closeConnection, hasErrorCode, blockConnection, hasBlockTimeMS;
+    long long errorCode, blockTimeMS;
 
     MONGO_FAIL_POINT_BLOCK_IF(failCommand, data, [&](const BSONObj& data) {
         closeConnection = data.hasField("closeConnection") &&
@@ -507,9 +507,13 @@ void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx,
             closeConnection;
         hasErrorCode = data.hasField("errorCode") &&
             bsonExtractIntegerField(data, "errorCode", &errorCode).isOK();
+        blockConnection = data.hasField("blockConnection") &&
+            bsonExtractBooleanField(data, "blockConnection", &blockConnection).isOK();
+        hasBlockTimeMS = data.hasField("blockTimeMS") &&
+            bsonExtractIntegerField(data, "blockTimeMS", &blockTimeMS).isOK();
 
         return shouldActivateFailCommandFailPoint(data, commandName, opCtx->getClient(), nss) &&
-            (closeConnection || hasErrorCode);
+            (closeConnection || blockConnection || hasErrorCode);
     }) {
         if (closeConnection) {
             opCtx->getClient()->session()->end();
@@ -525,6 +529,19 @@ void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx,
             uasserted(ErrorCodes::Error(errorCode),
                       "Failing command due to 'failCommand' failpoint");
         }
+
+        if (blockConnection) {
+            uassert(ErrorCodes::InvalidOptions,
+                    "must specify 'blockTimeMS' when 'blockConnection' is true",
+                    hasBlockTimeMS);
+            uassert(
+                ErrorCodes::InvalidOptions, "'blockTimeMS' must be non-negative", blockTimeMS >= 0);
+
+            log() << "Blocking command via 'failCommand' failpoint for " << blockTimeMS
+                  << " milliseconds";
+            opCtx->sleepFor(Milliseconds{blockTimeMS});
+            log() << "Unblocking command via 'failCommand' failpoint";
+        }
     }
 }