From 86a1a3a10511f24f374478fbe41b19a669f15e70 Mon Sep 17 00:00:00 2001 From: Cheahuychou Mao Date: Mon, 3 Feb 2020 17:59:47 -0500 Subject: SERVER-41070 Add blockConnection mode to failCommand --- jstests/core/failcommand_failpoint.js | 47 +++++++ .../failcommand_failpoint_not_parallel.js | 143 ++++++++++++++++++++- src/mongo/db/commands.cpp | 23 +++- 3 files changed, 211 insertions(+), 2 deletions(-) diff --git a/jstests/core/failcommand_failpoint.js b/jstests/core/failcommand_failpoint.js index 644d8060129..cd30696d145 100644 --- a/jstests/core/failcommand_failpoint.js +++ b/jstests/core/failcommand_failpoint.js @@ -19,6 +19,53 @@ const getThreadName = function() { let threadName = getThreadName(); +// Test idempotent configureFailPoint. +assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + errorCode: ErrorCodes.NotMaster, + failCommands: ["ping"], + threadName: threadName, + } +})); +assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster); +// Configure failCommand again and verify that it still works correctly. +assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + errorCode: ErrorCodes.NotMaster, + failCommands: ["ping"], + threadName: threadName, + } +})); +assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster); +assert.commandWorked(adminDB.runCommand({configureFailPoint: "failCommand", mode: "off"})); + +// Test switching command sets. +assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + errorCode: ErrorCodes.NotMaster, + failCommands: ["ping"], + threadName: threadName, + } +})); +assert.commandFailedWithCode(testDB.runCommand({ping: 1}), ErrorCodes.NotMaster); +assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + errorCode: ErrorCodes.NotMaster, + failCommands: ["isMaster"], + threadName: threadName, + } +})); +assert.commandWorked(testDB.runCommand({ping: 1})); +assert.commandWorked(adminDB.runCommand({configureFailPoint: "failCommand", mode: "off"})); + // Test failpoint with extraErrorInfo assert.commandWorked(adminDB.runCommand({ configureFailPoint: "failCommand", diff --git a/jstests/noPassthrough/failcommand_failpoint_not_parallel.js b/jstests/noPassthrough/failcommand_failpoint_not_parallel.js index 4cf36aed31b..f60ad7146d1 100644 --- a/jstests/noPassthrough/failcommand_failpoint_not_parallel.js +++ b/jstests/noPassthrough/failcommand_failpoint_not_parallel.js @@ -1,9 +1,12 @@ (function() { "use strict"; +load("jstests/libs/parallelTester.js"); + const conn = MongoRunner.runMongod(); assert.neq(null, conn); -const db = conn.getDB("test_failcommand_noparallel"); +const kDbName = "test_failcommand_noparallel"; +const db = conn.getDB(kDbName); // Test times when closing connection. // Use distinct because it is rarely used by internal operations, making it less likely unrelated @@ -21,5 +24,143 @@ assert.throws(() => db.runCommand({distinct: "c", key: "_id"})); assert.commandWorked(db.runCommand({distinct: "c", key: "_id"})); assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"})); +// Test the blockConnection patterns. +jsTest.log("Test validation of blockConnection fields"); +{ + // 'blockTimeMS' is required when 'blockConnection' is true. + assert.commandWorked(db.adminCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + blockConnection: true, + failCommands: ["isMaster"], + } + })); + assert.commandFailedWithCode(db.runCommand({isMaster: 1}), ErrorCodes.InvalidOptions); + + // 'blockTimeMS' must be non-negative. + assert.commandWorked(db.adminCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + blockConnection: true, + blockTimeMS: -100, + failCommands: ["isMaster"], + } + })); + assert.commandFailedWithCode(db.runCommand({isMaster: 1}), ErrorCodes.InvalidOptions); + + assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"})); +} + +// Insert a test document. +assert.commandWorked(db.runCommand({ + insert: "c", + documents: [{_id: 'block_test', run_id: 0}], +})); + +/** + * Returns the "run_id" of the test document. + */ +function checkRunId() { + const ret = db.runCommand({find: "c", filter: {_id: 'block_test'}}); + assert.commandWorked(ret); + + const doc = ret["cursor"]["firstBatch"][0]; + return doc["run_id"]; +} + +/** + * Runs update to increment the "run_id" of the test document by one. + */ +function incrementRunId() { + assert.commandWorked(db.runCommand({ + update: "c", + updates: [{q: {_id: 'block_test'}, u: {$inc: {run_id: 1}}}], + })); +} + +/** + * Starts and returns a thread for updating the test document by incrementing the + * "run_id" by one. + */ +function startIncrementRunIdThread() { + const latch = new CountDownLatch(1); + let thread = new Thread(function(connStr, dbName, latch) { + jsTest.log("Sending update"); + + const client = new Mongo(connStr); + const db = client.getDB(dbName); + latch.countDown(); + assert.commandWorked(db.runCommand({ + update: "c", + updates: [{q: {_id: 'block_test'}, u: {$inc: {run_id: 1}}}], + })); + + jsTest.log("Successfully applied update"); + }, conn.name, kDbName, latch); + thread.start(); + latch.await(); + return thread; +} + +assert.eq(checkRunId(), 0); +const kLargeBlockTimeMS = 60 * 1000; + +jsTest.log("Test that only commands listed in failCommands block"); +{ + assert.commandWorked(db.adminCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + blockConnection: true, + blockTimeMS: kLargeBlockTimeMS, + failCommands: ["update"], + } + })); + let thread = startIncrementRunIdThread(); + + // Check that other commands get through. + assert.commandWorked(db.runCommand({isMaster: 1})); + assert.eq(checkRunId(), 0); + + // Wait for the blocked update to get through. + thread.join(); + assert.soon(() => { + return checkRunId() == 1; + }); + + assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"})); +} + +jsTest.log("Test command changes"); +{ + assert.commandWorked(db.adminCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + blockConnection: true, + blockTimeMS: kLargeBlockTimeMS, + failCommands: ["update", "insert"], + } + })); + assert.eq(checkRunId(), 1); + + // Drop update from the command list and verify that the update gets through. + assert.commandWorked(db.adminCommand({ + configureFailPoint: "failCommand", + mode: "alwaysOn", + data: { + blockConnection: true, + blockTimeMS: kLargeBlockTimeMS, + failCommands: ["insert"], + } + })); + incrementRunId(); + assert.eq(checkRunId(), 2); + + assert.commandWorked(db.adminCommand({configureFailPoint: "failCommand", mode: "off"})); +} + MongoRunner.stopMongod(conn); }()); diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp index 338b427e024..e5707b01d37 100644 --- a/src/mongo/db/commands.cpp +++ b/src/mongo/db/commands.cpp @@ -540,6 +540,7 @@ bool CommandHelpers::shouldActivateFailCommandFailPoint(const BSONObj& data, void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx, const CommandInvocation* invocation) { bool closeConnection; + bool blockConnection; bool hasErrorCode; /** * Default value is used to suppress the uassert for `errorExtraInfo` if `errorCode` is not set. @@ -570,6 +571,23 @@ void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx, uasserted(50985, "Failing command due to 'failCommand' failpoint"); } + if (blockConnection) { + long long blockTimeMS = 0; + uassert(ErrorCodes::InvalidOptions, + "must specify 'blockTimeMS' when 'blockConnection' is true", + data.hasField("blockTimeMS") && + bsonExtractIntegerField(data, "blockTimeMS", &blockTimeMS).isOK()); + uassert(ErrorCodes::InvalidOptions, + "'blockTimeMS' must be non-negative", + blockTimeMS >= 0); + + log() << "Blocking command '" << cmd->getName() + << "' via 'failCommand' failpoint for " << blockTimeMS << " milliseconds"; + opCtx->sleepFor(Milliseconds{blockTimeMS}); + log() << "Unblocking command '" << cmd->getName() + << "' via 'failCommand' failpoint"; + } + if (hasExtraInfo) { log() << "Failing command '" << cmd->getName() << "' via 'failCommand' failpoint. Action: returning error code " << errorCode @@ -594,8 +612,11 @@ void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx, hasExtraInfo = data.hasField("errorExtraInfo") && bsonExtractTypedField(data, "errorExtraInfo", BSONType::Object, &errorExtraInfo) .isOK(); + blockConnection = data.hasField("blockConnection") && + bsonExtractBooleanField(data, "blockConnection", &blockConnection).isOK() && + blockConnection; return shouldActivateFailCommandFailPoint(data, invocation, opCtx->getClient()) && - (closeConnection || hasErrorCode); + (closeConnection || blockConnection || hasErrorCode); }); } -- cgit v1.2.1