SERVER-39165 Add waitForFailpoint command and update non-repl tests

19 files changed, 283 insertions, 42 deletions

diff --git a/jstests/core/background_validation.js b/jstests/core/background_validation.js
index e17885e8e2f..278c9acbc12 100644
--- a/jstests/core/background_validation.js
+++ b/jstests/core/background_validation.js
@@ -22,7 +22,7 @@
 (function() {
 'use strict';
 
-load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 const forceCheckpoint = () => {
     assert.commandWorked(db.fsyncLock());
@@ -67,8 +67,7 @@ assert(res.valid, "Validate cmd with {background:true} failed: " + tojson(res));
 
 // Set a failpoint in the background validation code to pause validation while holding a collection
 // lock.
-assert.commandWorked(
-    db.adminCommand({configureFailPoint: "pauseCollectionValidationWithLock", mode: "alwaysOn"}));
+let failPoint = configureFailPoint(db, "pauseCollectionValidationWithLock");
 
 // Start an asynchronous thread to run collection validation with {background:true}.
 let awaitValidateCommand = startParallelShell(function() {
@@ -82,7 +81,7 @@ let awaitValidateCommand = startParallelShell(function() {
 });
 
 // Wait for background validation command to start.
-checkLog.contains(db.getMongo(), "Failpoint 'pauseCollectionValidationWithLock' activated.");
+failPoint.wait();
 
 jsTest.log("Should start hanging now......");
 
@@ -96,8 +95,7 @@ assert.eq(1,
           "expected to find a single document, found: " + tojson(docRes.toArray()));
 
 // Clear the failpoint and make sure the validate command was successful.
-assert.commandWorked(
-    db.adminCommand({configureFailPoint: "pauseCollectionValidationWithLock", mode: "off"}));
+failPoint.off();
 awaitValidateCommand();
 
 /**
diff --git a/jstests/core/txns/kill_op_on_txn_expiry.js b/jstests/core/txns/kill_op_on_txn_expiry.js
index 49c5c72299c..daabb1048ce 100644
--- a/jstests/core/txns/kill_op_on_txn_expiry.js
+++ b/jstests/core/txns/kill_op_on_txn_expiry.js
@@ -3,8 +3,9 @@
 (function() {
 "use strict";
 
-load('jstests/libs/parallelTester.js');
 load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
+load('jstests/libs/parallelTester.js');
 
 const dbName = "test";
 const collName = "kill_op_on_txn_expiry";
@@ -46,10 +47,8 @@ try {
     }));
 
     jsTestLog("Enabling fail point to block batch inserts");
-    assert.commandWorked(
-        testDB.adminCommand({configureFailPoint: "hangDuringBatchInsert", mode: "alwaysOn"}));
-    // Clear ramlog so checkLog can't find log messages from previous times this fail point was
-    // enabled.
+    let failPoint = configureFailPoint(testDB, "hangDuringBatchInsert");
+    // Clear ramlog so checkLog can't find log messages from the previous times this test was run.
     assert.commandWorked(testDB.adminCommand({clearLog: 'global'}));
 
     jsTestLog("Starting insert operation in parallel thread");
@@ -70,15 +69,14 @@ try {
     workerThread.start();
 
     jsTestLog("Wait for insert to be blocked");
-    checkLog.contains(db.getMongo(), "hangDuringBatchInsert fail point enabled");
+    failPoint.wait();
 
     jsTestLog("Wait for the transaction to expire");
     checkLog.contains(db.getMongo(), "Aborting transaction with txnNumber " + txnNumber);
 
     jsTestLog("Disabling fail point to enable insert to proceed and detect that the session " +
               "has been killed");
-    assert.commandWorked(
-        testDB.adminCommand({configureFailPoint: "hangDuringBatchInsert", mode: "off"}));
+    failPoint.off();
 
     workerThread.join();
     assert(!workerThread.hasFailed());
diff --git a/jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js b/jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js
index a3334f6a7bc..8abe4c30a1e 100644
--- a/jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js
+++ b/jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js
@@ -6,8 +6,9 @@
  */
 (function() {
 'use strict';
-load("jstests/libs/check_log.js");
+
 load('jstests/core/txns/libs/prepare_helpers.js');
+load("jstests/libs/fail_point_util.js");
 load('jstests/libs/parallel_shell_helpers.js');
 
 TestData.dbName = 'test';
@@ -36,11 +37,12 @@ TestData.otherDocFilter = {
  * field. This function is run in a separate thread and tests that oplog visibility blocks
  * certain reads and that prepare conflicts block other types of reads.
  */
-const readThreadFunc = function(readFunc, _collName) {
+const readThreadFunc = function(readFunc, _collName, timesEntered) {
     load("jstests/libs/check_log.js");
 
     // Do not start reads until we are blocked in 'prepareTransaction'.
-    checkLog.contains(db.getMongo(), "hangAfterReservingPrepareTimestamp fail point enabled");
+    assert.commandWorked(db.adminCommand(
+        {waitForFailPoint: "hangAfterReservingPrepareTimestamp", timesEntered: timesEntered}));
 
     // Create a 'readFuncObj' from the 'readFunc'.
     const readFuncObj = readFunc(_collName);
@@ -65,8 +67,7 @@ function runTest(prefix, readFunc) {
     testColl.drop({writeConcern: {w: "majority"}});
     assert.commandWorked(testDB.runCommand({create: collName, writeConcern: {w: 'majority'}}));
 
-    assert.commandWorked(testDB.adminCommand(
-        {configureFailPoint: 'hangAfterReservingPrepareTimestamp', mode: 'alwaysOn'}));
+    let failPoint = configureFailPoint(testDB, "hangAfterReservingPrepareTimestamp");
 
     // Insert a document for the transaction.
     assert.commandWorked(testColl.insert(TestData.txnDoc));
@@ -89,7 +90,8 @@ function runTest(prefix, readFunc) {
     // Clear the log history to ensure we only see the most recent 'prepareTransaction'
     // failpoint log message.
     assert.commandWorked(db.adminCommand({clearLog: 'global'}));
-    const joinReadThread = startParallelShell(funWithArgs(readThreadFunc, readFunc, collName));
+    const joinReadThread = startParallelShell(
+        funWithArgs(readThreadFunc, readFunc, collName, failPoint.timesEntered + 1));
 
     jsTestLog("Preparing the transaction for " + prefix);
     const prepareTimestamp = PrepareHelpers.prepareTransaction(session);
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index 0f58f1a34da..46c2af89543 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -532,6 +532,7 @@ let viewsCommandTests = {
     voteCommitIndexBuild: {skip: isUnrelated},
     voteCommitTransaction: {skip: isUnrelated},
     voteAbortTransaction: {skip: isUnrelated},
+    waitForFailPoint: {skip: isUnrelated},
     whatsmyuri: {skip: isUnrelated},
     whatsmysni: {skip: isUnrelated}
 };
diff --git a/jstests/fail_point/fail_point.js b/jstests/fail_point/fail_point.js
index 6cd53fe5ad1..3355e07d435 100644
--- a/jstests/fail_point/fail_point.js
+++ b/jstests/fail_point/fail_point.js
@@ -79,6 +79,16 @@ function runTest(adminDB) {
     res = adminDB.runCommand({getParameter: 1, "failpoint.dummy": 1});
     assert.commandWorked(res);
     expectFailPointState(res["failpoint.dummy"], 1, {x: 1});
+
+    // Test that the timeout for waitForFailPoint can be set via maxTimeMS.
+    var configureFailPointRes = adminDB.runCommand({configureFailPoint: 'dummy', mode: 'alwaysOn'});
+    assert.commandWorked(configureFailPointRes);
+    assert.commandFailedWithCode(adminDB.adminCommand({
+        waitForFailPoint: "dummy",
+        timesEntered: configureFailPointRes.count + 1,
+        maxTimeMS: 10
+    }),
+                                 ErrorCodes.MaxTimeMSExpired);
 }
 
 var conn = MongoRunner.runMongod();
diff --git a/jstests/libs/fail_point_util.js b/jstests/libs/fail_point_util.js
new file mode 100644
index 00000000000..6016fd88a78
--- /dev/null
+++ b/jstests/libs/fail_point_util.js
@@ -0,0 +1,40 @@
+/**
+ * Utilities for turning on/off and waiting for fail points.
+ */
+
+var configureFailPoint;
+
+(function() {
+"use strict";
+
+if (configureFailPoint) {
+    return;  // Protect against this file being double-loaded.
+}
+
+configureFailPoint = function(conn, failPointName, data = {}, failPointMode = "alwaysOn") {
+    return {
+        conn: conn,
+        failPointName: failPointName,
+        timesEntered: assert
+                          .commandWorked(conn.adminCommand(
+                              {configureFailPoint: failPointName, mode: failPointMode, data: data}))
+                          .count,
+        wait:
+            function(additionalTimes = 1, maxTimeMS = 5 * 60 * 1000) {
+                // Can only be called once because this function does not keep track of the
+                // number of times the fail point is entered between the time it returns
+                // and the next time it gets called.
+                assert.commandWorked(conn.adminCommand({
+                    waitForFailPoint: failPointName,
+                    timesEntered: this.timesEntered + additionalTimes,
+                    maxTimeMS: maxTimeMS
+                }));
+            },
+        off:
+            function() {
+                assert.commandWorked(
+                    conn.adminCommand({configureFailPoint: failPointName, mode: "off"}));
+            }
+    };
+};
+})();
diff --git a/jstests/noPassthrough/drop_view_does_not_take_database_X.js b/jstests/noPassthrough/drop_view_does_not_take_database_X.js
index 69cafb65f58..ae40c0e0f6b 100644
--- a/jstests/noPassthrough/drop_view_does_not_take_database_X.js
+++ b/jstests/noPassthrough/drop_view_does_not_take_database_X.js
@@ -6,7 +6,8 @@
 
 (function() {
 "use strict";
-load("jstests/libs/check_log.js");
+
+load("jstests/libs/fail_point_util.js");
 
 const conn = MongoRunner.runMongod({});
 const db = conn.getDB("test");
@@ -14,19 +15,17 @@ const db = conn.getDB("test");
 assert.commandWorked(db.runCommand({insert: "a", documents: [{x: 1}]}));
 assert.commandWorked(db.createView("view", "a", []));
 
-assert.commandWorked(
-    db.adminCommand({configureFailPoint: "hangDuringDropCollection", mode: "alwaysOn"}));
+const failPoint = configureFailPoint(db, "hangDuringDropCollection");
 
 // This only holds a database IX lock.
 const awaitDrop =
     startParallelShell(() => assert(db.getSiblingDB("test")["view"].drop()), conn.port);
-checkLog.contains(conn, "hangDuringDropCollection fail point enabled");
+failPoint.wait();
 
 // This takes a database IX lock and should not be blocked.
 assert.commandWorked(db.runCommand({insert: "a", documents: [{y: 1}]}));
 
-assert.commandWorked(
-    db.adminCommand({configureFailPoint: "hangDuringDropCollection", mode: "off"}));
+failPoint.off();
 
 awaitDrop();
 MongoRunner.stopMongod(conn);
diff --git a/jstests/sharding/database_and_shard_versioning_all_commands.js b/jstests/sharding/database_and_shard_versioning_all_commands.js
index f19db677af5..f319ee7787d 100644
--- a/jstests/sharding/database_and_shard_versioning_all_commands.js
+++ b/jstests/sharding/database_and_shard_versioning_all_commands.js
@@ -710,6 +710,7 @@ let testCases = {
             },
         }
     },
+    waitForFailPoint: {skip: "executes locally on mongos (not sent to any remote node)"},
     whatsmyuri: {skip: "executes locally on mongos (not sent to any remote node)"},
 };
 
diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js
index 8c52deab905..aa08f7a6732 100644
--- a/jstests/sharding/safe_secondary_reads_drop_recreate.js
+++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js
@@ -303,6 +303,7 @@ let testCases = {
     updateZoneKeyRange: {skip: "primary only"},
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
+    waitForFailPoint: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}
 };
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
index 6013d8a6f7e..aa47096339a 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
@@ -338,6 +338,7 @@ let testCases = {
     updateZoneKeyRange: {skip: "primary only"},
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
+    waitForFailPoint: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}
 };
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
index 4853743699a..efa99b0c6ad 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
@@ -309,6 +309,7 @@ let testCases = {
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
+    waitForFailPoint: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}
 };
 
diff --git a/jstests/sharding/track_unsharded_collections_check_shard_version.js b/jstests/sharding/track_unsharded_collections_check_shard_version.js
index 9d7658c4449..27d2a19b84d 100644
--- a/jstests/sharding/track_unsharded_collections_check_shard_version.js
+++ b/jstests/sharding/track_unsharded_collections_check_shard_version.js
@@ -336,6 +336,7 @@ let testCases = {
             return {validate: collName};
         },
     },
+    waitForFailPoint: {skip: "executes locally on mongos (not sent to any remote node)"},
     whatsmyuri: {skip: "executes locally on mongos (not sent to any remote node)"},
 };
 
diff --git a/src/mongo/db/commands/fail_point_cmd.cpp b/src/mongo/db/commands/fail_point_cmd.cpp
index 4eb0d3e3a02..ffc7783b52d 100644
--- a/src/mongo/db/commands/fail_point_cmd.cpp
+++ b/src/mongo/db/commands/fail_point_cmd.cpp
@@ -37,6 +37,7 @@
 #include "mongo/db/auth/privilege.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/commands/test_commands_enabled.h"
+#include "mongo/s/request_types/wait_for_fail_point_gen.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/log.h"
 
@@ -102,10 +103,62 @@ public:
              const BSONObj& cmdObj,
              BSONObjBuilder& result) override {
         const std::string failPointName(cmdObj.firstElement().str());
-        setGlobalFailPoint(failPointName, cmdObj);
-
+        const auto timesEntered = setGlobalFailPoint(failPointName, cmdObj);
+        result.appendIntOrLL("count", timesEntered);
         return true;
     }
 };
+
+/**
+ * Command for waiting for installed fail points.
+ */
+class WaitForFailPointCommand : public TypedCommand<WaitForFailPointCommand> {
+public:
+    using Request = WaitForFailPoint;
+    class Invocation final : public InvocationBase {
+    public:
+        using InvocationBase::InvocationBase;
+
+        void typedRun(OperationContext* opCtx) {
+            const std::string failPointName = request().getCommandParameter().toString();
+            FailPoint* failPoint = globalFailPointRegistry().find(failPointName);
+            if (failPoint == nullptr)
+                uasserted(ErrorCodes::FailPointSetFailed, failPointName + " not found");
+            failPoint->waitForTimesEntered(opCtx, request().getTimesEntered());
+        }
+
+    private:
+        bool supportsWriteConcern() const override {
+            return false;
+        }
+
+        // The command parameter happens to be string so it's historically been interpreted
+        // by parseNs as a collection. Continuing to do so here for unexamined compatibility.
+        NamespaceString ns() const override {
+            return NamespaceString(request().getDbName(), "");
+        }
+
+        // No auth needed because it only works when enabled via command line.
+        void doCheckAuthorization(OperationContext* opCtx) const override {}
+    };
+
+    std::string help() const override {
+        return "wait for a fail point to be entered a certain number of times";
+    }
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kAlways;
+    }
+
+    bool adminOnly() const override {
+        return true;
+    }
+
+    bool requiresAuth() const override {
+        return false;
+    }
+
+} WaitForFailPointCmd;
+
 MONGO_REGISTER_TEST_COMMAND(FaultInjectCmd);
 }  // namespace mongo
diff --git a/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp b/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp
index 302f646fe14..96821e99e27 100644
--- a/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp
+++ b/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp
@@ -239,7 +239,7 @@ TEST_F(MongodbCAPITest, CreateIndex) {
     mongo::BSONObj inputObj = mongo::fromjson(
         R"raw_delimiter({
             createIndexes: 'items',
-            indexes: 
+            indexes:
             [
                 {
                     key: {
@@ -266,7 +266,7 @@ TEST_F(MongodbCAPITest, CreateBackgroundIndex) {
     mongo::BSONObj inputObj = mongo::fromjson(
         R"raw_delimiter({
             createIndexes: 'items',
-            indexes: 
+            indexes:
             [
                 {
                     key: {
@@ -292,7 +292,7 @@ TEST_F(MongodbCAPITest, CreateTTLIndex) {
     mongo::BSONObj inputObj = mongo::fromjson(
         R"raw_delimiter({
             createIndexes: 'items',
-            indexes: 
+            indexes:
             [
                 {
                     key: {
@@ -621,6 +621,7 @@ TEST_F(MongodbCAPITest, RunListCommands) {
                                           "trimMemory",
                                           "update",
                                           "validate",
+                                          "waitForFailPoint",
                                           "whatsmysni"};
 
     std::sort(whitelist.begin(), whitelist.end());
diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index a2198c003be..921c2a52b7a 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -188,7 +188,8 @@ env.Library(
         env.Idlc('request_types/shard_collection.idl')[0],
         env.Idlc('request_types/clone_collection_options_from_primary_shard.idl')[0],
         env.Idlc('request_types/refine_collection_shard_key.idl')[0],
-        env.Idlc('request_types/rename_collection.idl')[0]
+        env.Idlc('request_types/rename_collection.idl')[0],
+        env.Idlc('request_types/wait_for_fail_point.idl')[0],
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/client/connection_string',
diff --git a/src/mongo/s/request_types/wait_for_fail_point.idl b/src/mongo/s/request_types/wait_for_fail_point.idl
new file mode 100644
index 00000000000..182f391b9cf
--- /dev/null
+++ b/src/mongo/s/request_types/wait_for_fail_point.idl
@@ -0,0 +1,47 @@
+# Copyright (C) 2019-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+# waitForFailPoint IDL file
+
+global:
+    cpp_namespace: "mongo"
+
+imports:
+    - "mongo/idl/basic_types.idl"
+
+commands:
+    waitForFailPoint:
+        description: "wait for a fail point to be entered a certain number of times"
+        strict: false
+        namespace: type
+        type: string
+        fields:
+            timesEntered:
+                type: safeInt64
+                description: "The number of times the fail point has been entered."
+                optional: false
diff --git a/src/mongo/util/fail_point.cpp b/src/mongo/util/fail_point.cpp
index af757e2bd49..c353780e57b 100644
--- a/src/mongo/util/fail_point.cpp
+++ b/src/mongo/util/fail_point.cpp
@@ -76,7 +76,7 @@ void FailPoint::_shouldFailCloseBlock() {
     _fpInfo.subtractAndFetch(1);
 }
 
-void FailPoint::setMode(Mode mode, ValType val, BSONObj extra) {
+int64_t FailPoint::setMode(Mode mode, ValType val, BSONObj extra) {
     /**
      * Outline:
      *
@@ -103,6 +103,20 @@ void FailPoint::setMode(Mode mode, ValType val, BSONObj extra) {
     if (_mode != off) {
         _enable();
     }
+
+    return _timesEntered.load();
+}
+
+void FailPoint::waitForTimesEntered(int64_t timesEntered) {
+    while (_timesEntered.load() < timesEntered) {
+        sleepmillis(100);
+    };
+}
+
+void FailPoint::waitForTimesEntered(OperationContext* opCtx, int64_t timesEntered) {
+    while (_timesEntered.load() < timesEntered) {
+        opCtx->sleepFor(Milliseconds(100));
+    }
 }
 
 const BSONObj& FailPoint::_getData() const {
@@ -117,7 +131,7 @@ void FailPoint::_disable() {
     _fpInfo.fetchAndBitAnd(~kActiveBit);
 }
 
-FailPoint::RetCode FailPoint::_slowShouldFailOpenBlock(
+FailPoint::RetCode FailPoint::_slowShouldFailOpenBlockImpl(
     std::function<bool(const BSONObj&)> cb) noexcept {
     ValType localFpInfo = _fpInfo.addAndFetch(1);
 
@@ -142,15 +156,15 @@ FailPoint::RetCode FailPoint::_slowShouldFailOpenBlock(
         case nTimes: {
             if (_timesOrPeriod.subtractAndFetch(1) <= 0)
                 _disable();
-
             return slowOn;
         }
         case skip: {
             // Ensure that once the skip counter reaches within some delta from 0 we don't continue
             // decrementing it unboundedly because at some point it will roll over and become
             // positive again
-            if (_timesOrPeriod.load() <= 0 || _timesOrPeriod.subtractAndFetch(1) < 0)
+            if (_timesOrPeriod.load() <= 0 || _timesOrPeriod.subtractAndFetch(1) < 0) {
                 return slowOn;
+            }
 
             return slowOff;
         }
@@ -160,6 +174,15 @@ FailPoint::RetCode FailPoint::_slowShouldFailOpenBlock(
     }
 }
 
+FailPoint::RetCode FailPoint::_slowShouldFailOpenBlock(
+    std::function<bool(const BSONObj&)> cb) noexcept {
+    auto ret = _slowShouldFailOpenBlockImpl(cb);
+    if (ret == slowOn) {
+        _timesEntered.addAndFetch(1);
+    }
+    return ret;
+}
+
 StatusWith<FailPoint::ModeOptions> FailPoint::parseBSON(const BSONObj& obj) {
     Mode mode = FailPoint::alwaysOn;
     ValType val = 0;
@@ -254,6 +277,7 @@ BSONObj FailPoint::toBSON() const {
     stdx::lock_guard<Latch> scoped(_modMutex);
     builder.append("mode", _mode);
     builder.append("data", _data);
+    builder.append("timesEntered", _timesEntered.load());
 
     return builder.obj();
 }
@@ -267,12 +291,13 @@ FailPointRegistry& globalFailPointRegistry() {
     return p;
 }
 
-void setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj) {
+int64_t setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj) {
     FailPoint* failPoint = globalFailPointRegistry().find(failPointName);
     if (failPoint == nullptr)
         uasserted(ErrorCodes::FailPointSetFailed, failPointName + " not found");
-    failPoint->setMode(uassertStatusOK(FailPoint::parseBSON(cmdObj)));
+    auto timesEntered = failPoint->setMode(uassertStatusOK(FailPoint::parseBSON(cmdObj)));
     warning() << "failpoint: " << failPointName << " set to: " << failPoint->toBSON();
+    return timesEntered;
 }
 
 FailPointEnableBlock::FailPointEnableBlock(std::string failPointName)
diff --git a/src/mongo/util/fail_point.h b/src/mongo/util/fail_point.h
index c5c12e62c23..b14693d0db1 100644
--- a/src/mongo/util/fail_point.h
+++ b/src/mongo/util/fail_point.h
@@ -244,13 +244,29 @@ public:
      * @param extra arbitrary BSON object that can be stored to this fail point
      *     that can be referenced afterwards with #getData. Defaults to an empty
      *     document.
+     *
+     * @returns the number of times the fail point has been entered so far.
      */
-    void setMode(Mode mode, ValType val = 0, BSONObj extra = {});
-    void setMode(ModeOptions opt) {
-        setMode(std::move(opt.mode), std::move(opt.val), std::move(opt.extra));
+    int64_t setMode(Mode mode, ValType val = 0, BSONObj extra = {});
+    int64_t setMode(ModeOptions opt) {
+        return setMode(std::move(opt.mode), std::move(opt.val), std::move(opt.extra));
     }
 
     /**
+     * Waits until the fail point has been entered the desired number of times.
+     *
+     * @param timesEntered the number of times the fail point has been entered.
+     */
+    void waitForTimesEntered(int64_t timesEntered);
+
+    /**
+     * Like `waitForTimesEntered`, but interruptible via the `opCtx->sleepFor` mechanism.  See
+     * `mongo::Interruptible::sleepFor` (Interruptible is a base class of
+     * OperationContext).
+     */
+    void waitForTimesEntered(OperationContext* opCtx, int64_t timesEntered);
+
+    /**
      * @returns a BSON object showing the current mode and data stored.
      */
     BSONObj toBSON() const;
@@ -359,6 +375,14 @@ private:
      * If a callable is passed, and returns false, this will return userIgnored and avoid altering
      * the mode in any way.  The argument is the fail point payload.
      */
+    RetCode _slowShouldFailOpenBlockImpl(std::function<bool(const BSONObj&)> cb) noexcept;
+
+    /**
+     * slow path for #_shouldFailOpenBlock
+     *
+     * Calls _slowShouldFailOpenBlockImpl. If it returns slowOn, increments the number of times
+     * the fail point has been entered before returning the RetCode.
+     */
     RetCode _slowShouldFailOpenBlock(std::function<bool(const BSONObj&)> cb) noexcept;
 
     /**
@@ -374,6 +398,9 @@ private:
     // 0~30: unsigned ref counter for active dynamic instances.
     AtomicWord<std::uint32_t> _fpInfo{0};
 
+    // Total number of times the fail point has been entered.
+    AtomicWord<int64_t> _timesEntered{0};
+
     // Invariant: These should be read only if kActiveBit of _fpInfo is set.
     Mode _mode{off};
     AtomicWord<int> _timesOrPeriod{0};
@@ -435,10 +462,11 @@ private:
 
 /**
  * Set a fail point in the global registry to a given value via BSON
+ * @return the number of times the fail point has been entered so far.
  * @throw DBException corresponding to ErrorCodes::FailPointSetFailed if no failpoint
  * called failPointName exists.
  */
-void setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj);
+int64_t setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj);
 
 /**
  * Registration object for FailPoint. Its static-initializer registers FailPoint `fp`
diff --git a/src/mongo/util/fail_point_test.cpp b/src/mongo/util/fail_point_test.cpp
index af15040e0b3..efc7d56346e 100644
--- a/src/mongo/util/fail_point_test.cpp
+++ b/src/mongo/util/fail_point_test.cpp
@@ -36,11 +36,14 @@
 #include <string>
 #include <vector>
 
+#include "mongo/db/client.h"
 #include "mongo/platform/mutex.h"
 #include "mongo/stdx/thread.h"
 #include "mongo/unittest/unittest.h"
+#include "mongo/util/clock_source_mock.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/log.h"
+#include "mongo/util/tick_source_mock.h"
 #include "mongo/util/time_support.h"
 
 using mongo::BSONObj;
@@ -426,3 +429,33 @@ TEST(FailPoint, FailPointBlockIfBasicTest) {
     failPoint.executeIf([](auto&&) { ASSERT(!"shouldn't get here"); }, [](auto&&) { return true; });
 }
 }  // namespace mongo_test
+
+namespace mongo {
+
+TEST(FailPoint, WaitForFailPointTimeout) {
+    FailPoint failPoint;
+    failPoint.setMode(FailPoint::alwaysOn);
+
+    const auto service = ServiceContext::make();
+    const std::shared_ptr<ClockSourceMock> mockClock = std::make_shared<ClockSourceMock>();
+    service->setFastClockSource(std::make_unique<SharedClockSourceAdapter>(mockClock));
+    service->setPreciseClockSource(std::make_unique<SharedClockSourceAdapter>(mockClock));
+    service->setTickSource(std::make_unique<TickSourceMock<>>());
+
+    const auto client = service->makeClient("WaitForFailPointTest");
+    auto opCtx = client->makeOperationContext();
+    opCtx->setDeadlineAfterNowBy(Milliseconds{999}, ErrorCodes::ExceededTimeLimit);
+
+    stdx::thread waitForFailPoint([&] {
+        ASSERT_THROWS_CODE(failPoint.waitForTimesEntered(opCtx.get(), 1),
+                           AssertionException,
+                           ErrorCodes::ExceededTimeLimit);
+    });
+
+    mockClock->advance(Milliseconds{1000});
+    waitForFailPoint.join();
+
+    failPoint.setMode(FailPoint::off);
+}
+
+}  // namespace mongo