SERVER-39165 Add waitForFailpoint command and update non-repl tests

(cherry picked from commit f79498e9aa27765fc363fcb85a65f47af6e20ee1)

20 files changed, 274 insertions, 33 deletions

diff --git a/jstests/core/txns/kill_op_on_txn_expiry.js b/jstests/core/txns/kill_op_on_txn_expiry.js
index dde4930bfae..b725f4a6b75 100644
--- a/jstests/core/txns/kill_op_on_txn_expiry.js
+++ b/jstests/core/txns/kill_op_on_txn_expiry.js
@@ -3,8 +3,9 @@
 (function() {
 "use strict";
 
-load('jstests/libs/parallelTester.js');
 load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
+load('jstests/libs/parallelTester.js');
 
 const dbName = "test";
 const collName = "kill_op_on_txn_expiry";
@@ -46,10 +47,8 @@ try {
     }));
 
     jsTestLog("Enabling fail point to block batch inserts");
-    assert.commandWorked(
-        testDB.adminCommand({configureFailPoint: "hangDuringBatchInsert", mode: "alwaysOn"}));
-    // Clear ramlog so checkLog can't find log messages from previous times this fail point was
-    // enabled.
+    let failPoint = configureFailPoint(testDB, "hangDuringBatchInsert");
+    // Clear ramlog so checkLog can't find log messages from the previous times this test was run.
     assert.commandWorked(testDB.adminCommand({clearLog: 'global'}));
 
     jsTestLog("Starting insert operation in parallel thread");
@@ -70,15 +69,14 @@ try {
     workerThread.start();
 
     jsTestLog("Wait for insert to be blocked");
-    checkLog.contains(db.getMongo(), "hangDuringBatchInsert fail point enabled");
+    failPoint.wait();
 
     jsTestLog("Wait for the transaction to expire");
     checkLog.contains(db.getMongo(), "Aborting transaction with txnNumber " + txnNumber);
 
     jsTestLog("Disabling fail point to enable insert to proceed and detect that the session " +
               "has been killed");
-    assert.commandWorked(
-        testDB.adminCommand({configureFailPoint: "hangDuringBatchInsert", mode: "off"}));
+    failPoint.off();
 
     workerThread.join();
     assert(!workerThread.hasFailed());
diff --git a/jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js b/jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js
index a3334f6a7bc..8abe4c30a1e 100644
--- a/jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js
+++ b/jstests/core/txns/timestamped_reads_wait_for_prepare_oplog_visibility.js
@@ -6,8 +6,9 @@
  */
 (function() {
 'use strict';
-load("jstests/libs/check_log.js");
+
 load('jstests/core/txns/libs/prepare_helpers.js');
+load("jstests/libs/fail_point_util.js");
 load('jstests/libs/parallel_shell_helpers.js');
 
 TestData.dbName = 'test';
@@ -36,11 +37,12 @@ TestData.otherDocFilter = {
  * field. This function is run in a separate thread and tests that oplog visibility blocks
  * certain reads and that prepare conflicts block other types of reads.
  */
-const readThreadFunc = function(readFunc, _collName) {
+const readThreadFunc = function(readFunc, _collName, timesEntered) {
     load("jstests/libs/check_log.js");
 
     // Do not start reads until we are blocked in 'prepareTransaction'.
-    checkLog.contains(db.getMongo(), "hangAfterReservingPrepareTimestamp fail point enabled");
+    assert.commandWorked(db.adminCommand(
+        {waitForFailPoint: "hangAfterReservingPrepareTimestamp", timesEntered: timesEntered}));
 
     // Create a 'readFuncObj' from the 'readFunc'.
     const readFuncObj = readFunc(_collName);
@@ -65,8 +67,7 @@ function runTest(prefix, readFunc) {
     testColl.drop({writeConcern: {w: "majority"}});
     assert.commandWorked(testDB.runCommand({create: collName, writeConcern: {w: 'majority'}}));
 
-    assert.commandWorked(testDB.adminCommand(
-        {configureFailPoint: 'hangAfterReservingPrepareTimestamp', mode: 'alwaysOn'}));
+    let failPoint = configureFailPoint(testDB, "hangAfterReservingPrepareTimestamp");
 
     // Insert a document for the transaction.
     assert.commandWorked(testColl.insert(TestData.txnDoc));
@@ -89,7 +90,8 @@ function runTest(prefix, readFunc) {
     // Clear the log history to ensure we only see the most recent 'prepareTransaction'
     // failpoint log message.
     assert.commandWorked(db.adminCommand({clearLog: 'global'}));
-    const joinReadThread = startParallelShell(funWithArgs(readThreadFunc, readFunc, collName));
+    const joinReadThread = startParallelShell(
+        funWithArgs(readThreadFunc, readFunc, collName, failPoint.timesEntered + 1));
 
     jsTestLog("Preparing the transaction for " + prefix);
     const prepareTimestamp = PrepareHelpers.prepareTransaction(session);
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index 48374ac51ac..d753b9eaa43 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -528,6 +528,7 @@ let viewsCommandTests = {
     voteCommitIndexBuild: {skip: isUnrelated},
     voteCommitTransaction: {skip: isUnrelated},
     voteAbortTransaction: {skip: isUnrelated},
+    waitForFailPoint: {skip: isUnrelated},
     whatsmyuri: {skip: isUnrelated}
 };
 
diff --git a/jstests/fail_point/fail_point.js b/jstests/fail_point/fail_point.js
index 6cd53fe5ad1..3355e07d435 100644
--- a/jstests/fail_point/fail_point.js
+++ b/jstests/fail_point/fail_point.js
@@ -79,6 +79,16 @@ function runTest(adminDB) {
     res = adminDB.runCommand({getParameter: 1, "failpoint.dummy": 1});
     assert.commandWorked(res);
     expectFailPointState(res["failpoint.dummy"], 1, {x: 1});
+
+    // Test that the timeout for waitForFailPoint can be set via maxTimeMS.
+    var configureFailPointRes = adminDB.runCommand({configureFailPoint: 'dummy', mode: 'alwaysOn'});
+    assert.commandWorked(configureFailPointRes);
+    assert.commandFailedWithCode(adminDB.adminCommand({
+        waitForFailPoint: "dummy",
+        timesEntered: configureFailPointRes.count + 1,
+        maxTimeMS: 10
+    }),
+                                 ErrorCodes.MaxTimeMSExpired);
 }
 
 var conn = MongoRunner.runMongod();
diff --git a/jstests/libs/fail_point_util.js b/jstests/libs/fail_point_util.js
new file mode 100644
index 00000000000..6016fd88a78
--- /dev/null
+++ b/jstests/libs/fail_point_util.js
@@ -0,0 +1,40 @@
+/**
+ * Utilities for turning on/off and waiting for fail points.
+ */
+
+var configureFailPoint;
+
+(function() {
+"use strict";
+
+if (configureFailPoint) {
+    return;  // Protect against this file being double-loaded.
+}
+
+configureFailPoint = function(conn, failPointName, data = {}, failPointMode = "alwaysOn") {
+    return {
+        conn: conn,
+        failPointName: failPointName,
+        timesEntered: assert
+                          .commandWorked(conn.adminCommand(
+                              {configureFailPoint: failPointName, mode: failPointMode, data: data}))
+                          .count,
+        wait:
+            function(additionalTimes = 1, maxTimeMS = 5 * 60 * 1000) {
+                // Can only be called once because this function does not keep track of the
+                // number of times the fail point is entered between the time it returns
+                // and the next time it gets called.
+                assert.commandWorked(conn.adminCommand({
+                    waitForFailPoint: failPointName,
+                    timesEntered: this.timesEntered + additionalTimes,
+                    maxTimeMS: maxTimeMS
+                }));
+            },
+        off:
+            function() {
+                assert.commandWorked(
+                    conn.adminCommand({configureFailPoint: failPointName, mode: "off"}));
+            }
+    };
+};
+})();
diff --git a/jstests/noPassthrough/drop_view_does_not_take_database_X.js b/jstests/noPassthrough/drop_view_does_not_take_database_X.js
index 69cafb65f58..ae40c0e0f6b 100644
--- a/jstests/noPassthrough/drop_view_does_not_take_database_X.js
+++ b/jstests/noPassthrough/drop_view_does_not_take_database_X.js
@@ -6,7 +6,8 @@
 
 (function() {
 "use strict";
-load("jstests/libs/check_log.js");
+
+load("jstests/libs/fail_point_util.js");
 
 const conn = MongoRunner.runMongod({});
 const db = conn.getDB("test");
@@ -14,19 +15,17 @@ const db = conn.getDB("test");
 assert.commandWorked(db.runCommand({insert: "a", documents: [{x: 1}]}));
 assert.commandWorked(db.createView("view", "a", []));
 
-assert.commandWorked(
-    db.adminCommand({configureFailPoint: "hangDuringDropCollection", mode: "alwaysOn"}));
+const failPoint = configureFailPoint(db, "hangDuringDropCollection");
 
 // This only holds a database IX lock.
 const awaitDrop =
     startParallelShell(() => assert(db.getSiblingDB("test")["view"].drop()), conn.port);
-checkLog.contains(conn, "hangDuringDropCollection fail point enabled");
+failPoint.wait();
 
 // This takes a database IX lock and should not be blocked.
 assert.commandWorked(db.runCommand({insert: "a", documents: [{y: 1}]}));
 
-assert.commandWorked(
-    db.adminCommand({configureFailPoint: "hangDuringDropCollection", mode: "off"}));
+failPoint.off();
 
 awaitDrop();
 MongoRunner.stopMongod(conn);
diff --git a/jstests/sharding/database_and_shard_versioning_all_commands.js b/jstests/sharding/database_and_shard_versioning_all_commands.js
index a0fe0fa5da6..8a5d4ce6db1 100644
--- a/jstests/sharding/database_and_shard_versioning_all_commands.js
+++ b/jstests/sharding/database_and_shard_versioning_all_commands.js
@@ -434,6 +434,7 @@ let testCases = {
             assert(mongosConn.getDB(dbName).getCollection(collName).drop());
         },
     },
+    waitForFailPoint: {skip: "executes locally on mongos (not sent to any remote node)"},
     whatsmyuri: {skip: "executes locally on mongos (not sent to any remote node)"},
 };
 
diff --git a/jstests/sharding/libs/last_stable_mongos_commands.js b/jstests/sharding/libs/last_stable_mongos_commands.js
index e9470a1794a..67011fa9f13 100644
--- a/jstests/sharding/libs/last_stable_mongos_commands.js
+++ b/jstests/sharding/libs/last_stable_mongos_commands.js
@@ -22,4 +22,5 @@ const commandsAddedToMongosIn42 = [
     'setIndexCommitQuorum',
     'startRecordingTraffic',
     'stopRecordingTraffic',
+    'waitForFailPoint',
 ];
diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js
index a507f876fce..bbf34b5a2a3 100644
--- a/jstests/sharding/safe_secondary_reads_drop_recreate.js
+++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js
@@ -301,6 +301,7 @@ let testCases = {
     updateZoneKeyRange: {skip: "primary only"},
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
+    waitForFailPoint: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}
 };
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
index 234f0873076..363c4cc06ca 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
@@ -336,6 +336,7 @@ let testCases = {
     updateZoneKeyRange: {skip: "primary only"},
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
+    waitForFailPoint: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}
 };
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
index 866080326d2..7b3a5be24f8 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
@@ -307,6 +307,7 @@ let testCases = {
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
+    waitForFailPoint: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}
 };
 
diff --git a/src/mongo/db/commands/fail_point_cmd.cpp b/src/mongo/db/commands/fail_point_cmd.cpp
index a50cc4ff06d..51c6385e003 100644
--- a/src/mongo/db/commands/fail_point_cmd.cpp
+++ b/src/mongo/db/commands/fail_point_cmd.cpp
@@ -37,6 +37,7 @@
 #include "mongo/db/auth/privilege.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/commands/test_commands_enabled.h"
+#include "mongo/s/request_types/wait_for_fail_point_gen.h"
 #include "mongo/util/fail_point_service.h"
 #include "mongo/util/log.h"
 
@@ -97,10 +98,62 @@ public:
              const BSONObj& cmdObj,
              BSONObjBuilder& result) override {
         const std::string failPointName(cmdObj.firstElement().str());
-        setGlobalFailPoint(failPointName, cmdObj);
-
+        const auto timesEntered = setGlobalFailPoint(failPointName, cmdObj);
+        result.appendIntOrLL("count", timesEntered);
         return true;
     }
 };
+
+/**
+ * Command for waiting for installed fail points.
+ */
+class WaitForFailPointCommand : public TypedCommand<WaitForFailPointCommand> {
+public:
+    using Request = WaitForFailPoint;
+    class Invocation final : public InvocationBase {
+    public:
+        using InvocationBase::InvocationBase;
+
+        void typedRun(OperationContext* opCtx) {
+            const std::string failPointName = request().getCommandParameter().toString();
+            FailPoint* failPoint = getGlobalFailPointRegistry()->getFailPoint(failPointName);
+            if (failPoint == nullptr)
+                uasserted(ErrorCodes::FailPointSetFailed, failPointName + " not found");
+            failPoint->waitForTimesEntered(opCtx, request().getTimesEntered());
+        }
+
+    private:
+        bool supportsWriteConcern() const override {
+            return false;
+        }
+
+        // The command parameter happens to be string so it's historically been interpreted
+        // by parseNs as a collection. Continuing to do so here for unexamined compatibility.
+        NamespaceString ns() const override {
+            return NamespaceString(request().getDbName(), "");
+        }
+
+        // No auth needed because it only works when enabled via command line.
+        void doCheckAuthorization(OperationContext* opCtx) const override {}
+    };
+
+    std::string help() const override {
+        return "wait for a fail point to be entered a certain number of times";
+    }
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kAlways;
+    }
+
+    bool adminOnly() const override {
+        return true;
+    }
+
+    bool requiresAuth() const override {
+        return false;
+    }
+
+} WaitForFailPointCmd;
+
 MONGO_REGISTER_TEST_COMMAND(FaultInjectCmd);
 }  // namespace mongo
diff --git a/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp b/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp
index 1cde189eb41..acf4af16380 100644
--- a/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp
+++ b/src/mongo/embedded/mongo_embedded/mongo_embedded_test.cpp
@@ -238,7 +238,7 @@ TEST_F(MongodbCAPITest, CreateIndex) {
     mongo::BSONObj inputObj = mongo::fromjson(
         R"raw_delimiter({
             createIndexes: 'items',
-            indexes: 
+            indexes:
             [
                 {
                     key: {
@@ -265,7 +265,7 @@ TEST_F(MongodbCAPITest, CreateBackgroundIndex) {
     mongo::BSONObj inputObj = mongo::fromjson(
         R"raw_delimiter({
             createIndexes: 'items',
-            indexes: 
+            indexes:
             [
                 {
                     key: {
@@ -291,7 +291,7 @@ TEST_F(MongodbCAPITest, CreateTTLIndex) {
     mongo::BSONObj inputObj = mongo::fromjson(
         R"raw_delimiter({
             createIndexes: 'items',
-            indexes: 
+            indexes:
             [
                 {
                     key: {
@@ -560,7 +560,6 @@ TEST_F(MongodbCAPITest, InsertAndUpdate) {
 
 TEST_F(MongodbCAPITest, RunListCommands) {
     auto client = createClient();
-
     std::vector<std::string> whitelist = {
         "_hashBSONElement",
         "aggregate",
@@ -623,6 +622,7 @@ TEST_F(MongodbCAPITest, RunListCommands) {
         "twoPhaseCreateIndexes",
         "update",
         "validate",
+        "waitForFailPoint",
     };
     std::sort(whitelist.begin(), whitelist.end());
 
diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index 9c5072d1e2e..1854d61023c 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -185,6 +185,7 @@ env.Library(
         env.Idlc('request_types/move_primary.idl')[0],
         env.Idlc('request_types/shard_collection.idl')[0],
         env.Idlc('request_types/clone_collection_options_from_primary_shard.idl')[0],
+        env.Idlc('request_types/wait_for_fail_point.idl')[0],
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/client/connection_string',
diff --git a/src/mongo/s/request_types/wait_for_fail_point.idl b/src/mongo/s/request_types/wait_for_fail_point.idl
new file mode 100644
index 00000000000..182f391b9cf
--- /dev/null
+++ b/src/mongo/s/request_types/wait_for_fail_point.idl
@@ -0,0 +1,47 @@
+# Copyright (C) 2019-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+# waitForFailPoint IDL file
+
+global:
+    cpp_namespace: "mongo"
+
+imports:
+    - "mongo/idl/basic_types.idl"
+
+commands:
+    waitForFailPoint:
+        description: "wait for a fail point to be entered a certain number of times"
+        strict: false
+        namespace: type
+        type: string
+        fields:
+            timesEntered:
+                type: safeInt64
+                description: "The number of times the fail point has been entered."
+                optional: false
diff --git a/src/mongo/util/fail_point.cpp b/src/mongo/util/fail_point.cpp
index d384493d5e0..2e76454032f 100644
--- a/src/mongo/util/fail_point.cpp
+++ b/src/mongo/util/fail_point.cpp
@@ -83,7 +83,7 @@ void FailPoint::shouldFailCloseBlock() {
     _fpInfo.subtractAndFetch(1);
 }
 
-void FailPoint::setMode(Mode mode, ValType val, const BSONObj& extra) {
+int64_t FailPoint::setMode(Mode mode, ValType val, const BSONObj& extra) {
     /**
      * Outline:
      *
@@ -110,6 +110,20 @@ void FailPoint::setMode(Mode mode, ValType val, const BSONObj& extra) {
     if (_mode != off) {
         enableFailPoint();
     }
+
+    return _timesEntered.load();
+}
+
+void FailPoint::waitForTimesEntered(int64_t timesEntered) {
+    while (_timesEntered.load() < timesEntered) {
+        sleepmillis(100);
+    };
+}
+
+void FailPoint::waitForTimesEntered(OperationContext* opCtx, int64_t timesEntered) {
+    while (_timesEntered.load() < timesEntered) {
+        opCtx->sleepFor(Milliseconds(100));
+    }
 }
 
 const BSONObj& FailPoint::getData() const {
@@ -124,7 +138,7 @@ void FailPoint::disableFailPoint() {
     _fpInfo.fetchAndBitAnd(~ACTIVE_BIT);
 }
 
-FailPoint::RetCode FailPoint::slowShouldFailOpenBlock(
+FailPoint::RetCode FailPoint::slowShouldFailOpenBlockImpl(
     stdx::function<bool(const BSONObj&)> cb) noexcept {
     ValType localFpInfo = _fpInfo.addAndFetch(1);
 
@@ -167,6 +181,15 @@ FailPoint::RetCode FailPoint::slowShouldFailOpenBlock(
     }
 }
 
+FailPoint::RetCode FailPoint::slowShouldFailOpenBlock(
+    stdx::function<bool(const BSONObj&)> cb) noexcept {
+    auto ret = slowShouldFailOpenBlockImpl(cb);
+    if (ret == slowOn) {
+        _timesEntered.addAndFetch(1);
+    }
+    return ret;
+}
+
 StatusWith<std::tuple<FailPoint::Mode, FailPoint::ValType, BSONObj>> FailPoint::parseBSON(
     const BSONObj& obj) {
     Mode mode = FailPoint::alwaysOn;
@@ -263,6 +286,7 @@ BSONObj FailPoint::toBSON() const {
     stdx::lock_guard<stdx::mutex> scoped(_modMutex);
     builder.append("mode", _mode);
     builder.append("data", _data);
+    builder.append("timesEntered", _timesEntered.load());
 
     return builder.obj();
 }
diff --git a/src/mongo/util/fail_point.h b/src/mongo/util/fail_point.h
index 367eeab3632..1e1704b90a8 100644
--- a/src/mongo/util/fail_point.h
+++ b/src/mongo/util/fail_point.h
@@ -155,8 +155,24 @@ public:
      * @param extra arbitrary BSON object that can be stored to this fail point
      *     that can be referenced afterwards with #getData. Defaults to an empty
      *     document.
+     *
+     * @returns the number of times the fail point has been entered so far.
+     */
+    int64_t setMode(Mode mode, ValType val = 0, const BSONObj& extra = BSONObj());
+
+    /**
+     * Waits until the fail point has been entered the desired number of times.
+     *
+     * @param timesEntered the number of times the fail point has been entered.
+     */
+    void waitForTimesEntered(int64_t timesEntered);
+
+    /**
+     * Like `waitForTimesEntered`, but interruptible via the `opCtx->sleepFor` mechanism.  See
+     * `mongo::Interruptible::sleepFor` (Interruptible is a base class of
+     * OperationContext).
      */
-    void setMode(Mode mode, ValType val = 0, const BSONObj& extra = BSONObj());
+    void waitForTimesEntered(OperationContext* opCtx, int64_t timesEntered);
 
     /**
      * @returns a BSON object showing the current mode and data stored.
@@ -172,6 +188,9 @@ private:
     // 0~30: unsigned ref counter for active dynamic instances.
     AtomicWord<unsigned> _fpInfo{0};
 
+    // Total number of times the fail point has been entered.
+    AtomicWord<int64_t> _timesEntered{0};
+
     // Invariant: These should be read only if ACTIVE_BIT of _fpInfo is set.
     Mode _mode{off};
     AtomicWord<int> _timesOrPeriod{0};
@@ -196,7 +215,15 @@ private:
      * If a callable is passed, and returns false, this will return userIgnored and avoid altering
      * the mode in any way.  The argument is the fail point payload.
      */
-    RetCode slowShouldFailOpenBlock(stdx::function<bool(const BSONObj&)> cb) noexcept;
+    RetCode slowShouldFailOpenBlockImpl(std::function<bool(const BSONObj&)> cb) noexcept;
+
+    /**
+     * slow path for #shouldFailOpenBlock
+     *
+     * Calls slowShouldFailOpenBlockImpl. If it returns slowOn, increments the number of times
+     * the fail point has been entered before returning the RetCode.
+     */
+    RetCode slowShouldFailOpenBlock(std::function<bool(const BSONObj&)> cb) noexcept;
 
     /**
      * @return the stored BSONObj in this fail point. Note that this cannot be safely
diff --git a/src/mongo/util/fail_point_service.cpp b/src/mongo/util/fail_point_service.cpp
index 77068f82100..0cf2bab95e1 100644
--- a/src/mongo/util/fail_point_service.cpp
+++ b/src/mongo/util/fail_point_service.cpp
@@ -62,7 +62,7 @@ FailPointRegistry* getGlobalFailPointRegistry() {
     return _fpRegistry.get();
 }
 
-void setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj) {
+int64_t setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj) {
     FailPointRegistry* registry = getGlobalFailPointRegistry();
     FailPoint* failPoint = registry->getFailPoint(failPointName);
 
@@ -74,8 +74,9 @@ void setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj)
     BSONObj data;
     std::tie(mode, val, data) = uassertStatusOK(FailPoint::parseBSON(cmdObj));
 
-    failPoint->setMode(mode, val, data);
+    auto timesEntered = failPoint->setMode(mode, val, data);
     warning() << "failpoint: " << failPointName << " set to: " << failPoint->toBSON();
+    return timesEntered;
 }
 
 FailPointEnableBlock::FailPointEnableBlock(const std::string& failPointName)
diff --git a/src/mongo/util/fail_point_service.h b/src/mongo/util/fail_point_service.h
index 64c511942fc..334a97244e4 100644
--- a/src/mongo/util/fail_point_service.h
+++ b/src/mongo/util/fail_point_service.h
@@ -42,9 +42,10 @@ FailPointRegistry* getGlobalFailPointRegistry();
 
 /**
  * Set a fail point in the global registry to a given value via BSON
+ * @return the number of times the fail point has been entered so far.
  * @throw DBException If no failpoint called failPointName exists.
  */
-void setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj);
+int64_t setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj);
 
 /**
  * Convenience macro for defining a fail point. Must be used at namespace scope.
diff --git a/src/mongo/util/fail_point_test.cpp b/src/mongo/util/fail_point_test.cpp
index d5f9e909f37..8ff5279cd6e 100644
--- a/src/mongo/util/fail_point_test.cpp
+++ b/src/mongo/util/fail_point_test.cpp
@@ -38,9 +38,11 @@
 #include "mongo/stdx/functional.h"
 #include "mongo/stdx/thread.h"
 #include "mongo/unittest/unittest.h"
+#include "mongo/util/clock_source_mock.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/fail_point_service.h"
 #include "mongo/util/log.h"
+#include "mongo/util/tick_source_mock.h"
 #include "mongo/util/time_support.h"
 
 using mongo::BSONObj;
@@ -448,3 +450,33 @@ TEST(FailPoint, FailPointBlockIfBasicTest) {
     }
 }
 }  // namespace mongo_test
+
+namespace mongo {
+
+TEST(FailPoint, WaitForFailPointTimeout) {
+    FailPoint failPoint;
+    failPoint.setMode(FailPoint::alwaysOn);
+
+    const auto service = ServiceContext::make();
+    const std::shared_ptr<ClockSourceMock> mockClock = std::make_shared<ClockSourceMock>();
+    service->setFastClockSource(std::make_unique<SharedClockSourceAdapter>(mockClock));
+    service->setPreciseClockSource(std::make_unique<SharedClockSourceAdapter>(mockClock));
+    service->setTickSource(std::make_unique<TickSourceMock<>>());
+
+    const auto client = service->makeClient("WaitForFailPointTest");
+    auto opCtx = client->makeOperationContext();
+    opCtx->setDeadlineAfterNowBy(Milliseconds{999}, ErrorCodes::ExceededTimeLimit);
+
+    stdx::thread waitForFailPoint([&] {
+        ASSERT_THROWS_CODE(failPoint.waitForTimesEntered(opCtx.get(), 1),
+                           AssertionException,
+                           ErrorCodes::ExceededTimeLimit);
+    });
+
+    mockClock->advance(Milliseconds{1000});
+    waitForFailPoint.join();
+
+    failPoint.setMode(FailPoint::off);
+}
+
+}  // namespace mongo