From 8fc28f0773ca1efb0a43cc5590b9ca8b9e50559e Mon Sep 17 00:00:00 2001
From: Cheahuychou Mao <cheahuychou.mao@mongodb.com>
Date: Fri, 25 Oct 2019 19:08:50 +0000
Subject: SERVER-39165 Add waitForFailpoint command and update other repl tests

---
 .../abort_in_progress_transactions_on_step_up.js    | 21 +++++++--------------
 ...e_stream_speculative_majority_lastApplied_lag.js | 11 +++++------
 ...transaction_initial_sync_data_already_applied.js |  5 ++---
 .../create_drop_database_different_casing.js        | 10 ++++------
 .../disconnect_on_legacy_write_to_secondary.js      |  9 ++++-----
 ...advance_commit_point_beyond_last_applied_term.js | 15 ++++++---------
 .../drop_collections_two_phase_write_concern.js     | 10 ++++------
 .../hang_before_releasing_transaction_oplog_hole.js |  4 ++--
 jstests/replsets/interrupted_batch_insert.js        |  9 ++++-----
 ...reads_with_prepare_conflicts_during_step_down.js |  7 +++----
 ...l_reads_with_prepare_conflicts_during_step_up.js |  6 +++---
 jstests/replsets/kill_ttl_on_stepdown.js            | 10 ++++------
 jstests/replsets/libs/two_phase_drops.js            |  7 +++----
 ...ajority_writes_wait_for_all_durable_timestamp.js | 15 +++++----------
 jstests/replsets/read_committed_stale_history.js    | 15 ++++++---------
 jstests/replsets/reconfig_during_election.js        | 15 +++++----------
 ...econstruct_prepared_transactions_initial_sync.js |  4 ++--
 ...repared_transactions_initial_sync_index_build.js |  4 ++--
 ...pared_transactions_initial_sync_on_oplog_seed.js |  7 ++++---
 jstests/replsets/rollback_remote_cursor_retry.js    | 20 +++++++++-----------
 .../rollback_with_socket_error_then_steady_state.js | 11 +++++------
 jstests/replsets/rslib.js                           |  9 ++++++---
 ...cv_document_succeeds_if_initial_sync_flag_set.js | 12 +++++-------
 jstests/replsets/step_down_during_draining2.js      | 10 ++++------
 jstests/replsets/step_down_on_secondary.js          |  6 +++---
 25 files changed, 107 insertions(+), 145 deletions(-)

(limited to 'jstests/replsets')

diff --git a/jstests/replsets/abort_in_progress_transactions_on_step_up.js b/jstests/replsets/abort_in_progress_transactions_on_step_up.js
index a47baee071a..e233e2ed97e 100644
--- a/jstests/replsets/abort_in_progress_transactions_on_step_up.js
+++ b/jstests/replsets/abort_in_progress_transactions_on_step_up.js
@@ -7,6 +7,7 @@
 "use strict";
 load("jstests/replsets/rslib.js");  // For reconnect()
 load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 function getTxnTableEntry(db) {
     let txnTableEntries = db.getSiblingDB("config")["transactions"].find().toArray();
@@ -46,18 +47,14 @@ testDB.dropDatabase();
 assert.commandWorked(testDB.runCommand({create: collName, writeConcern: {w: "majority"}}));
 
 // Prevent the priority: 0 node from fetching new ops so that it can vote for the new primary.
-assert.commandWorked(
-    replTest.nodes[2].adminCommand({configureFailPoint: 'stopReplProducer', mode: 'alwaysOn'}));
+const stopReplProducerFailPoint = configureFailPoint(replTest.nodes[2], 'stopReplProducer');
 
 jsTest.log("Stop secondary oplog replication before the last operation in the transaction.");
 // The stopReplProducerOnDocument failpoint ensures that secondary stops replicating before
 // applying the last operation in the transaction. This depends on the oplog fetcher batch size
 // being 1.
-assert.commandWorked(newPrimary.adminCommand({
-    configureFailPoint: "stopReplProducerOnDocument",
-    mode: "alwaysOn",
-    data: {document: {"applyOps.o._id": "last in txn"}}
-}));
+const stopReplProducerOnDocumentFailPoint = configureFailPoint(
+    newPrimary, "stopReplProducerOnDocument", {document: {"applyOps.o._id": "last in txn"}});
 
 jsTestLog("Starting transaction");
 const session = primary.startSession({causalConsistency: false});
@@ -80,7 +77,7 @@ assert.eq(txnTableEntry.state, "committed");
 const commitOpTime = getTxnTableEntry(testDB).lastWriteOpTime;
 
 jsTestLog("Wait for the new primary to block on fail point.");
-checkLog.contains(newPrimary, "stopReplProducerOnDocument fail point is enabled.");
+stopReplProducerOnDocumentFailPoint.wait();
 
 // Now the transaction should be in-progress on newPrimary.
 txnTableEntry = getTxnTableEntry(newTestDB);
@@ -98,14 +95,10 @@ jsTestLog("Wait for the new primary to stop replication after primary catch-up."
 checkLog.contains(newPrimary, "Stopping replication producer");
 
 jsTestLog("Enable replication on the new primary so that it can finish state transition");
-assert.commandWorked(newPrimary.adminCommand({
-    configureFailPoint: "stopReplProducerOnDocument",
-    mode: "off",
-}));
+stopReplProducerOnDocumentFailPoint.off();
 
 assert.eq(replTest.getPrimary(), newPrimary);
-assert.commandWorked(
-    replTest.nodes[2].adminCommand({configureFailPoint: 'stopReplProducer', mode: 'off'}));
+stopReplProducerFailPoint.off();
 replTest.awaitReplication();
 
 jsTestLog("The transaction has been aborted on the new primary.");
diff --git a/jstests/replsets/change_stream_speculative_majority_lastApplied_lag.js b/jstests/replsets/change_stream_speculative_majority_lastApplied_lag.js
index 7c782617ae8..8db86cf6849 100644
--- a/jstests/replsets/change_stream_speculative_majority_lastApplied_lag.js
+++ b/jstests/replsets/change_stream_speculative_majority_lastApplied_lag.js
@@ -9,8 +9,8 @@
 "use strict";
 
 load('jstests/libs/change_stream_util.js');  // For ChangeStreamTest.
-load("jstests/libs/check_log.js");           // For checkLog.
-load("jstests/libs/parallelTester.js");      // for Thread.
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/parallelTester.js");  // for Thread.
 
 const name = "change_stream_speculative_majority_lastApplied_lag";
 const replTest = new ReplSetTest({
@@ -39,8 +39,7 @@ let startOperTime = res.operationTime;
 
 // Make the primary hang after it has completed a write but before it has advanced lastApplied
 // for that write.
-primaryDB.adminCommand(
-    {configureFailPoint: "hangBeforeLogOpAdvancesLastApplied", mode: "alwaysOn"});
+let failPoint = configureFailPoint(primaryDB, "hangBeforeLogOpAdvancesLastApplied");
 
 // Function which will be used by the background thread to perform an update on the specified
 // host, database, and collection.
@@ -59,7 +58,7 @@ primaryWrite.start();
 // should be visible. 'lastApplied', however, has not yet been advanced yet. We check both the
 // document state and the logs to make sure we hit the failpoint for the correct operation.
 assert.soon(() => (primaryColl.findOne({_id: 1}).v === 2));
-checkLog.contains(primary, 'hangBeforeLogOpAdvancesLastApplied fail point enabled.');
+failPoint.wait();
 
 // Open a change stream on the primary. The stream should only return the initial insert and the
 // first of the two update events, since the second update is not yet majority-committed.
@@ -98,7 +97,7 @@ cursor = cst.getNextBatch(cursor);
 assert.eq(cursor.nextBatch.length, 0);
 
 // Disable the failpoint to let the test complete.
-primaryDB.adminCommand({configureFailPoint: "hangBeforeLogOpAdvancesLastApplied", mode: "off"});
+failPoint.off();
 
 primaryWrite.join();
 replTest.stopSet();
diff --git a/jstests/replsets/commit_transaction_initial_sync_data_already_applied.js b/jstests/replsets/commit_transaction_initial_sync_data_already_applied.js
index 606e3bc5019..6b1ea579cc8 100644
--- a/jstests/replsets/commit_transaction_initial_sync_data_already_applied.js
+++ b/jstests/replsets/commit_transaction_initial_sync_data_already_applied.js
@@ -15,7 +15,6 @@
 (function() {
 "use strict";
 load("jstests/core/txns/libs/prepare_helpers.js");
-load("jstests/libs/check_log.js");
 
 const replTest = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0, votes: 0}}]});
 replTest.startSet();
@@ -51,8 +50,8 @@ secondary = replTest.restart(secondary, {
 });
 
 // Wait for fail point message to be logged so that we know that initial sync is paused.
-checkLog.contains(secondary,
-                  'initial sync - initialSyncHangBeforeCopyingDatabases fail point enabled');
+assert.commandWorked(secondary.adminCommand(
+    {waitForFailPoint: "initialSyncHangBeforeCopyingDatabases", timesEntered: 1}));
 
 jsTestLog("Initial sync paused");
 
diff --git a/jstests/replsets/create_drop_database_different_casing.js b/jstests/replsets/create_drop_database_different_casing.js
index 6ba4465d8da..d0709700053 100644
--- a/jstests/replsets/create_drop_database_different_casing.js
+++ b/jstests/replsets/create_drop_database_different_casing.js
@@ -17,6 +17,7 @@
 'use strict';
 
 load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]});
 rst.startSet();
@@ -30,14 +31,12 @@ const primary = rst.getPrimary();
 let upperDB = primary.getDB(dbNameUpper);
 assert.commandWorked(upperDB.createCollection("test"));
 
-assert.commandWorked(upperDB.adminCommand(
-    {configureFailPoint: 'dropDatabaseHangBeforeInMemoryDrop', mode: "alwaysOn"}));
+let failPoint = configureFailPoint(upperDB, 'dropDatabaseHangBeforeInMemoryDrop');
 let awaitDropUpper = startParallelShell(() => {
     db.getSiblingDB("A").dropDatabase();
 }, primary.port);
 
-checkLog.contains(primary, "dropDatabase - fail point dropDatabaseHangBeforeInMemoryDrop enabled.");
-
+failPoint.wait();
 let lowerDB = primary.getDB(dbNameLower);
 
 // The oplog entry to the secondaries to drop database "A" was sent, but the primary has not yet
@@ -45,8 +44,7 @@ let lowerDB = primary.getDB(dbNameLower);
 assert.commandFailedWithCode(lowerDB.createCollection("test"), ErrorCodes.DatabaseDifferCase);
 
 rst.awaitReplication();
-assert.commandWorked(
-    lowerDB.adminCommand({configureFailPoint: 'dropDatabaseHangBeforeInMemoryDrop', mode: "off"}));
+failPoint.off();
 
 checkLog.contains(primary, "dropDatabase " + dbNameUpper + " - finished");
 assert.commandWorked(lowerDB.createCollection("test"));
diff --git a/jstests/replsets/disconnect_on_legacy_write_to_secondary.js b/jstests/replsets/disconnect_on_legacy_write_to_secondary.js
index 9a5474e190c..edf5950a62b 100644
--- a/jstests/replsets/disconnect_on_legacy_write_to_secondary.js
+++ b/jstests/replsets/disconnect_on_legacy_write_to_secondary.js
@@ -4,7 +4,7 @@
 (function() {
 "use strict";
 
-load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]});
 rst.startSet();
@@ -59,18 +59,17 @@ function getNotMasterLegacyUnackWritesCounter() {
 
 function runStepDownTest({description, failpoint, operation}) {
     jsTestLog("Enabling failpoint to block " + description + "s");
-    assert.commandWorked(
-        primaryAdmin.adminCommand({configureFailPoint: failpoint, mode: "alwaysOn"}));
+    let failPoint = configureFailPoint(primaryAdmin, failpoint);
 
     let failedLegacyUnackWritesBefore = getNotMasterLegacyUnackWritesCounter();
 
     jsTestLog("Trying legacy " + description + " on stepping-down primary");
     operation();
-    checkLog.contains(primary, failpoint + " fail point enabled");
+    failPoint.wait();
     jsTestLog("Within " + description + ": stepping down and disabling failpoint");
     assert.commandWorked(primaryAdmin.adminCommand({replSetStepDown: 60, force: true}));
     rst.waitForState(primary, ReplSetTest.State.SECONDARY);
-    assert.commandWorked(primaryAdmin.adminCommand({configureFailPoint: failpoint, mode: "off"}));
+    failPoint.off();
     res = assert.throws(() => primaryDb.adminCommand({ping: 1}));
     assert(isNetworkError(res));
     // We should automatically reconnect after the failed command.
diff --git a/jstests/replsets/do_not_advance_commit_point_beyond_last_applied_term.js b/jstests/replsets/do_not_advance_commit_point_beyond_last_applied_term.js
index bceff6a1cfa..23d4221851d 100644
--- a/jstests/replsets/do_not_advance_commit_point_beyond_last_applied_term.js
+++ b/jstests/replsets/do_not_advance_commit_point_beyond_last_applied_term.js
@@ -7,7 +7,7 @@
 (function() {
 "use strict";
 
-load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 load("jstests/libs/write_concern_util.js");  // for [stop|restart]ServerReplication.
 
 const dbName = "test";
@@ -80,13 +80,11 @@ jsTest.log("Node E syncs from a majority node and learns the new commit point in
 // The stopReplProducerOnDocument failpoint ensures that Node E stops replicating before
 // applying the document {msg: "new primary"}, which is the first document of term 3. This
 // depends on the oplog fetcher batch size being 1.
-assert.commandWorked(nodeE.adminCommand({
-    configureFailPoint: "stopReplProducerOnDocument",
-    mode: "alwaysOn",
-    data: {document: {msg: "new primary"}}
-}));
+const failPoint =
+    configureFailPoint(nodeE, "stopReplProducerOnDocument", {document: {msg: "new primary"}});
 nodeE.reconnect([nodeA, nodeC, nodeD]);
-checkLog.contains(nodeE, "stopReplProducerOnDocument fail point is enabled.");
+failPoint.wait();
+
 assert.soon(() => {
     return 1 === nodeE.getDB(dbName)[collName].find({term: 1}).itcount();
 });
@@ -96,8 +94,7 @@ jsTest.log("Node E switches its sync source to B and replicates the stale branch
 nodeE.disconnect([nodeA, nodeC, nodeD]);
 nodeB.reconnect(nodeE);
 rst.awaitSyncSource(nodeE, nodeB);
-assert.commandWorked(
-    nodeE.adminCommand({configureFailPoint: "stopReplProducerOnDocument", mode: "off"}));
+failPoint.off();
 assert.soon(() => {
     return 1 === nodeE.getDB(dbName)[collName].find({term: 2}).itcount();
 });
diff --git a/jstests/replsets/drop_collections_two_phase_write_concern.js b/jstests/replsets/drop_collections_two_phase_write_concern.js
index 7a55c28d967..b0eda43d14c 100644
--- a/jstests/replsets/drop_collections_two_phase_write_concern.js
+++ b/jstests/replsets/drop_collections_two_phase_write_concern.js
@@ -6,7 +6,7 @@
 (function() {
 'use strict';
 
-load('jstests/libs/check_log.js');
+load("jstests/libs/fail_point_util.js");
 load('jstests/replsets/libs/two_phase_drops.js');  // For TwoPhaseDropCollectionTest.
 
 // Alias to logging function in two_phase_drops.js
@@ -59,8 +59,7 @@ const writeConcernForTimedOutOp = {
 assertTimeout(collForInserts.insert({_id: 1}, {writeConcern: writeConcernForTimedOutOp}));
 
 // Prevent drop collection reaper from making progress after resuming oplog application.
-assert.commandWorked(primary.adminCommand(
-    {configureFailPoint: 'dropPendingCollectionReaperHang', mode: 'alwaysOn'}));
+const failPoint = configureFailPoint(primary, 'dropPendingCollectionReaperHang');
 
 try {
     // Ensure that drop pending collection is not removed after resuming oplog application.
@@ -68,14 +67,13 @@ try {
     twoPhaseDropTest.resumeOplogApplication(twoPhaseDropTest.replTest.getSecondary());
 
     // Ensure that we've hit the failpoint before moving on.
-    checkLog.contains(primary, 'fail point dropPendingCollectionReaperHang enabled');
+    failPoint.wait();
 
     // While the drop pending collection reaper is blocked, an operation waiting on a majority
     // write concern should time out.
     assertTimeout(collForInserts.insert({_id: 2}, {writeConcern: writeConcernForTimedOutOp}));
 } finally {
-    assert.commandWorked(
-        primary.adminCommand({configureFailPoint: 'dropPendingCollectionReaperHang', mode: 'off'}));
+    failPoint.off();
 }
 
 // After the reaper is unblocked, an operation waiting on a majority write concern should run
diff --git a/jstests/replsets/hang_before_releasing_transaction_oplog_hole.js b/jstests/replsets/hang_before_releasing_transaction_oplog_hole.js
index 7156721fbc3..31fdde90f22 100644
--- a/jstests/replsets/hang_before_releasing_transaction_oplog_hole.js
+++ b/jstests/replsets/hang_before_releasing_transaction_oplog_hole.js
@@ -8,7 +8,6 @@
 
 (function() {
 'use strict';
-load("jstests/libs/check_log.js");
 
 const rst = new ReplSetTest({nodes: 1});
 rst.startSet();
@@ -50,7 +49,8 @@ function transactionFn() {
 const joinTransaction = startParallelShell(transactionFn, rst.ports[0]);
 
 jsTestLog("Waiting to hang with the oplog hole held open.");
-checkLog.contains(node, "hangBeforeReleasingTransactionOplogHole fail point enabled");
+assert.commandWorked(testDB.adminCommand(
+    {waitForFailPoint: 'hangBeforeReleasingTransactionOplogHole', timesEntered: 1}));
 
 jsTestLog("Waiting for 'commitTransaction' to advance lastApplied.");
 sleep(5 * 1000);
diff --git a/jstests/replsets/interrupted_batch_insert.js b/jstests/replsets/interrupted_batch_insert.js
index b5c9e62c18b..58e50e26825 100644
--- a/jstests/replsets/interrupted_batch_insert.js
+++ b/jstests/replsets/interrupted_batch_insert.js
@@ -13,6 +13,7 @@
 (function() {
 "use strict";
 
+load("jstests/libs/fail_point_util.js");
 load('jstests/libs/parallelTester.js');
 load("jstests/replsets/rslib.js");
 
@@ -43,8 +44,7 @@ stopServerReplication(conns[2]);
 
 // Allow the primary to insert the first 5 batches of documents. After that, the fail point
 // activates, and the client thread hangs until the fail point gets turned off.
-assert.commandWorked(primary.getDB("db").adminCommand(
-    {configureFailPoint: "hangDuringBatchInsert", mode: {skip: 5}}));
+let failPoint = configureFailPoint(primary.getDB("db"), "hangDuringBatchInsert", {}, {skip: 5});
 
 // In a background thread, issue an insert command to the primary that will insert 10 batches of
 // documents.
@@ -62,7 +62,7 @@ worker.start();
 
 // Wait long enough to guarantee that all 5 batches of inserts have executed and the primary is
 // hung on the "hangDuringBatchInsert" fail point.
-checkLog.contains(primary, "hangDuringBatchInsert fail point enabled");
+failPoint.wait();
 
 // Make sure the insert command is, in fact, running in the background.
 assert.eq(primary.getDB("db").currentOp({"command.insert": name, active: true}).inprog.length, 1);
@@ -91,8 +91,7 @@ replTest.waitForState(primary, ReplSetTest.State.PRIMARY);
 assert.eq(replTest.nodes[0], replTest.getPrimary());
 
 // Allow the batch insert to continue.
-assert.commandWorked(
-    primary.getDB("db").adminCommand({configureFailPoint: "hangDuringBatchInsert", mode: "off"}));
+failPoint.off();
 
 // Wait until the insert command is done.
 assert.soon(
diff --git a/jstests/replsets/kill_reads_with_prepare_conflicts_during_step_down.js b/jstests/replsets/kill_reads_with_prepare_conflicts_during_step_down.js
index 016f05fd5ea..56f46b58155 100644
--- a/jstests/replsets/kill_reads_with_prepare_conflicts_during_step_down.js
+++ b/jstests/replsets/kill_reads_with_prepare_conflicts_during_step_down.js
@@ -9,7 +9,7 @@
 "use strict";
 
 load("jstests/core/txns/libs/prepare_helpers.js");
-load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 // Start one of the nodes with priority: 0 to avoid elections.
 const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]});
@@ -32,8 +32,7 @@ const sessionID = session.getSessionId();
 let sessionDB = session.getDatabase(dbName);
 const sessionColl = sessionDB.getCollection(collName);
 
-assert.commandWorked(
-    primaryAdmin.adminCommand({configureFailPoint: "WTPrintPrepareConflictLog", mode: "alwaysOn"}));
+let failPoint = configureFailPoint(primaryAdmin, "WTPrintPrepareConflictLog");
 
 // Insert a document that we will later modify in a transaction.
 assert.commandWorked(primaryColl.insert({_id: 1}));
@@ -70,7 +69,7 @@ const readBlockedOnPrepareConflictThread = startParallelShell(() => {
 }, primary.port);
 
 jsTestLog("Waiting for failpoint");
-checkLog.contains(primary, "WTPrintPrepareConflictLog fail point enabled");
+failPoint.wait();
 
 // Once we have confirmed that the find command has hit a prepare conflict, we can perform
 // a step down.
diff --git a/jstests/replsets/kill_reads_with_prepare_conflicts_during_step_up.js b/jstests/replsets/kill_reads_with_prepare_conflicts_during_step_up.js
index 3b0b0b49e8b..9fccc50d7fb 100644
--- a/jstests/replsets/kill_reads_with_prepare_conflicts_during_step_up.js
+++ b/jstests/replsets/kill_reads_with_prepare_conflicts_during_step_up.js
@@ -10,6 +10,7 @@
 
 load("jstests/core/txns/libs/prepare_helpers.js");
 load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 const rst = new ReplSetTest({nodes: 2});
 rst.startSet();
@@ -36,8 +37,7 @@ const sessionID = session.getSessionId();
 let sessionDB = session.getDatabase(dbName);
 const sessionColl = sessionDB.getCollection(collName);
 
-assert.commandWorked(
-    secondary.adminCommand({configureFailPoint: "WTPrintPrepareConflictLog", mode: "alwaysOn"}));
+let failPoint = configureFailPoint(secondary, "WTPrintPrepareConflictLog");
 
 // Insert a document that we will later modify in a transaction.
 assert.commandWorked(primaryColl.insert({_id: 1}));
@@ -94,7 +94,7 @@ const waitForSecondaryReadBlockedOnPrepareConflictThread = startParallelShell(()
 }, secondary.port);
 
 jsTestLog("Waiting for failpoint");
-checkLog.contains(secondary, "WTPrintPrepareConflictLog fail point enabled");
+failPoint.wait();
 
 // Once we've confirmed that the find command has hit a prepare conflict on the secondary, cause
 // that secondary to step up.
diff --git a/jstests/replsets/kill_ttl_on_stepdown.js b/jstests/replsets/kill_ttl_on_stepdown.js
index 18738a3e7ff..31d8174ffd5 100644
--- a/jstests/replsets/kill_ttl_on_stepdown.js
+++ b/jstests/replsets/kill_ttl_on_stepdown.js
@@ -8,6 +8,7 @@
 "use strict";
 
 load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 const dbName = "kill_ttl_on_stepdown";
 
@@ -34,10 +35,8 @@ assert.soon(() => {
     return getNumTTLPasses() > 0;
 }, "TTLMonitor never did any passes.");
 
-assert.commandWorked(
-    primary.adminCommand({configureFailPoint: "hangTTLMonitorWithLock", mode: "alwaysOn"}));
-
-checkLog.contains(rst.getPrimary(), "Hanging due to hangTTLMonitorWithLock fail point");
+let failPoint = configureFailPoint(primary, "hangTTLMonitorWithLock");
+failPoint.wait();
 
 // See how many passes the TTLMonitor has done, before we stepdown the primary, killing it.
 let ttlPassesBeforeStepdown = getNumTTLPasses();
@@ -55,8 +54,7 @@ primary = rst.getPrimary();
 checkLog.contains(primary, "TTLMonitor was interrupted");
 
 // Disable the failpoint on the node that stepped down.
-assert.commandWorked(
-    primary.adminCommand({configureFailPoint: "hangTTLMonitorWithLock", mode: "off"}));
+failPoint.off();
 
 // Wait until the number TTLMonitor passes increases, informing us that the TTLMonitor thread
 // was not killed entirely and will continue to run after stepdown finishes.
diff --git a/jstests/replsets/libs/two_phase_drops.js b/jstests/replsets/libs/two_phase_drops.js
index bb772012fdb..5d638bcad88 100644
--- a/jstests/replsets/libs/two_phase_drops.js
+++ b/jstests/replsets/libs/two_phase_drops.js
@@ -16,7 +16,7 @@
  */
 "use strict";
 
-load("jstests/libs/check_log.js");            // For 'checkLog'.
+load("jstests/libs/fail_point_util.js");
 load("jstests/libs/fixture_helpers.js");      // For 'FixtureHelpers'.
 load("jstests/aggregation/extras/utils.js");  // For 'arrayEq'.
 
@@ -60,9 +60,8 @@ class TwoPhaseDropCollectionTest {
      * Pause oplog application on a specified node.
      */
     pauseOplogApplication(node) {
-        assert.commandWorked(node.adminCommand(
-            {configureFailPoint: this.oplogApplicationFailpoint, mode: "alwaysOn"}));
-        checkLog.contains(node, this.oplogApplicationFailpoint + " fail point enabled");
+        let failPoint = configureFailPoint(node, this.oplogApplicationFailpoint);
+        failPoint.wait();
     }
 
     /**
diff --git a/jstests/replsets/majority_writes_wait_for_all_durable_timestamp.js b/jstests/replsets/majority_writes_wait_for_all_durable_timestamp.js
index 5d0d962cda8..1e5d57a212c 100644
--- a/jstests/replsets/majority_writes_wait_for_all_durable_timestamp.js
+++ b/jstests/replsets/majority_writes_wait_for_all_durable_timestamp.js
@@ -8,7 +8,7 @@
 (function() {
 "use strict";
 
-load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 function assertWriteConcernTimeout(result) {
     assert.writeErrorWithCode(result, ErrorCodes.WriteConcernFailed);
@@ -32,11 +32,8 @@ TestData.collName = collName;
 testDB.runCommand({drop: collName, writeConcern: {w: "majority"}});
 assert.commandWorked(testDB.createCollection(collName, {writeConcern: {w: "majority"}}));
 
-assert.commandWorked(testDB.adminCommand({
-    configureFailPoint: "hangAfterCollectionInserts",
-    mode: "alwaysOn",
-    data: {collectionNS: testColl.getFullName(), first_id: "b"}
-}));
+const failPoint = configureFailPoint(
+    testDB, "hangAfterCollectionInserts", {collectionNS: testColl.getFullName(), first_id: "b"});
 
 jsTestLog(
     "Insert a document to hang before the insert completes to hold back the all durable timestamp.");
@@ -44,16 +41,14 @@ const joinHungWrite = startParallelShell(() => {
     assert.commandWorked(db.getSiblingDB(TestData.dbName)[TestData.collName].insert({_id: "b"}));
 }, primary.port);
 jsTestLog("Checking that the log contains fail point enabled.");
-checkLog.contains(testDB.getMongo(),
-                  "hangAfterCollectionInserts fail point enabled for " + testColl.getFullName());
+failPoint.wait();
 
 try {
     jsTest.log("Do a write with majority write concern that should time out.");
     assertWriteConcernTimeout(
         testColl.insert({_id: 0}, {writeConcern: {w: "majority", wtimeout: 2 * 1000}}));
 } finally {
-    assert.commandWorked(
-        primary.adminCommand({configureFailPoint: 'hangAfterCollectionInserts', mode: 'off'}));
+    failPoint.off();
 }
 
 joinHungWrite();
diff --git a/jstests/replsets/read_committed_stale_history.js b/jstests/replsets/read_committed_stale_history.js
index 79564b62821..97ea50423c6 100644
--- a/jstests/replsets/read_committed_stale_history.js
+++ b/jstests/replsets/read_committed_stale_history.js
@@ -5,7 +5,7 @@
 (function() {
 'use strict';
 
-load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 load("jstests/libs/write_concern_util.js");
 load("jstests/replsets/rslib.js");
 
@@ -63,8 +63,7 @@ assert.commandWorked(nodes[0].getDB(dbName).getCollection(collName).insert(
 // Stop the secondaries from replicating.
 stopServerReplication(secondaries);
 // Stop the primary from being able to complete stepping down.
-assert.commandWorked(
-    nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'alwaysOn'}));
+var blockHeartbeatStepdownFailPoint = configureFailPoint(nodes[0], 'blockHeartbeatStepdown');
 
 jsTestLog("Do a write that won't ever reach a majority of nodes");
 assert.commandWorked(nodes[0].getDB(dbName).getCollection(collName).insert({a: 2}));
@@ -73,8 +72,7 @@ assert.commandWorked(nodes[0].getDB(dbName).getCollection(collName).insert({a: 2
 checkDocNotCommitted(nodes[0], {a: 2});
 
 // Prevent the primary from rolling back later on.
-assert.commandWorked(
-    nodes[0].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'}));
+var rollbackHangBeforeStartFailPoint = configureFailPoint(nodes[0], 'rollbackHangBeforeStart');
 
 jsTest.log("Disconnect primary from all secondaries");
 nodes[0].disconnect(nodes[1]);
@@ -109,7 +107,7 @@ checkDocNotCommitted(nodes[0], {a: 2});
 jsTest.log("Allow the old primary to finish stepping down and become secondary");
 var res = null;
 try {
-    res = nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'off'});
+    blockHeartbeatStepdownFailPoint.off();
 } catch (e) {
     // Expected - once we disable the fail point the stepdown will proceed and it's racy whether
     // the stepdown closes all connections before or after the configureFailPoint command
@@ -124,12 +122,11 @@ reconnect(nodes[0]);
 // At this point the former primary will attempt to go into rollback, but the
 // 'rollbackHangBeforeStart' will prevent it from doing so.
 checkDocNotCommitted(nodes[0], {a: 2});
-checkLog.contains(nodes[0], 'rollback - rollbackHangBeforeStart fail point enabled');
+rollbackHangBeforeStartFailPoint.wait();
 checkDocNotCommitted(nodes[0], {a: 2});
 
 jsTest.log("Allow the original primary to roll back its write and catch up to the new primary");
-assert.adminCommandWorkedAllowingNetworkError(
-    nodes[0], {configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'});
+rollbackHangBeforeStartFailPoint.off();
 
 assert.soonNoExcept(function() {
     return null == nodes[0].getDB(dbName).getCollection(collName).findOne({a: 2});
diff --git a/jstests/replsets/reconfig_during_election.js b/jstests/replsets/reconfig_during_election.js
index 20e67a483c8..86cc5006d91 100644
--- a/jstests/replsets/reconfig_during_election.js
+++ b/jstests/replsets/reconfig_during_election.js
@@ -4,8 +4,8 @@
 
 (function() {
 "use strict";
+load("jstests/libs/fail_point_util.js");
 load("jstests/replsets/libs/election_handoff.js");
-load("jstests/libs/check_log.js");
 
 const rst = ReplSetTest({nodes: 2});
 const nodes = rst.startSet();
@@ -22,11 +22,8 @@ const candidate = rst.getSecondary();
 
 jsTestLog("Step down");
 
-assert.commandWorked(candidate.adminCommand({
-    configureFailPoint: "electionHangsBeforeUpdateMemberState",
-    mode: "alwaysOn",
-    data: {waitForMillis: 10 * 1000}
-}));
+const failPoint = configureFailPoint(
+    candidate, "electionHangsBeforeUpdateMemberState", {waitForMillis: 10 * 1000});
 
 // The incumbent sends replSetStepUp to the candidate for election handoff.
 assert.commandWorked(incumbent.adminCommand({
@@ -36,8 +33,7 @@ assert.commandWorked(incumbent.adminCommand({
 
 jsTestLog("Wait for candidate to win the election");
 
-checkLog.contains(candidate,
-                  "election succeeded - electionHangsBeforeUpdateMemberState fail point enabled");
+failPoint.wait();
 
 jsTestLog("Try to interrupt it with a reconfig");
 
@@ -45,8 +41,7 @@ config.members[nodes.indexOf(candidate)].priority = 2;
 config.version++;
 assert.commandWorked(candidate.adminCommand({replSetReconfig: config, force: true}));
 
-assert.commandWorked(candidate.adminCommand(
-    {configureFailPoint: "electionHangsBeforeUpdateMemberState", mode: "off"}));
+failPoint.off();
 
 rst.stopSet();
 })();
diff --git a/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js b/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js
index 2cc177445e1..82e4a126755 100644
--- a/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js
+++ b/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js
@@ -14,7 +14,6 @@
 (function() {
 "use strict";
 
-load("jstests/libs/check_log.js");
 load("jstests/core/txns/libs/prepare_helpers.js");
 
 const replTest = new ReplSetTest({nodes: 2});
@@ -94,7 +93,8 @@ secondary = replTest.start(
     true /* wait */);
 
 // Wait for failpoint to be reached so we know that collection cloning is paused.
-checkLog.contains(secondary, "initialSyncHangDuringCollectionClone fail point enabled");
+assert.commandWorked(secondary.adminCommand(
+    {waitForFailPoint: "initialSyncHangDuringCollectionClone", timesEntered: 1}));
 
 jsTestLog("Running operations while collection cloning is paused");
 
diff --git a/jstests/replsets/reconstruct_prepared_transactions_initial_sync_index_build.js b/jstests/replsets/reconstruct_prepared_transactions_initial_sync_index_build.js
index c09bc058739..414ad493e23 100644
--- a/jstests/replsets/reconstruct_prepared_transactions_initial_sync_index_build.js
+++ b/jstests/replsets/reconstruct_prepared_transactions_initial_sync_index_build.js
@@ -12,7 +12,6 @@
 (function() {
 "use strict";
 
-load("jstests/libs/check_log.js");
 load("jstests/core/txns/libs/prepare_helpers.js");
 
 const replTest = new ReplSetTest({nodes: 2});
@@ -57,7 +56,8 @@ secondary = replTest.start(
     true /* wait */);
 
 // Wait for failpoint to be reached so we know that collection cloning is paused.
-checkLog.contains(secondary, "initialSyncHangDuringCollectionClone fail point enabled");
+assert.commandWorked(secondary.adminCommand(
+    {waitForFailPoint: "initialSyncHangDuringCollectionClone", timesEntered: 1}));
 
 jsTestLog("Running operations while collection cloning is paused");
 
diff --git a/jstests/replsets/reconstruct_prepared_transactions_initial_sync_on_oplog_seed.js b/jstests/replsets/reconstruct_prepared_transactions_initial_sync_on_oplog_seed.js
index 30f9c497e48..73306f51375 100644
--- a/jstests/replsets/reconstruct_prepared_transactions_initial_sync_on_oplog_seed.js
+++ b/jstests/replsets/reconstruct_prepared_transactions_initial_sync_on_oplog_seed.js
@@ -14,7 +14,6 @@
 (function() {
 "use strict";
 
-load("jstests/libs/check_log.js");
 load("jstests/core/txns/libs/prepare_helpers.js");
 
 const replTest = new ReplSetTest({nodes: 2});
@@ -64,7 +63,8 @@ secondary = replTest.start(
     true /* wait */);
 
 // Wait for failpoint to be reached so we know that collection cloning is paused.
-checkLog.contains(secondary, "initialSyncHangDuringCollectionClone fail point enabled");
+assert.commandWorked(secondary.adminCommand(
+    {waitForFailPoint: "initialSyncHangDuringCollectionClone", timesEntered: 1}));
 
 jsTestLog("Running operations while collection cloning is paused");
 
@@ -80,7 +80,8 @@ assert.commandWorked(secondary.adminCommand(
 
 // Wait for failpoint to be reached so we know that first attempt is finishing and is about to
 // fail.
-checkLog.contains(secondary, "failAndHangInitialSync fail point enabled");
+assert.commandWorked(
+    secondary.adminCommand({waitForFailPoint: "failAndHangInitialSync", timesEntered: 1}));
 
 jsTestLog("Preparing the transaction before the second attempt of initial sync");
 
diff --git a/jstests/replsets/rollback_remote_cursor_retry.js b/jstests/replsets/rollback_remote_cursor_retry.js
index 2e2db8c5df6..7a44d00bf9e 100644
--- a/jstests/replsets/rollback_remote_cursor_retry.js
+++ b/jstests/replsets/rollback_remote_cursor_retry.js
@@ -7,8 +7,8 @@
 
 (function() {
 "use strict";
+load("jstests/libs/fail_point_util.js");
 load("jstests/replsets/libs/rollback_test.js");
-load("jstests/libs/check_log.js");
 
 const testName = "rollback_remote_cursor_retry";
 const dbName = testName;
@@ -25,25 +25,23 @@ const syncSource = rollbackTest.transitionToSyncSourceOperationsBeforeRollback()
 // This failpoint is used to make sure that we have started rollback before turning on
 // 'failCommand'. Otherwise, we would be failing the 'find' command that we issue against
 // the sync source before we decide to go into rollback.
-assert.commandWorked(
-    rollbackNode.adminCommand({configureFailPoint: "rollbackHangBeforeStart", mode: "alwaysOn"}));
+const rollbackHangBeforeStartFailPoint =
+    configureFailPoint(rollbackNode, "rollbackHangBeforeStart");
 
 rollbackTest.transitionToSyncSourceOperationsDuringRollback();
 
 // Ensure that we've hit the failpoint before moving on.
-checkLog.contains(rollbackNode, "rollback - rollbackHangBeforeStart fail point enabled");
+rollbackHangBeforeStartFailPoint.wait();
 
 // Fail the 'find' command exactly twice.
 jsTestLog("Failing the next two 'find' commands.");
-assert.commandWorked(syncSource.adminCommand({
-    configureFailPoint: "failCommand",
-    data: {errorCode: 279, failInternalCommands: true, failCommands: ["find"]},
-    mode: {times: 2}
-}));
+configureFailPoint(syncSource,
+                   "failCommand",
+                   {errorCode: 279, failInternalCommands: true, failCommands: ["find"]},
+                   {times: 2});
 
 // Let rollback proceed.
-assert.commandWorked(
-    rollbackNode.adminCommand({configureFailPoint: "rollbackHangBeforeStart", mode: "off"}));
+rollbackHangBeforeStartFailPoint.off();
 
 rollbackTest.transitionToSteadyStateOperations();
 rollbackTest.stop();
diff --git a/jstests/replsets/rollback_with_socket_error_then_steady_state.js b/jstests/replsets/rollback_with_socket_error_then_steady_state.js
index b7437d63b41..8ce86983224 100644
--- a/jstests/replsets/rollback_with_socket_error_then_steady_state.js
+++ b/jstests/replsets/rollback_with_socket_error_then_steady_state.js
@@ -7,6 +7,7 @@
 'use strict';
 
 load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 load("jstests/replsets/rslib.js");
 
 var collName = "test.coll";
@@ -71,8 +72,7 @@ jsTestLog("Do a write to node 1 on the [1,3,4] side of the partition.");
 assert.commandWorked(nodes[1].getCollection(collName).insert({a: counter++}));
 
 // Turn on failpoint on node 2 to pause rollback before doing anything.
-assert.commandWorked(
-    nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'}));
+let failPoint = configureFailPoint(nodes[2], 'rollbackHangBeforeStart');
 
 jsTestLog("Repartition to: [0] and [1,2,3,4].");
 nodes[2].disconnect(nodes[0]);
@@ -82,13 +82,13 @@ nodes[2].reconnect(nodes[4]);
 
 jsTestLog("Wait for node 2 to decide to go into ROLLBACK and start syncing from node 1.");
 // Since nodes 1 and 2 have now diverged, node 2 should go into rollback. The failpoint will
-// stop it from actually transitioning to rollback, so the checkLog bellow will ensure that we
+// stop it from actually transitioning to rollback, so the wait bellow will ensure that we
 // have decided to rollback, but haven't actually started yet.
 rst.awaitSyncSource(nodes[2], nodes[1]);
 
 jsTestLog("Wait for failpoint on node 2 to pause rollback before it starts");
 // Wait for fail point message to be logged.
-checkLog.contains(nodes[2], 'rollback - rollbackHangBeforeStart fail point enabled');
+failPoint.wait();
 
 jsTestLog("Repartition to: [1] and [0,2,3,4].");
 nodes[1].disconnect(nodes[3]);
@@ -101,8 +101,7 @@ nodes[4].reconnect(nodes[0]);
 nodes[4].reconnect(nodes[2]);
 
 // Turn off failpoint on node 2 to allow rollback against node 1 to fail with a network error.
-assert.adminCommandWorkedAllowingNetworkError(
-    nodes[2], {configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'});
+failPoint.off();
 
 // Make node 0 ahead of node 2 again so node 2 will pick it as a sync source.
 
diff --git a/jstests/replsets/rslib.js b/jstests/replsets/rslib.js
index 06bb6680df9..0182c3e5574 100644
--- a/jstests/replsets/rslib.js
+++ b/jstests/replsets/rslib.js
@@ -492,12 +492,15 @@ stopReplicationAndEnforceNewPrimaryToCatchUp = function(rst, node) {
 };
 
 /**
- * Sets the specified failpoint to 'alwaysOn' on the node.
+ * Sets the specified failpoint to 'alwaysOn' on the node and returns the number of
+ * times the fail point has been entered so far.
  */
 setFailPoint = function(node, failpoint, data = {}) {
     jsTestLog("Setting fail point " + failpoint);
-    assert.commandWorked(
-        node.adminCommand({configureFailPoint: failpoint, mode: "alwaysOn", data}));
+    let configureFailPointRes =
+        node.adminCommand({configureFailPoint: failpoint, mode: "alwaysOn", data: data});
+    assert.commandWorked(configureFailPointRes);
+    return configureFailPointRes.count;
 };
 
 /**
diff --git a/jstests/replsets/startup_without_fcv_document_succeeds_if_initial_sync_flag_set.js b/jstests/replsets/startup_without_fcv_document_succeeds_if_initial_sync_flag_set.js
index 5687a396db6..f39d62fff36 100644
--- a/jstests/replsets/startup_without_fcv_document_succeeds_if_initial_sync_flag_set.js
+++ b/jstests/replsets/startup_without_fcv_document_succeeds_if_initial_sync_flag_set.js
@@ -4,7 +4,8 @@
  */
 
 (function() {
-load("jstests/libs/check_log.js");
+
+load("jstests/libs/fail_point_util.js");
 
 rst = new ReplSetTest({nodes: 1});
 rst.startSet();
@@ -18,13 +19,10 @@ const nss = adminDbName + "." + versionCollName;
 
 // Hang initial sync before cloning the FCV document.
 let secondary = rst.add({rsConfig: {priority: 0}});
-assert.commandWorked(secondary.getDB('admin').runCommand({
-    configureFailPoint: 'initialSyncHangBeforeCollectionClone',
-    mode: 'alwaysOn',
-    data: {namespace: nss}
-}));
+let failPoint =
+    configureFailPoint(secondary, 'initialSyncHangBeforeCollectionClone', {namespace: nss});
 rst.reInitiate();
-checkLog.contains(secondary, "initialSyncHangBeforeCollectionClone fail point enabled.");
+failPoint.wait();
 
 jsTestLog("Restarting secondary in the early stages of initial sync.");
 rst.restart(secondary);
diff --git a/jstests/replsets/step_down_during_draining2.js b/jstests/replsets/step_down_during_draining2.js
index b7930e4caae..ad37a858dc4 100644
--- a/jstests/replsets/step_down_during_draining2.js
+++ b/jstests/replsets/step_down_during_draining2.js
@@ -14,7 +14,7 @@
 "use strict";
 
 load("jstests/replsets/rslib.js");
-load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 var replSet = new ReplSetTest({name: 'testSet', nodes: 3});
 var nodes = replSet.nodeList();
@@ -98,15 +98,14 @@ assert.commandFailedWithCode(
 
 // Prevent the current primary from stepping down
 jsTest.log("disallowing heartbeat stepdown " + secondary.host);
-assert.commandWorked(
-    secondary.adminCommand({configureFailPoint: "blockHeartbeatStepdown", mode: 'alwaysOn'}));
+var blockHeartbeatStepdownFailPoint = configureFailPoint(secondary, "blockHeartbeatStepdown");
 jsTestLog("Shut down the rest of the set so the primary-elect has to step down");
 replSet.stop(primary);
 disableFailPoint(replSet.nodes[2]);  // Fail point needs to be off when node is shut down.
 replSet.stop(2);
 
 jsTestLog("Waiting for secondary to begin stepping down while in drain mode");
-checkLog.contains(secondary, "stepDown - blockHeartbeatStepdown fail point enabled");
+blockHeartbeatStepdownFailPoint.wait();
 
 // Disable fail point to allow replication and allow secondary to finish drain mode while in the
 // process of stepping down.
@@ -135,8 +134,7 @@ assert.eq(ReplSetTest.State.PRIMARY, secondary.adminCommand({replSetGetStatus: 1
 assert(!secondary.adminCommand('ismaster').ismaster);
 
 jsTest.log("allowing heartbeat stepdown " + secondary.host);
-assert.commandWorked(
-    secondary.adminCommand({configureFailPoint: "blockHeartbeatStepdown", mode: 'off'}));
+blockHeartbeatStepdownFailPoint.off();
 
 jsTestLog("Checking that node successfully stepped down");
 replSet.waitForState(secondary, ReplSetTest.State.SECONDARY);
diff --git a/jstests/replsets/step_down_on_secondary.js b/jstests/replsets/step_down_on_secondary.js
index 42712c00c18..04cbed234d5 100644
--- a/jstests/replsets/step_down_on_secondary.js
+++ b/jstests/replsets/step_down_on_secondary.js
@@ -18,6 +18,7 @@ load('jstests/libs/parallelTester.js');
 load("jstests/libs/curop_helpers.js");  // for waitForCurOpByFailPoint().
 load("jstests/core/txns/libs/prepare_helpers.js");
 load("jstests/libs/check_log.js");
+load("jstests/libs/fail_point_util.js");
 
 const dbName = "test";
 const collName = "coll";
@@ -90,8 +91,7 @@ TestData.clusterTimeAfterPrepare =
 rst.awaitReplication();
 
 jsTestLog("Do a read that hits a prepare conflict on the old primary");
-assert.commandWorked(
-    primary.adminCommand({configureFailPoint: "WTPrintPrepareConflictLog", mode: "alwaysOn"}));
+const wTPrintPrepareConflictLogFailPoint = configureFailPoint(primary, "WTPrintPrepareConflictLog");
 
 const joinReadThread = startParallelShell(() => {
     db.getMongo().setSlaveOk(true);
@@ -106,7 +106,7 @@ const joinReadThread = startParallelShell(() => {
 }, primary.port);
 
 jsTestLog("Wait to hit a prepare conflict");
-checkLog.contains(primary, "WTPrintPrepareConflictLog fail point enabled");
+wTPrintPrepareConflictLogFailPoint.wait();
 
 jsTestLog("Allow step down to complete");
 assert.commandWorked(
-- 
cgit v1.2.1