SERVER-50955 oplog_rollover.js pauses the OplogCapMaintainerThread until truncation is needed

(cherry picked from commit 297a44e948fae079039882c88f6191984b58990a)

2 files changed, 40 insertions, 1 deletions

diff --git a/jstests/replsets/oplog_rollover.js b/jstests/replsets/oplog_rollover.js
index 5199a1802fc..42d25b412fc 100644
--- a/jstests/replsets/oplog_rollover.js
+++ b/jstests/replsets/oplog_rollover.js
@@ -8,9 +8,19 @@
 function doTest(storageEngine) {
     jsTestLog("Testing with storageEngine: " + storageEngine);
 
+    // Pause the oplog cap maintainer thread for this test until oplog truncation is needed. The
+    // truncation thread can hold a mutex for a short period of time which prevents new oplog stones
+    // from being created during an insertion if the mutex cannot be obtained immediately. Instead,
+    // the next insertion will attempt to create a new oplog stone, which this test does not do.
     const replSet = new ReplSetTest({
         // Set the syncdelay to 1s to speed up checkpointing.
-        nodeOptions: {syncdelay: 1, setParameter: {logComponentVerbosity: tojson({storage: 1})}},
+        nodeOptions: {
+            syncdelay: 1,
+            setParameter: {
+                logComponentVerbosity: tojson({storage: 1}),
+                'failpoint.hangOplogCapMaintainerThread': tojson({mode: 'alwaysOn'})
+            }
+        },
         nodes: [{}, {rsConfig: {priority: 0, votes: 0}}]
     });
     // Set max oplog size to 1MB.
@@ -22,6 +32,10 @@ function doTest(storageEngine) {
     const secondary = replSet.getSecondary();
     const secondaryOplog = secondary.getDB("local").oplog.rs;
 
+    // Verify that the oplog cap maintainer thread is paused.
+    checkLog.contains(primary, "Hanging the oplog cap maintainer thread due to fail point");
+    checkLog.contains(secondary, "Hanging the oplog cap maintainer thread due to fail point");
+
     const coll = primary.getDB("test").foo;
     // 400KB each so that oplog can keep at most two insert oplog entries.
     const longString = new Array(400 * 1024).join("a");
@@ -96,6 +110,17 @@ function doTest(storageEngine) {
                 .operationTime;
         jsTestLog("Third insert timestamp: " + thirdInsertTimestamp);
 
+        // Verify that there are three oplog entries while the oplog cap maintainer thread is
+        // paused.
+        assert.eq(3, numInsertOplogEntry(primaryOplog));
+        assert.eq(3, numInsertOplogEntry(secondaryOplog));
+
+        // Let the oplog cap maintainer thread start truncating the oplog.
+        assert.commandWorked(primary.adminCommand(
+            {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"}));
+        assert.commandWorked(secondary.adminCommand(
+            {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"}));
+
         // Test that oplog entry of the initial insert rolls over on both primary and secondary.
         // Use assert.soon to wait for oplog truncater thread to run.
         assert.soon(() => {
@@ -113,6 +138,12 @@ function doTest(storageEngine) {
                 res.oplogTruncation.totalTimeTruncatingMicros, 0, tojson(res.oplogTruncation));
         }
     } else {
+        // Let the oplog cap maintainer thread start truncating the oplog.
+        assert.commandWorked(primary.adminCommand(
+            {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"}));
+        assert.commandWorked(secondary.adminCommand(
+            {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"}));
+
         // Only test that oplog truncation will eventually happen.
         let numInserted = 2;
         assert.soon(function() {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store_mongod.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store_mongod.cpp
index 325f34cd048..89ee2ad5ab9 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store_mongod.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store_mongod.cpp
@@ -49,12 +49,15 @@
 #include "mongo/stdx/mutex.h"
 #include "mongo/util/background.h"
 #include "mongo/util/exit.h"
+#include "mongo/util/fail_point.h"
 #include "mongo/util/log.h"
 
 namespace mongo {
 
 namespace {
 
+MONGO_FAIL_POINT_DEFINE(hangOplogCapMaintainerThread);
+
 std::set<NamespaceString> _backgroundThreadNamespaces;
 Mutex _backgroundThreadMutex;
 
@@ -132,6 +135,11 @@ public:
         ThreadClient tc(_name, getGlobalServiceContext());
 
         while (!globalInShutdownDeprecated()) {
+            if (MONGO_FAIL_POINT(hangOplogCapMaintainerThread)) {
+                log() << "Hanging the oplog cap maintainer thread due to fail point";
+                MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangOplogCapMaintainerThread);
+            }
+
             if (!_deleteExcessDocuments()) {
                 sleepmillis(1000);  // Back off in case there were problems deleting.
             }