SERVER-32935 add characterization test for exception handling in oplog application loop

(cherry picked from commit a5cbd93aeaa7de9857b82396888a91ef488c9c7c)

2 files changed, 48 insertions, 0 deletions

diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index d8e21a7d7c6..0ff1dcc1a75 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -699,6 +699,7 @@ env.CppUnitTest(
         '$BUILD_DIR/mongo/db/dbdirectclient',
         '$BUILD_DIR/mongo/db/commands/mongod_fcv',
         'idempotency_test_fixture',
+        'oplog_buffer_blocking_queue',
         'oplog_interface_local',
         'sync_tail_test_fixture',
     ],
diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp
index e718ad23304..326108a41f3 100644
--- a/src/mongo/db/repl/sync_tail_test.cpp
+++ b/src/mongo/db/repl/sync_tail_test.cpp
@@ -53,8 +53,10 @@
 #include "mongo/db/repl/drop_pending_collection_reaper.h"
 #include "mongo/db/repl/idempotency_test_fixture.h"
 #include "mongo/db/repl/oplog.h"
+#include "mongo/db/repl/oplog_buffer_blocking_queue.h"
 #include "mongo/db/repl/oplog_interface_local.h"
 #include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/db/repl/replication_coordinator_mock.h"
 #include "mongo/db/repl/replication_process.h"
 #include "mongo/db/repl/storage_interface.h"
 #include "mongo/db/repl/sync_tail.h"
@@ -1107,6 +1109,51 @@ TEST_F(SyncTailTest,
     ASSERT_EQUALS(ErrorCodes::CollectionIsEmpty, iter->next().getStatus());
 }
 
+namespace {
+
+class ReplicationCoordinatorSignalDrainCompleteThrows : public ReplicationCoordinatorMock {
+public:
+    ReplicationCoordinatorSignalDrainCompleteThrows(ServiceContext* service,
+                                                    const ReplSettings& settings)
+        : ReplicationCoordinatorMock(service, settings) {}
+    void signalDrainComplete(OperationContext*, long long) final {
+        uasserted(ErrorCodes::OperationFailed, "failed to signal drain complete");
+    }
+};
+
+}  // namespace
+
+DEATH_TEST_F(SyncTailTest,
+             OplogApplicationLogsExceptionFromSignalDrainCompleteBeforeAborting,
+             "Invariant failure _isDead") {
+    // Leave oplog buffer empty so that SyncTail calls
+    // ReplicationCoordinator::signalDrainComplete() during oplog application.
+    auto oplogBuffer = std::make_unique<OplogBufferBlockingQueue>();
+
+    auto applyOperationFn =
+        [](OperationContext*, MultiApplier::OperationPtrs*, SyncTail*, WorkerMultikeyPathInfo*) {
+            return Status::OK();
+        };
+    auto writerPool = SyncTail::makeWriterPool();
+    OplogApplier::Options options;
+    SyncTail syncTail(nullptr,  // observer. not required by oplogApplication().
+                      _consistencyMarkers.get(),
+                      _storageInterface.get(),
+                      applyOperationFn,
+                      writerPool.get(),
+                      options);
+
+    auto service = getServiceContext();
+    auto currentReplCoord = ReplicationCoordinator::get(_opCtx.get());
+    ReplicationCoordinatorSignalDrainCompleteThrows replCoord(service,
+                                                              currentReplCoord->getSettings());
+    ASSERT_OK(replCoord.setFollowerMode(MemberState::RS_PRIMARY));
+
+    // SyncTail::oplogApplication() creates its own OperationContext in the current thread context.
+    _opCtx = {};
+    syncTail.oplogApplication(oplogBuffer.get(), &replCoord);
+}
+
 TEST_F(IdempotencyTest, Geo2dsphereIndexFailedOnUpdate) {
     ASSERT_OK(
         ReplicationCoordinator::get(_opCtx.get())->setFollowerMode(MemberState::RS_RECOVERING));