SERVER-33627: InitialSyncer needs to handle exceptions

(cherry picked from commit 07fe606fcf78e668ee821a93af489f078323851a)

3 files changed, 40 insertions, 7 deletions

diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp
index fff887f02eb..20fef105f26 100644
--- a/src/mongo/db/repl/initial_syncer.cpp
+++ b/src/mongo/db/repl/initial_syncer.cpp
@@ -444,7 +444,7 @@ void InitialSyncer::_tearDown_inlock(OperationContext* opCtx,
 void InitialSyncer::_startInitialSyncAttemptCallback(
     const executor::TaskExecutor::CallbackArgs& callbackArgs,
     std::uint32_t initialSyncAttempt,
-    std::uint32_t initialSyncMaxAttempts) {
+    std::uint32_t initialSyncMaxAttempts) noexcept {
     auto status = _checkForShutdownAndConvertStatus_inlock(
         callbackArgs,
         str::stream() << "error while starting initial sync attempt " << (initialSyncAttempt + 1)
@@ -509,7 +509,7 @@ void InitialSyncer::_chooseSyncSourceCallback(
     const executor::TaskExecutor::CallbackArgs& callbackArgs,
     std::uint32_t chooseSyncSourceAttempt,
     std::uint32_t chooseSyncSourceMaxAttempts,
-    std::shared_ptr<OnCompletionGuard> onCompletionGuard) {
+    std::shared_ptr<OnCompletionGuard> onCompletionGuard) noexcept try {
     stdx::unique_lock<stdx::mutex> lock(_mutex);
     // Cancellation should be treated the same as other errors. In this case, the most likely cause
     // of a failed _chooseSyncSourceCallback() task is a cancellation triggered by
@@ -591,6 +591,10 @@ void InitialSyncer::_chooseSyncSourceCallback(
         return;
     }
     _getBaseRollbackIdHandle = scheduleResult.getValue();
+} catch (const DBException&) {
+    // Report exception as an initial syncer failure.
+    stdx::unique_lock<stdx::mutex> lock(_mutex);
+    onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, exceptionToStatus());
 }
 
 Status InitialSyncer::_truncateOplogAndDropReplicatedDatabases() {
@@ -975,7 +979,7 @@ void InitialSyncer::_lastOplogEntryFetcherCallbackForStopTimestamp(
 
 void InitialSyncer::_getNextApplierBatchCallback(
     const executor::TaskExecutor::CallbackArgs& callbackArgs,
-    std::shared_ptr<OnCompletionGuard> onCompletionGuard) {
+    std::shared_ptr<OnCompletionGuard> onCompletionGuard) noexcept try {
     stdx::lock_guard<stdx::mutex> lock(_mutex);
     auto status =
         _checkForShutdownAndConvertStatus_inlock(callbackArgs, "error getting next applier batch");
@@ -1060,6 +1064,10 @@ void InitialSyncer::_getNextApplierBatchCallback(
         onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, status);
         return;
     }
+} catch (const DBException&) {
+    // Report exception as an initial syncer failure.
+    stdx::unique_lock<stdx::mutex> lock(_mutex);
+    onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, exceptionToStatus());
 }
 
 void InitialSyncer::_multiApplierCallback(const Status& multiApplierStatus,
diff --git a/src/mongo/db/repl/initial_syncer.h b/src/mongo/db/repl/initial_syncer.h
index b7e8a8b11bd..eb4c5610bd0 100644
--- a/src/mongo/db/repl/initial_syncer.h
+++ b/src/mongo/db/repl/initial_syncer.h
@@ -355,7 +355,7 @@ private:
      */
     void _startInitialSyncAttemptCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs,
                                           std::uint32_t initialSyncAttempt,
-                                          std::uint32_t initialSyncMaxAttempts);
+                                          std::uint32_t initialSyncMaxAttempts) noexcept;
 
     /**
      * Callback to obtain sync source from sync source selector.
@@ -366,7 +366,7 @@ private:
     void _chooseSyncSourceCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs,
                                    std::uint32_t chooseSyncSourceAttempt,
                                    std::uint32_t chooseSyncSourceMaxAttempts,
-                                   std::shared_ptr<OnCompletionGuard> onCompletionGuard);
+                                   std::shared_ptr<OnCompletionGuard> onCompletionGuard) noexcept;
 
     /**
      * This function does the following:
@@ -424,8 +424,9 @@ private:
     /**
      * Callback to obtain next batch of operations to apply.
      */
-    void _getNextApplierBatchCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs,
-                                      std::shared_ptr<OnCompletionGuard> onCompletionGuard);
+    void _getNextApplierBatchCallback(
+        const executor::TaskExecutor::CallbackArgs& callbackArgs,
+        std::shared_ptr<OnCompletionGuard> onCompletionGuard) noexcept;
 
     /**
      * Callback for MultiApplier completion.
diff --git a/src/mongo/db/repl/initial_syncer_test.cpp b/src/mongo/db/repl/initial_syncer_test.cpp
index 1ad077599a7..7a616406389 100644
--- a/src/mongo/db/repl/initial_syncer_test.cpp
+++ b/src/mongo/db/repl/initial_syncer_test.cpp
@@ -1036,6 +1036,30 @@ TEST_F(InitialSyncerTest, InitialSyncerTruncatesOplogAndDropsReplicatedDatabases
     ASSERT_TRUE(_storageInterfaceWorkDone.droppedUserDBs);
 }
 
+TEST_F(InitialSyncerTest, InitialSyncerTruncatesOplogAndDropsReplicatedDatabasesException) {
+    // Simulate an exception thrown at the dropReplicatedDatabases stage and test that the initial
+    // syncer handles exceptions correctly.
+    auto oldDropUserDBsFn = _storageInterface->dropUserDBsFn;
+    _storageInterface->dropUserDBsFn = [oldDropUserDBsFn](OperationContext* opCtx) {
+        ASSERT_OK(oldDropUserDBsFn(opCtx));
+        uasserted(ErrorCodes::OperationFailed, "drop userdbs failed");
+        return Status::OK();
+    };
+
+    auto initialSyncer = &getInitialSyncer();
+    auto opCtx = makeOpCtx();
+
+    _syncSourceSelector->setChooseNewSyncSourceResult_forTest(HostAndPort("localhost", 12345));
+    ASSERT_OK(initialSyncer->startup(opCtx.get(), maxAttempts));
+
+    initialSyncer->join();
+    ASSERT_EQUALS(ErrorCodes::OperationFailed, _lastApplied);
+
+    LockGuard lock(_storageInterfaceWorkDoneMutex);
+    ASSERT_TRUE(_storageInterfaceWorkDone.truncateCalled);
+    ASSERT_TRUE(_storageInterfaceWorkDone.droppedUserDBs);
+}
+
 TEST_F(InitialSyncerTest, InitialSyncerPassesThroughGetRollbackIdScheduleError) {
     auto initialSyncer = &getInitialSyncer();
     auto opCtx = makeOpCtx();