diff options
author | Lingzhi Deng <lingzhi.deng@mongodb.com> | 2020-02-27 11:43:55 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-03-02 16:18:48 +0000 |
commit | 07fe606fcf78e668ee821a93af489f078323851a (patch) | |
tree | 9ceaa398c5c386a8ff21cb777bf1591ccea29cc6 | |
parent | dc5e4045d2fa6e29fc52fe9f9e10b457d67043dc (diff) | |
download | mongo-07fe606fcf78e668ee821a93af489f078323851a.tar.gz |
SERVER-33627: InitialSyncer needs to handle exceptions
-rw-r--r-- | src/mongo/db/repl/initial_syncer.cpp | 14 | ||||
-rw-r--r-- | src/mongo/db/repl/initial_syncer.h | 9 | ||||
-rw-r--r-- | src/mongo/db/repl/initial_syncer_test.cpp | 24 |
3 files changed, 40 insertions, 7 deletions
diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp index 6c57a4efc79..6bc07e723b7 100644 --- a/src/mongo/db/repl/initial_syncer.cpp +++ b/src/mongo/db/repl/initial_syncer.cpp @@ -554,7 +554,7 @@ void InitialSyncer::_tearDown_inlock(OperationContext* opCtx, void InitialSyncer::_startInitialSyncAttemptCallback( const executor::TaskExecutor::CallbackArgs& callbackArgs, std::uint32_t initialSyncAttempt, - std::uint32_t initialSyncMaxAttempts) { + std::uint32_t initialSyncMaxAttempts) noexcept { auto status = _checkForShutdownAndConvertStatus_inlock( callbackArgs, str::stream() << "error while starting initial sync attempt " << (initialSyncAttempt + 1) @@ -637,7 +637,7 @@ void InitialSyncer::_chooseSyncSourceCallback( const executor::TaskExecutor::CallbackArgs& callbackArgs, std::uint32_t chooseSyncSourceAttempt, std::uint32_t chooseSyncSourceMaxAttempts, - std::shared_ptr<OnCompletionGuard> onCompletionGuard) { + std::shared_ptr<OnCompletionGuard> onCompletionGuard) noexcept try { stdx::unique_lock<Latch> lock(_mutex); // Cancellation should be treated the same as other errors. In this case, the most likely cause // of a failed _chooseSyncSourceCallback() task is a cancellation triggered by @@ -730,6 +730,10 @@ void InitialSyncer::_chooseSyncSourceCallback( return; } _getBaseRollbackIdHandle = scheduleResult.getValue(); +} catch (const DBException&) { + // Report exception as an initial syncer failure. + stdx::unique_lock<Latch> lock(_mutex); + onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, exceptionToStatus()); } Status InitialSyncer::_truncateOplogAndDropReplicatedDatabases() { @@ -1388,7 +1392,7 @@ void InitialSyncer::_lastOplogEntryFetcherCallbackForStopTimestamp( void InitialSyncer::_getNextApplierBatchCallback( const executor::TaskExecutor::CallbackArgs& callbackArgs, - std::shared_ptr<OnCompletionGuard> onCompletionGuard) { + std::shared_ptr<OnCompletionGuard> onCompletionGuard) noexcept try { stdx::lock_guard<Latch> lock(_mutex); auto status = _checkForShutdownAndConvertStatus_inlock(callbackArgs, "error getting next applier batch"); @@ -1477,6 +1481,10 @@ void InitialSyncer::_getNextApplierBatchCallback( onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, status); return; } +} catch (const DBException&) { + // Report exception as an initial syncer failure. + stdx::unique_lock<Latch> lock(_mutex); + onCompletionGuard->setResultAndCancelRemainingWork_inlock(lock, exceptionToStatus()); } void InitialSyncer::_multiApplierCallback(const Status& multiApplierStatus, diff --git a/src/mongo/db/repl/initial_syncer.h b/src/mongo/db/repl/initial_syncer.h index b9f742cf015..4ebed91fe97 100644 --- a/src/mongo/db/repl/initial_syncer.h +++ b/src/mongo/db/repl/initial_syncer.h @@ -454,7 +454,7 @@ private: */ void _startInitialSyncAttemptCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs, std::uint32_t initialSyncAttempt, - std::uint32_t initialSyncMaxAttempts); + std::uint32_t initialSyncMaxAttempts) noexcept; /** * Callback to obtain sync source from sync source selector. @@ -465,7 +465,7 @@ private: void _chooseSyncSourceCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs, std::uint32_t chooseSyncSourceAttempt, std::uint32_t chooseSyncSourceMaxAttempts, - std::shared_ptr<OnCompletionGuard> onCompletionGuard); + std::shared_ptr<OnCompletionGuard> onCompletionGuard) noexcept; /** * This function does the following: @@ -551,8 +551,9 @@ private: /** * Callback to obtain next batch of operations to apply. */ - void _getNextApplierBatchCallback(const executor::TaskExecutor::CallbackArgs& callbackArgs, - std::shared_ptr<OnCompletionGuard> onCompletionGuard); + void _getNextApplierBatchCallback( + const executor::TaskExecutor::CallbackArgs& callbackArgs, + std::shared_ptr<OnCompletionGuard> onCompletionGuard) noexcept; /** * Callback for MultiApplier completion. diff --git a/src/mongo/db/repl/initial_syncer_test.cpp b/src/mongo/db/repl/initial_syncer_test.cpp index 39ed893a7bd..6344cefc503 100644 --- a/src/mongo/db/repl/initial_syncer_test.cpp +++ b/src/mongo/db/repl/initial_syncer_test.cpp @@ -1102,6 +1102,30 @@ TEST_F(InitialSyncerTest, InitialSyncerTruncatesOplogAndDropsReplicatedDatabases ASSERT_TRUE(_storageInterfaceWorkDone.droppedUserDBs); } +TEST_F(InitialSyncerTest, InitialSyncerTruncatesOplogAndDropsReplicatedDatabasesException) { + // Simulate an exception thrown at the dropReplicatedDatabases stage and test that the initial + // syncer handles exceptions correctly. + auto oldDropUserDBsFn = _storageInterface->dropUserDBsFn; + _storageInterface->dropUserDBsFn = [oldDropUserDBsFn](OperationContext* opCtx) { + ASSERT_OK(oldDropUserDBsFn(opCtx)); + uasserted(ErrorCodes::OperationFailed, "drop userdbs failed"); + return Status::OK(); + }; + + auto initialSyncer = &getInitialSyncer(); + auto opCtx = makeOpCtx(); + + _syncSourceSelector->setChooseNewSyncSourceResult_forTest(HostAndPort("localhost", 12345)); + ASSERT_OK(initialSyncer->startup(opCtx.get(), maxAttempts)); + + initialSyncer->join(); + ASSERT_EQUALS(ErrorCodes::OperationFailed, _lastApplied); + + LockGuard lock(_storageInterfaceWorkDoneMutex); + ASSERT_TRUE(_storageInterfaceWorkDone.truncateCalled); + ASSERT_TRUE(_storageInterfaceWorkDone.droppedUserDBs); +} + TEST_F(InitialSyncerTest, InitialSyncerPassesThroughGetRollbackIdScheduleError) { auto initialSyncer = &getInitialSyncer(); auto opCtx = makeOpCtx(); |