diff options
author | Randolph Tan <randolph@10gen.com> | 2021-01-14 15:20:00 +0000 |
---|---|---|
committer | Randolph Tan <randolph@10gen.com> | 2021-04-23 13:46:09 +0000 |
commit | 77530ffba57e2affb7574da8d04b7e91a66aa248 (patch) | |
tree | d865e59515375e09bdabd739920be3f9163eeda6 | |
parent | 1786a10c7278a9242566ec5647687fa95bf68f12 (diff) | |
download | mongo-77530ffba57e2affb7574da8d04b7e91a66aa248.tar.gz |
SERVER-52564 Deadlock between step down and MongoDOperationContextSession
(cherry picked from commit 6ee5a25cfc951f6e914dcc9f7d1a63d2e7aeaa67)
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/session_catalog.cpp | 22 | ||||
-rw-r--r-- | src/mongo/db/session_catalog.h | 32 |
3 files changed, 59 insertions, 0 deletions
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index d375b54f735..16bc218e87e 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -2509,6 +2509,11 @@ void ReplicationCoordinatorImpl::stepDown(OperationContext* opCtx, CurOpFailpointHelpers::waitWhileFailPointEnabled( &stepdownHangBeforeRSTLEnqueue, opCtx, "stepdownHangBeforeRSTLEnqueue"); + // To prevent a deadlock between session checkout and RSTL lock taking, disallow new sessions + // from being checked out. Existing sessions currently checked out will be killed by the + // killOpThread. + ScopedBlockSessionCheckouts blockSessions(opCtx); + // Using 'force' sets the default for the wait time to zero, which means the stepdown will // fail if it does not acquire the lock immediately. In such a scenario, we use the // stepDownUntil deadline instead. diff --git a/src/mongo/db/session_catalog.cpp b/src/mongo/db/session_catalog.cpp index 55911f5a802..23934323e84 100644 --- a/src/mongo/db/session_catalog.cpp +++ b/src/mongo/db/session_catalog.cpp @@ -80,6 +80,10 @@ SessionCatalog::ScopedCheckedOutSession SessionCatalog::_checkOutSession(Operati invariant(!opCtx->lockState()->isLocked()); stdx::unique_lock<Latch> ul(_mutex); + uassert(ErrorCodes::InterruptedDueToReplStateChange, + "a stepdown process started, can't checkout sessions except for killing", + _checkoutAllowed); + auto sri = _getOrCreateSessionRuntimeInfo(ul, opCtx, *opCtx->getLogicalSessionId()); // Wait until the session is no longer checked out and until the previously scheduled kill has @@ -176,6 +180,16 @@ void SessionCatalog::scanSessions(const SessionKiller::Matcher& matcher, } } +void SessionCatalog::_disallowCheckoutsExceptForKilling() { + stdx::unique_lock<Latch> ul(_mutex); + _checkoutAllowed = false; +} + +void SessionCatalog::_allowCheckouts() { + stdx::lock_guard<Latch> lg(_mutex); + _checkoutAllowed = true; +} + SessionCatalog::KillToken SessionCatalog::killSession(const LogicalSessionId& lsid) { stdx::lock_guard<Latch> lg(_mutex); auto it = _sessions.find(lsid); @@ -325,4 +339,12 @@ void OperationContextSession::checkOut(OperationContext* opCtx) { checkedOutSession.emplace(std::move(scopedCheckedOutSession)); } +ScopedBlockSessionCheckouts::ScopedBlockSessionCheckouts(OperationContext* opCtx) : _opCtx(opCtx) { + SessionCatalog::get(_opCtx)->_disallowCheckoutsExceptForKilling(); +} + +ScopedBlockSessionCheckouts::~ScopedBlockSessionCheckouts() { + SessionCatalog::get(_opCtx)->_allowCheckouts(); +} + } // namespace mongo diff --git a/src/mongo/db/session_catalog.h b/src/mongo/db/session_catalog.h index 6841d85a6f6..7e30d62798e 100644 --- a/src/mongo/db/session_catalog.h +++ b/src/mongo/db/session_catalog.h @@ -46,6 +46,7 @@ namespace mongo { class ObservableSession; +class ScopedBlockSessionCheckouts; /** * Keeps track of the transaction runtime state for every active session on this instance. @@ -116,6 +117,7 @@ public: size_t size() const; private: + friend ScopedBlockSessionCheckouts; struct SessionRuntimeInfo { SessionRuntimeInfo(LogicalSessionId lsid) : session(std::move(lsid)) {} ~SessionRuntimeInfo(); @@ -153,12 +155,27 @@ private: */ void _releaseSession(SessionRuntimeInfo* sri, boost::optional<KillToken> killToken); + /** + * Disallow checkouts that are not for killing. + */ + void _disallowCheckoutsExceptForKilling(); + + /** + * Re-enable checkouts if it was disallowed earlier. + */ + void _allowCheckouts(); + // Protects the state below mutable Mutex _mutex = MONGO_MAKE_LATCH(HierarchicalAcquisitionLevel(0), "SessionCatalog::_mutex"); // Owns the Session objects for all current Sessions. SessionRuntimeInfoMap _sessions; + + // If false no new sessions can be checked out. Reasons why this could be true is because step + // down is in progress and we should not allow new sessions to get checked out in order to + // prevent deadlocks. + bool _checkoutAllowed{true}; }; /** @@ -374,4 +391,19 @@ private: OperationContext* const _opCtx; }; +/** + * Scoped object, while active will prevent the checkout of sessions except for killing. + */ +class ScopedBlockSessionCheckouts { + ScopedBlockSessionCheckouts(const ScopedBlockSessionCheckouts&) = delete; + ScopedBlockSessionCheckouts& operator=(const ScopedBlockSessionCheckouts&) = delete; + +public: + ScopedBlockSessionCheckouts(OperationContext* opCtx); + ~ScopedBlockSessionCheckouts(); + +private: + OperationContext* const _opCtx; +}; + } // namespace mongo |