SERVER-66556 Block shutdown when releasing cursors to avoid race conditions

(cherry picked from commit 66e938045f87c72f379eb0def449b62bde97721c)

2 files changed, 25 insertions, 8 deletions

diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
index ff545f445b4..64185e7eb3b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
@@ -131,6 +131,10 @@ WT_CURSOR* WiredTigerSession::getReadOnceCursor(const std::string& uri, bool all
 }
 
 void WiredTigerSession::releaseCursor(uint64_t id, WT_CURSOR* cursor) {
+    // When releasing the cursor, we would want to check if the session cache is already in shutdown
+    // and prevent the race condition that the shutdown starts after the check.
+    WiredTigerSessionCache::BlockShutdown blockShutdown(_cache);
+
     // Avoids the cursor already being destroyed during the shutdown.
     if (_cache->isShuttingDown()) {
         return;
@@ -224,7 +228,7 @@ void WiredTigerSessionCache::shuttingDown() {
     if (_shuttingDown.fetchAndBitOr(kShuttingDownMask) & kShuttingDownMask)
         return;
 
-    // Spin as long as there are threads in releaseSession
+    // Spin as long as there are threads blocking shutdown.
     while (_shuttingDown.load() != kShuttingDownMask) {
         sleepmillis(1);
     }
@@ -243,12 +247,11 @@ void WiredTigerSessionCache::waitUntilDurable(bool forceCheckpoint, bool stableC
         return;
     }
 
-    const int shuttingDown = _shuttingDown.fetchAndAdd(1);
-    ON_BLOCK_EXIT([this] { _shuttingDown.fetchAndSubtract(1); });
+    BlockShutdown blockShutdown(this);
 
     uassert(ErrorCodes::ShutdownInProgress,
             "Cannot wait for durability because a shutdown is in progress",
-            !(shuttingDown & kShuttingDownMask));
+            !isShuttingDown());
 
     // Stable checkpoints are only meaningful in a replica set. Replication sets the "stable
     // timestamp". If the stable timestamp is unset, WiredTiger takes a full checkpoint, which is
@@ -428,10 +431,9 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) {
     // We might have skipped releasing some cursors during the shutdown.
     invariant(session->cursorsOut() == 0 || isShuttingDown());
 
-    const int shuttingDown = _shuttingDown.fetchAndAdd(1);
-    ON_BLOCK_EXIT([this] { _shuttingDown.fetchAndSubtract(1); });
+    BlockShutdown blockShutdown(this);
 
-    if (shuttingDown & kShuttingDownMask) {
+    if (isShuttingDown()) {
         // There is a race condition with clean shutdown, where the storage engine is ripped from
         // underneath OperationContexts, which are not "active" (i.e., do not have any locks), but
         // are just about to delete the recovery unit. See SERVER-16031 for more information. Since
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
index 89d95ad7ed0..04b1773f8c0 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h
@@ -213,6 +213,21 @@ public:
         void operator()(WiredTigerSession* session) const;
     };
 
+    // RAII type to block and unblock the WiredTigerSessionCache to shut down.
+    class BlockShutdown {
+    public:
+        BlockShutdown(WiredTigerSessionCache* cache) : _cache(cache) {
+            _cache->_shuttingDown.fetchAndAdd(1);
+        }
+
+        ~BlockShutdown() {
+            _cache->_shuttingDown.fetchAndSubtract(1);
+        }
+
+    private:
+        WiredTigerSessionCache* _cache;
+    };
+
     /**
      * Indicates that WiredTiger should be configured to cache cursors.
      */
@@ -324,7 +339,7 @@ private:
     WiredTigerSnapshotManager _snapshotManager;
 
     // Used as follows:
-    //   The low 31 bits are a count of active calls to releaseSession.
+    //   The low 31 bits are a count of active calls that need to block shutdown.
     //   The high bit is a flag that is set if and only if we're shutting down.
     AtomicWord<unsigned> _shuttingDown;
     static const uint32_t kShuttingDownMask = 1 << 31;