SERVER-59888 Ensure consistent acquisition order for `_statesMutex` and `_idleMutex` in the `BucketCatalog`

(cherry picked from commit 6bd9072c982bec88a61573ec723bce711a6c54a1)

2 files changed, 25 insertions, 26 deletions

diff --git a/src/mongo/db/timeseries/bucket_catalog.cpp b/src/mongo/db/timeseries/bucket_catalog.cpp
index a0353b46449..e65da2ce66f 100644
--- a/src/mongo/db/timeseries/bucket_catalog.cpp
+++ b/src/mongo/db/timeseries/bucket_catalog.cpp
@@ -886,6 +886,9 @@ BucketCatalog::BucketAccess::BucketAccess(BucketCatalog* catalog,
                                           Bucket* bucket,
                                           boost::optional<BucketState> targetState)
     : _catalog(catalog) {
+    invariant(!targetState || targetState == BucketState::kNormal ||
+              targetState == BucketState::kPrepared);
+
     {
         auto lk = _catalog->_lockShared();
         auto bucketIt = _catalog->_allBuckets.find(bucket);
@@ -897,17 +900,8 @@ BucketCatalog::BucketAccess::BucketAccess(BucketCatalog* catalog,
         _acquire();
     }
 
-    boost::optional<BucketState> state{BucketState::kCleared};
-    if (targetState) {
-        invariant(*targetState == BucketState::kNormal || *targetState == BucketState::kPrepared);
-        state = _catalog->_setBucketState(_bucket->_id, *targetState);
-    } else {
-        stdx::lock_guard statesLk{_catalog->_statesMutex};
-        auto statesIt = _catalog->_bucketStates.find(_bucket->_id);
-        if (statesIt != _catalog->_bucketStates.end()) {
-            state = statesIt->second;
-        }
-    }
+    auto state =
+        targetState ? _catalog->_setBucketState(_bucket->_id, *targetState) : _getBucketState();
     if (!state || state == BucketState::kCleared || state == BucketState::kPreparedAndCleared) {
         release();
     }
@@ -919,6 +913,12 @@ BucketCatalog::BucketAccess::~BucketAccess() {
     }
 }
 
+boost::optional<BucketCatalog::BucketState> BucketCatalog::BucketAccess::_getBucketState() const {
+    stdx::lock_guard lk{_catalog->_statesMutex};
+    auto it = _catalog->_bucketStates.find(_bucket->_id);
+    return it != _catalog->_bucketStates.end() ? boost::make_optional(it->second) : boost::none;
+}
+
 BucketCatalog::BucketState BucketCatalog::BucketAccess::_findOpenBucketThenLock(
     const HashedBucketKey& key) {
     {
@@ -970,10 +970,7 @@ BucketCatalog::BucketState BucketCatalog::BucketAccess::_findOpenBucketThenLockA
 }
 
 BucketCatalog::BucketState BucketCatalog::BucketAccess::_confirmStateForAcquiredBucket() {
-    stdx::lock_guard statesLk{_catalog->_statesMutex};
-    auto statesIt = _catalog->_bucketStates.find(_bucket->_id);
-    invariant(statesIt != _catalog->_bucketStates.end());
-    auto& [_, state] = *statesIt;
+    auto state = *_getBucketState();
     if (state == BucketState::kCleared || state == BucketState::kPreparedAndCleared) {
         release();
     } else {
@@ -995,15 +992,10 @@ void BucketCatalog::BucketAccess::_findOrCreateOpenBucketThenLock(
     _bucket = it->second;
     _acquire();
 
-    {
-        stdx::lock_guard statesLk{_catalog->_statesMutex};
-        auto statesIt = _catalog->_bucketStates.find(_bucket->_id);
-        invariant(statesIt != _catalog->_bucketStates.end());
-        auto& [_, state] = *statesIt;
-        if (state == BucketState::kNormal || state == BucketState::kPrepared) {
-            _catalog->_markBucketNotIdle(_bucket, false /* locked */);
-            return;
-        }
+    auto state = *_getBucketState();
+    if (state == BucketState::kNormal || state == BucketState::kPrepared) {
+        _catalog->_markBucketNotIdle(_bucket, false /* locked */);
+        return;
     }
 
     _catalog->_abort(_guard, _bucket, nullptr, boost::none);
diff --git a/src/mongo/db/timeseries/bucket_catalog.h b/src/mongo/db/timeseries/bucket_catalog.h
index d8a5ffcf574..44c59a9ede1 100644
--- a/src/mongo/db/timeseries/bucket_catalog.h
+++ b/src/mongo/db/timeseries/bucket_catalog.h
@@ -576,6 +576,11 @@ private:
 
     private:
         /**
+         * Returns the state of the bucket, or boost::none if there is no state for the bucket.
+         */
+        boost::optional<BucketState> _getBucketState() const;
+
+        /**
          * Helper to find and lock an open bucket for the given metadata if it exists. Takes a
          * shared lock on the catalog. Returns the state of the bucket if it is locked and usable.
          * In case the bucket does not exist or was previously cleared and thus is not usable, the
@@ -721,11 +726,13 @@ private:
     stdx::unordered_map<BucketKey, Bucket*, BucketHasher, BucketEq> _openBuckets;
 
     // Bucket state
-    mutable Mutex _statesMutex = MONGO_MAKE_LATCH("BucketCatalog::_statesMutex");
+    mutable Mutex _statesMutex =
+        MONGO_MAKE_LATCH(HierarchicalAcquisitionLevel(0), "BucketCatalog::_statesMutex");
     stdx::unordered_map<OID, BucketState, OID::Hasher> _bucketStates;
 
     // This mutex protects access to _idleBuckets
-    mutable Mutex _idleMutex = MONGO_MAKE_LATCH("BucketCatalog::_idleMutex");
+    mutable Mutex _idleMutex =
+        MONGO_MAKE_LATCH(HierarchicalAcquisitionLevel(1), "BucketCatalog::_idleMutex");
 
     // Buckets that do not have any writers.
     IdleList _idleBuckets;