SERVER-68553 Add local DDL lock acquisition reason

7 files changed, 51 insertions, 21 deletions

diff --git a/src/mongo/db/commands/internal_rename_if_options_and_indexes_match_cmd.cpp b/src/mongo/db/commands/internal_rename_if_options_and_indexes_match_cmd.cpp
index a374ad2ab6d..623440c6e1a 100644
--- a/src/mongo/db/commands/internal_rename_if_options_and_indexes_match_cmd.cpp
+++ b/src/mongo/db/commands/internal_rename_if_options_and_indexes_match_cmd.cpp
@@ -87,11 +87,12 @@ public:
             // - Serialize with sharded DDLs, ensuring no concurrent modifications of the
             // collections.
             // - Check safely if the target collection is sharded or not.
+            static constexpr StringData lockReason{"internalRenameCollection"_sd};
             auto distLockManager = DistLockManager::get(opCtx);
             auto fromLocalDistlock = distLockManager->lockDirectLocally(
-                opCtx, fromNss.ns(), DistLockManager::kDefaultLockTimeout);
+                opCtx, fromNss.ns(), lockReason, DistLockManager::kDefaultLockTimeout);
             auto toLocalDistLock = distLockManager->lockDirectLocally(
-                opCtx, toNss.ns(), DistLockManager::kDefaultLockTimeout);
+                opCtx, toNss.ns(), lockReason, DistLockManager::kDefaultLockTimeout);
 
             uassert(ErrorCodes::IllegalOperation,
                     str::stream() << "cannot rename to sharded collection '" << toNss << "'",
diff --git a/src/mongo/db/s/balancer/balancer_dist_locks.cpp b/src/mongo/db/s/balancer/balancer_dist_locks.cpp
index a7384f2d1eb..e1ce0904f8c 100644
--- a/src/mongo/db/s/balancer/balancer_dist_locks.cpp
+++ b/src/mongo/db/s/balancer/balancer_dist_locks.cpp
@@ -46,7 +46,10 @@ Status BalancerDistLocks::acquireFor(OperationContext* opCtx, const NamespaceStr
         boost::optional<DistLockManager::ScopedLock> scopedLock;
         try {
             scopedLock.emplace(DistLockManager::get(opCtx)->lockDirectLocally(
-                opCtx, nss.ns(), DistLockManager::kSingleLockAttemptTimeout));
+                opCtx,
+                nss.ns(),
+                "moveRange" /* reason */,
+                DistLockManager::kSingleLockAttemptTimeout));
 
             const std::string whyMessage(str::stream()
                                          << "Migrating chunk(s) in collection " << nss.ns());
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp
index f3f2a6fd4a6..7288c2dfe31 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp
@@ -144,7 +144,7 @@ DatabaseType ShardingCatalogManager::createDatabase(
         // Do another loop, with the db lock held in order to avoid taking the expensive path on
         // concurrent create database operations
         dbLock.emplace(DistLockManager::get(opCtx)->lockDirectLocally(
-            opCtx, dbName, DistLockManager::kDefaultLockTimeout));
+            opCtx, dbName, "createDatabase" /* reason */, DistLockManager::kDefaultLockTimeout));
     }
 
     // Expensive createDatabase code path
diff --git a/src/mongo/db/s/dist_lock_manager.cpp b/src/mongo/db/s/dist_lock_manager.cpp
index ac7d5483f9b..5f894be7d73 100644
--- a/src/mongo/db/s/dist_lock_manager.cpp
+++ b/src/mongo/db/s/dist_lock_manager.cpp
@@ -28,11 +28,10 @@
  */
 
 
-#include "mongo/platform/basic.h"
-
 #include "mongo/db/s/dist_lock_manager.h"
 
 #include "mongo/db/operation_context.h"
+#include "mongo/logv2/log.h"
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding
 
@@ -101,16 +100,16 @@ void DistLockManager::create(ServiceContext* service,
 
 StatusWith<DistLockManager::ScopedDistLock> DistLockManager::lock(OperationContext* opCtx,
                                                                   StringData name,
-                                                                  StringData whyMessage,
+                                                                  StringData reason,
                                                                   Milliseconds waitFor) {
     boost::optional<ScopedLock> scopedLock;
     try {
-        scopedLock.emplace(lockDirectLocally(opCtx, name, waitFor));
+        scopedLock.emplace(lockDirectLocally(opCtx, name, reason, waitFor));
     } catch (const DBException& ex) {
         return ex.toStatus();
     }
 
-    auto status = lockDirect(opCtx, name, whyMessage, waitFor);
+    auto status = lockDirect(opCtx, name, reason, waitFor);
     if (!status.isOK()) {
         return status;
     }
@@ -120,33 +119,43 @@ StatusWith<DistLockManager::ScopedDistLock> DistLockManager::lock(OperationConte
 
 DistLockManager::ScopedLock DistLockManager::lockDirectLocally(OperationContext* opCtx,
                                                                StringData ns,
+                                                               StringData reason,
                                                                Milliseconds waitFor) {
     stdx::unique_lock<Latch> lock(_mutex);
     auto iter = _inProgressMap.find(ns);
 
     if (iter == _inProgressMap.end()) {
-        _inProgressMap.try_emplace(ns, std::make_shared<NSLock>());
+        _inProgressMap.try_emplace(ns, std::make_shared<NSLock>(reason));
     } else {
         auto nsLock = iter->second;
         nsLock->numWaiting++;
         ScopeGuard guard([&] { nsLock->numWaiting--; });
         if (!opCtx->waitForConditionOrInterruptFor(
                 nsLock->cvLocked, lock, waitFor, [nsLock]() { return !nsLock->isInProgress; })) {
-            uasserted(ErrorCodes::LockBusy,
-                      str::stream() << "Failed to acquire dist lock " << ns << " locally");
+            using namespace fmt::literals;
+            uasserted(
+                ErrorCodes::LockBusy,
+                "Failed to acquire DDL lock for namespace '{}' after {} that is currently locked with reason '{}'"_format(
+                    ns, waitFor.toString(), reason));
         }
         guard.dismiss();
+        nsLock->reason = reason.toString();
         nsLock->isInProgress = true;
     }
 
-    return ScopedLock(ns, this);
+    LOGV2(6855301, "Acquired DDL lock", "resource"_attr = ns, "reason"_attr = reason);
+    return ScopedLock(ns, reason, this);
 }
 
-DistLockManager::ScopedLock::ScopedLock(StringData ns, DistLockManager* distLockManager)
-    : _ns(ns.toString()), _lockManager(distLockManager) {}
+DistLockManager::ScopedLock::ScopedLock(StringData ns,
+                                        StringData reason,
+                                        DistLockManager* distLockManager)
+    : _ns(ns.toString()), _reason(reason.toString()), _lockManager(distLockManager) {}
 
 DistLockManager::ScopedLock::ScopedLock(ScopedLock&& other)
-    : _ns(std::move(other._ns)), _lockManager(other._lockManager) {
+    : _ns(std::move(other._ns)),
+      _reason(std::move(other._reason)),
+      _lockManager(other._lockManager) {
     other._lockManager = nullptr;
 }
 
@@ -156,12 +165,14 @@ DistLockManager::ScopedLock::~ScopedLock() {
         auto iter = _lockManager->_inProgressMap.find(_ns);
 
         iter->second->numWaiting--;
+        iter->second->reason.clear();
         iter->second->isInProgress = false;
         iter->second->cvLocked.notify_one();
 
         if (iter->second->numWaiting == 0) {
             _lockManager->_inProgressMap.erase(_ns);
         }
+        LOGV2(6855302, "Released DDL lock", "resource"_attr = _ns, "reason"_attr = _reason);
     }
 }
 
diff --git a/src/mongo/db/s/dist_lock_manager.h b/src/mongo/db/s/dist_lock_manager.h
index 7d6bdf8379a..f9e333639ab 100644
--- a/src/mongo/db/s/dist_lock_manager.h
+++ b/src/mongo/db/s/dist_lock_manager.h
@@ -71,7 +71,7 @@ public:
         ScopedLock& operator=(const ScopedLock&) = delete;
 
     public:
-        ScopedLock(StringData lockName, DistLockManager* distLockManager);
+        ScopedLock(StringData lockName, StringData reason, DistLockManager* distLockManager);
         ~ScopedLock();
 
         ScopedLock(ScopedLock&& other);
@@ -79,9 +79,13 @@ public:
         StringData getNs() {
             return _ns;
         }
+        StringData getReason() {
+            return _reason;
+        }
 
     private:
         std::string _ns;
+        std::string _reason;
         DistLockManager* _lockManager;
     };
 
@@ -158,7 +162,10 @@ public:
     /**
      * Ensures that two dist lock within the same process will serialise with each other.
      */
-    ScopedLock lockDirectLocally(OperationContext* opCtx, StringData ns, Milliseconds waitFor);
+    ScopedLock lockDirectLocally(OperationContext* opCtx,
+                                 StringData ns,
+                                 StringData reason,
+                                 Milliseconds waitFor);
 
     /**
      * Same behavior as lock(...) above, except doesn't return a scoped object, so it is the
@@ -204,9 +211,12 @@ protected:
     const OID _lockSessionID;
 
     struct NSLock {
+        NSLock(StringData reason) : reason(reason.toString()) {}
+
         stdx::condition_variable cvLocked;
         int numWaiting = 1;
         bool isInProgress = true;
+        std::string reason;
     };
 
     Mutex _mutex = MONGO_MAKE_LATCH("NamespaceSerializer::_mutex");
diff --git a/src/mongo/db/s/sharding_ddl_coordinator.cpp b/src/mongo/db/s/sharding_ddl_coordinator.cpp
index 75475adfd78..39ef22f0254 100644
--- a/src/mongo/db/s/sharding_ddl_coordinator.cpp
+++ b/src/mongo/db/s/sharding_ddl_coordinator.cpp
@@ -215,7 +215,8 @@ ExecutorFuture<void> ShardingDDLCoordinator::_acquireLockAsync(
                    return DistLockManager::kDefaultLockTimeout;
                }();
 
-               auto distLock = distLockManager->lockDirectLocally(opCtx, resource, lockTimeOut);
+               auto distLock =
+                   distLockManager->lockDirectLocally(opCtx, resource, coorName, lockTimeOut);
 
                uassertStatusOK(distLockManager->lockDirect(opCtx, resource, coorName, lockTimeOut));
                _scopedLocks.emplace(std::move(distLock));
diff --git a/src/mongo/db/s/shardsvr_drop_indexes_command.cpp b/src/mongo/db/s/shardsvr_drop_indexes_command.cpp
index 2fe7075be30..bd1de49e3fe 100644
--- a/src/mongo/db/s/shardsvr_drop_indexes_command.cpp
+++ b/src/mongo/db/s/shardsvr_drop_indexes_command.cpp
@@ -156,8 +156,11 @@ ShardsvrDropIndexesCommand::Invocation::Response ShardsvrDropIndexesCommand::Inv
         return DistLockManager::kDefaultLockTimeout;
     }();
 
+    static constexpr StringData lockReason{"dropIndexes"_sd};
+
     auto distLockManager = DistLockManager::get(opCtx);
-    auto dbLocalLock = distLockManager->lockDirectLocally(opCtx, ns().db(), lockTimeout);
+    auto dbLocalLock =
+        distLockManager->lockDirectLocally(opCtx, ns().db(), lockReason, lockTimeout);
 
     // Check under the dbLock if this is still the primary shard for the database
     catalog_helper::assertIsPrimaryShardForDb(opCtx, ns().db());
@@ -175,7 +178,8 @@ ShardsvrDropIndexesCommand::Invocation::Response ShardsvrDropIndexesCommand::Inv
         resolvedNs = ns().makeTimeseriesBucketsNamespace();
     }
 
-    auto nsLocalLock = distLockManager->lockDirectLocally(opCtx, resolvedNs.ns(), lockTimeout);
+    auto nsLocalLock =
+        distLockManager->lockDirectLocally(opCtx, resolvedNs.ns(), lockReason, lockTimeout);
 
     StaleConfigRetryState retryState;
     return shardVersionRetry(