SERVER-45367 Log open cursors for a namespace when submitting tasks to the range deleter

10 files changed, 143 insertions, 15 deletions

diff --git a/src/mongo/db/cursor_manager.cpp b/src/mongo/db/cursor_manager.cpp
index 9349e69e457..61b37f886c0 100644
--- a/src/mongo/db/cursor_manager.cpp
+++ b/src/mongo/db/cursor_manager.cpp
@@ -671,6 +671,22 @@ void CursorManager::appendActiveCursors(std::vector<GenericCursor>* cursors) con
     }
 }
 
+std::vector<CursorId> CursorManager::getCursorIdsForNamespace(const NamespaceString& nss) const {
+    std::vector<CursorId> cursors;
+
+    auto allPartitions = _cursorMap->lockAllPartitions();
+    for (auto&& partition : allPartitions) {
+        for (auto&& entry : partition) {
+            auto cursor = entry.second;
+            if (cursor->nss() == nss) {
+                cursors.emplace_back(cursor->cursorid());
+            }
+        }
+    }
+
+    return cursors;
+}
+
 stdx::unordered_set<CursorId> CursorManager::getCursorsForSession(LogicalSessionId lsid) const {
     stdx::unordered_set<CursorId> cursors;
 
diff --git a/src/mongo/db/cursor_manager.h b/src/mongo/db/cursor_manager.h
index e500dfd3034..54598173c12 100644
--- a/src/mongo/db/cursor_manager.h
+++ b/src/mongo/db/cursor_manager.h
@@ -205,6 +205,11 @@ public:
      */
     void appendActiveCursors(std::vector<GenericCursor>* cursors) const;
 
+    /**
+     * Returns a vector of all open cursors for the given namespace.
+     */
+    std::vector<CursorId> getCursorIdsForNamespace(const NamespaceString& nss) const;
+
     /*
      * Returns a list of all open cursors for the given session.
      */
diff --git a/src/mongo/db/s/cleanup_orphaned_cmd.cpp b/src/mongo/db/s/cleanup_orphaned_cmd.cpp
index c1676b79b18..f5e947fa959 100644
--- a/src/mongo/db/s/cleanup_orphaned_cmd.cpp
+++ b/src/mongo/db/s/cleanup_orphaned_cmd.cpp
@@ -112,7 +112,8 @@ CleanupResult cleanupOrphanedData(OperationContext* opCtx,
 
         *stoppedAtKey = targetRange->getMax();
 
-        notifn = css->cleanUpRange(*targetRange, CollectionShardingRuntime::kNow);
+        notifn = css->cleanUpRange(
+            opCtx, autoColl.getCollection(), *targetRange, CollectionShardingRuntime::kNow);
     }
 
     // Sleep waiting for our own deletion. We don't actually care about any others, so there is no
diff --git a/src/mongo/db/s/collection_sharding_runtime.cpp b/src/mongo/db/s/collection_sharding_runtime.cpp
index 6fc514ca2a7..f39326d48e5 100644
--- a/src/mongo/db/s/collection_sharding_runtime.cpp
+++ b/src/mongo/db/s/collection_sharding_runtime.cpp
@@ -111,11 +111,13 @@ void CollectionShardingRuntime::forgetReceive(const ChunkRange& range) {
     _metadataManager->forgetReceive(range);
 }
 
-auto CollectionShardingRuntime::cleanUpRange(ChunkRange const& range, CleanWhen when)
-    -> CleanupNotification {
+auto CollectionShardingRuntime::cleanUpRange(OperationContext* opCtx,
+                                             const Collection* collection,
+                                             ChunkRange const& range,
+                                             CleanWhen when) -> CleanupNotification {
     Date_t time =
         (when == kNow) ? Date_t{} : Date_t::now() + Seconds(orphanCleanupDelaySecs.load());
-    return _metadataManager->cleanUpRange(range, time);
+    return _metadataManager->cleanUpRange(opCtx, collection, range, time);
 }
 
 Status CollectionShardingRuntime::waitForClean(OperationContext* opCtx,
diff --git a/src/mongo/db/s/collection_sharding_runtime.h b/src/mongo/db/s/collection_sharding_runtime.h
index fe678268ca6..3544accb8b9 100644
--- a/src/mongo/db/s/collection_sharding_runtime.h
+++ b/src/mongo/db/s/collection_sharding_runtime.h
@@ -108,7 +108,10 @@ public:
      * result.abandon(), instead of waitStatus, to ignore the outcome.
      */
     enum CleanWhen { kNow, kDelayed };
-    CleanupNotification cleanUpRange(ChunkRange const& range, CleanWhen when);
+    CleanupNotification cleanUpRange(OperationContext* opCtx,
+                                     const Collection* collection,
+                                     ChunkRange const& range,
+                                     CleanWhen when);
 
     /**
      * Tracks deletion of any documents within the range, returning when deletion is complete.
diff --git a/src/mongo/db/s/metadata_manager.cpp b/src/mongo/db/s/metadata_manager.cpp
index 836370d5d0e..2c16eb5e051 100644
--- a/src/mongo/db/s/metadata_manager.cpp
+++ b/src/mongo/db/s/metadata_manager.cpp
@@ -37,6 +37,8 @@
 #include "mongo/base/string_data.h"
 #include "mongo/bson/simple_bsonobj_comparator.h"
 #include "mongo/bson/util/builder.h"
+#include "mongo/db/cursor_manager.h"
+#include "mongo/db/db_raii.h"
 #include "mongo/db/query/internal_plans.h"
 #include "mongo/db/range_arithmetic.h"
 #include "mongo/db/s/collection_sharding_state.h"
@@ -163,6 +165,49 @@ void scheduleCleanup(executor::TaskExecutor* executor,
     }
 }
 
+void logRangeDeletionWaitingOnOpenCursors(const OperationContext* opCtx,
+                                          const Collection* collection,
+                                          const NamespaceString& nss,
+                                          const ChunkRange& range) {
+    invariant(opCtx->lockState()->isCollectionLockedForMode(nss.toString(), MODE_IS));
+    std::vector<CursorId> cursorIds;
+    // If the collection exists, gather a list of all cursors related to the collection.
+    if (collection) {
+        auto cursorIdsFromCollectionCursorManager =
+            collection->getCursorManager()->getCursorIdsForNamespace(nss);
+        cursorIds.insert(cursorIds.end(),
+                         cursorIdsFromCollectionCursorManager.begin(),
+                         cursorIdsFromCollectionCursorManager.end());
+
+        // Aggregation cursors are registered on the global cursor manager. A cursor on
+        // the global cursor manager can involve any number of collections, but is registered with
+        // the namespace of the aggregate command. This works well for this purpose since any other
+        // namespaces would come through a $lookup or $graphLookup which cannot read from sharded
+        // collections and so could not be contributing to the delay of a range deletion. The
+        // namespace of the aggregate command can be sharded, so we do want to include those
+        // aggregation cursors in this message.
+        auto cursorIdsFromGlobalCursorManager =
+            CursorManager::getGlobalCursorManager()->getCursorIdsForNamespace(nss);
+        cursorIds.insert(cursorIds.end(),
+                         cursorIdsFromGlobalCursorManager.begin(),
+                         cursorIdsFromGlobalCursorManager.end());
+    }
+
+    // Join cursorIds as a comma-separated list.
+    std::string cursorIdList =
+        cursorIds.empty() ? "" : std::accumulate(std::next(cursorIds.begin()),
+                                                 cursorIds.end(),
+                                                 std::to_string(cursorIds[0]),
+                                                 [](std::string a, CursorId b) {
+                                                     return std::move(a) + ',' + std::to_string(b);
+                                                 });
+
+    log() << "Deletion of " << nss.ns() << " range " << redact(range.toString())
+          << " will be scheduled after all possibly dependent queries finish. "
+             "All open cursors for namespace: ["
+          << cursorIdList << "]";
+}
+
 }  // namespace
 
 class RangePreserver : public ScopedCollectionMetadata::Impl {
@@ -487,8 +532,10 @@ void MetadataManager::forgetReceive(ChunkRange const& range) {
     _pushRangeToClean(lg, range, Date_t{}).abandon();
 }
 
-auto MetadataManager::cleanUpRange(ChunkRange const& range, Date_t whenToDelete)
-    -> CleanupNotification {
+auto MetadataManager::cleanUpRange(OperationContext* opCtx,
+                                   const Collection* collection,
+                                   ChunkRange const& range,
+                                   Date_t whenToDelete) -> CleanupNotification {
     stdx::lock_guard<stdx::mutex> lg(_managerLock);
     invariant(!_metadata.empty());
 
@@ -514,8 +561,7 @@ auto MetadataManager::cleanUpRange(ChunkRange const& range, Date_t whenToDelete)
         return _pushRangeToClean(lg, range, whenToDelete);
     }
 
-    log() << "Deletion of " << _nss.ns() << " range " << redact(range.toString())
-          << " will be scheduled after all possibly dependent queries finish";
+    logRangeDeletionWaitingOnOpenCursors(opCtx, collection, _nss, range);
 
     // Put it on the oldest metadata permissible; the current one might live a long time.
     auto& orphans = overlapMetadata->orphans;
diff --git a/src/mongo/db/s/metadata_manager.h b/src/mongo/db/s/metadata_manager.h
index ba83f3e6ada..722d9d56c22 100644
--- a/src/mongo/db/s/metadata_manager.h
+++ b/src/mongo/db/s/metadata_manager.h
@@ -122,7 +122,10 @@ public:
      *
      * Call waitStatus(opCtx) on the result to wait for the deletion to complete or fail.
      */
-    CleanupNotification cleanUpRange(ChunkRange const& range, Date_t whenToDelete);
+    CleanupNotification cleanUpRange(OperationContext* opCtx,
+                                     const Collection* collection,
+                                     ChunkRange const& range,
+                                     Date_t whenToDelete);
 
     /**
      * Returns the number of ranges scheduled to be cleaned, exclusive of such ranges that might
diff --git a/src/mongo/db/s/metadata_manager_test.cpp b/src/mongo/db/s/metadata_manager_test.cpp
index 29810e0d2da..6ce0a6ac81c 100644
--- a/src/mongo/db/s/metadata_manager_test.cpp
+++ b/src/mongo/db/s/metadata_manager_test.cpp
@@ -66,6 +66,13 @@ protected:
     void setUp() override {
         ShardServerTestFixture::setUp();
         _manager = std::make_shared<MetadataManager>(getServiceContext(), kNss, executor());
+        _autoColl.emplace(operationContext(), kNss, MODE_IS);
+    }
+
+    void tearDown() override {
+        _autoColl.reset();
+        _manager.reset();
+        ShardServerTestFixture::tearDown();
     }
 
     /**
@@ -154,7 +161,12 @@ protected:
         return CollectionMetadata(std::make_shared<ChunkManager>(rt, boost::none), kThisShard);
     }
 
+    Collection* collection() {
+        return _autoColl->getCollection();
+    }
+
     std::shared_ptr<MetadataManager> _manager;
+    boost::optional<AutoGetCollection> _autoColl;
 };
 
 TEST_F(MetadataManagerTest, InitialMetadataIsUnknown) {
@@ -208,7 +220,7 @@ TEST_F(MetadataManagerTest, AddRangeNotificationsBlockAndYield) {
 
     ChunkRange cr1(BSON("key" << 0), BSON("key" << 10));
 
-    auto notifn1 = _manager->cleanUpRange(cr1, Date_t{});
+    auto notifn1 = _manager->cleanUpRange(operationContext(), collection(), cr1, Date_t{});
     ASSERT_FALSE(notifn1.ready());
     ASSERT_EQ(_manager->numberOfRangesToClean(), 1UL);
 
@@ -253,7 +265,7 @@ TEST_F(MetadataManagerTest, NotificationBlocksUntilDeletion) {
         ASSERT_EQ(_manager->numberOfMetadataSnapshots(), 0UL);
         ASSERT_EQ(_manager->numberOfRangesToClean(), 0UL);
 
-        optNotif = _manager->cleanUpRange(cr1, Date_t{});
+        optNotif = _manager->cleanUpRange(operationContext(), collection(), cr1, Date_t{});
         ASSERT(optNotif);
         ASSERT_EQ(_manager->numberOfMetadataSnapshots(), 0UL);
         ASSERT_EQ(_manager->numberOfRangesToClean(), 1UL);
@@ -291,7 +303,7 @@ TEST_F(MetadataManagerTest, CleanupNotificationsAreSignaledOnDropAndRecreate) {
     _manager->setFilteringMetadata(
         cloneMetadataMinusChunk(*_manager->getActiveMetadata(_manager, boost::none), rangeToClean));
 
-    auto notif = _manager->cleanUpRange(rangeToClean, Date_t{});
+    auto notif = _manager->cleanUpRange(operationContext(), collection(), rangeToClean, Date_t{});
     ASSERT(!notif.ready());
 
     auto optNotif = _manager->trackOrphanedDataCleanup(rangeToClean);
@@ -386,7 +398,7 @@ TEST_F(MetadataManagerTest, RangesToCleanMembership) {
 
     ASSERT_EQ(0UL, _manager->numberOfRangesToClean());
 
-    auto notifn = _manager->cleanUpRange(cr, Date_t{});
+    auto notifn = _manager->cleanUpRange(operationContext(), collection(), cr, Date_t{});
     ASSERT(!notifn.ready());
     ASSERT_EQ(1UL, _manager->numberOfRangesToClean());
 
diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp
index 15ae251d65f..4fa8fade623 100644
--- a/src/mongo/db/s/migration_source_manager.cpp
+++ b/src/mongo/db/s/migration_source_manager.cpp
@@ -563,7 +563,8 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* opC
                                                           : CollectionShardingRuntime::kDelayed;
         UninterruptibleLockGuard noInterrupt(opCtx->lockState());
         AutoGetCollection autoColl(opCtx, getNss(), MODE_IS);
-        return CollectionShardingRuntime::get(opCtx, getNss())->cleanUpRange(range, whenToClean);
+        return CollectionShardingRuntime::get(opCtx, getNss())
+            ->cleanUpRange(opCtx, autoColl.getCollection(), range, whenToClean);
     }();
 
     if (!MONGO_FAIL_POINT(doNotRefreshRecipientAfterCommit)) {
diff --git a/src/mongo/dbtests/cursor_manager_test.cpp b/src/mongo/dbtests/cursor_manager_test.cpp
index af26c405b2f..6771833260e 100644
--- a/src/mongo/dbtests/cursor_manager_test.cpp
+++ b/src/mongo/dbtests/cursor_manager_test.cpp
@@ -683,5 +683,44 @@ TEST_F(CursorManagerTestCustomOpCtx, MultipleCursorsMultipleSessions) {
     ASSERT(cursors2.find(cursor2) != cursors2.end());
 }
 
+TEST_F(CursorManagerTestCustomOpCtx,
+       GetCursorIdsForNamespaceReturnsSingleEntryForMatchingNamespace) {
+    auto opCtx = _queryServiceContext->makeOperationContext();
+    auto pinned = makeCursor(opCtx.get());
+    auto cursorId = pinned.getCursor()->cursorid();
+    auto cursorsForNamespace = useCursorManager()->getCursorIdsForNamespace(kTestNss);
+    ASSERT_EQUALS(cursorsForNamespace.size(), 1ull);
+    ASSERT_EQUALS(cursorsForNamespace[0], cursorId);
+}
+
+TEST_F(CursorManagerTestCustomOpCtx,
+       GetCursorIdsForNamespaceReturnsMultipleEntriesForMatchingNamespace) {
+    auto opCtx = _queryServiceContext->makeOperationContext();
+    auto pinned1 = makeCursor(opCtx.get());
+    auto pinned2 = makeCursor(opCtx.get());
+    auto cursorId1 = pinned1.getCursor()->cursorid();
+    auto cursorId2 = pinned2.getCursor()->cursorid();
+    auto cursorsForNamespace = useCursorManager()->getCursorIdsForNamespace(kTestNss);
+    ASSERT_EQUALS(cursorsForNamespace.size(), 2ull);
+
+    // The results for cursorsForNamespace won't necessarily be the same as the order of insertion.
+    std::set<CursorId> cursorsForNamespaceSet(cursorsForNamespace.begin(),
+                                              cursorsForNamespace.end());
+
+    ASSERT_EQUALS(cursorsForNamespaceSet.count(cursorId1), 1ull);
+    ASSERT_EQUALS(cursorsForNamespaceSet.count(cursorId2), 1ull);
+}
+
+TEST_F(CursorManagerTestCustomOpCtx,
+       GetCursorIdsForNamespaceDoesNotReturnEntriesForNonMatchingNamespace) {
+    auto opCtx = _queryServiceContext->makeOperationContext();
+    // Add a cursor for kTestNss.
+    auto pinned = makeCursor(opCtx.get());
+    // Get cursors for a different NamespaceString.
+    auto cursorsForNamespace =
+        useCursorManager()->getCursorIdsForNamespace(NamespaceString("somerandom.nss"));
+    ASSERT_EQUALS(cursorsForNamespace.size(), 0ull);
+}
+
 }  // namespace
 }  // namespace mongo