SERVER-72073 Ensure TTL deletes don't fall too far behind

4 files changed, 124 insertions, 5 deletions

diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp
index 88f8eb00c0a..a8cafb3fee2 100644
--- a/src/mongo/db/ttl.cpp
+++ b/src/mongo/db/ttl.cpp
@@ -95,6 +95,70 @@ std::unique_ptr<BatchedDeleteStageParams> getBatchedDeleteStageParams(bool batch
     return batchedDeleteParams;
 }
 
+AdmissionContext::Priority computeTTLPriority(
+    const UUID& uuid, const stdx::unordered_map<UUID, long long, UUID::Hash>& collSubpassHistory) {
+    if (auto it = collSubpassHistory.find(uuid); it != collSubpassHistory.end()) {
+        if (it->second >= ttlCollLowPrioritySubpassLimit.load()) {
+            return AdmissionContext::Priority::kNormal;
+        }
+    }
+    return AdmissionContext::Priority::kLow;
+}
+
+// Given the set of current TTL collections via 'ttlCollectionInfo', populates the 'ttlPriorityMap'
+// with TTL delete priority for each collection based on the 'collSubpassHistory'.
+//
+// Returns the number of collections whose TTL deletes should be executed with non-default priority
+// 'AdmissionContext::Priority::kNormal'.
+long long populateTTLPriorityMap(
+    const TTLCollectionCache::InfoMap& ttlCollectionInfo,
+    const stdx::unordered_map<UUID, long long, UUID::Hash>& collSubpassHistory,
+    stdx::unordered_map<UUID, AdmissionContext::Priority, UUID::Hash>& ttlPriorityMap) {
+    long long normalPriorityCount = 0;
+    for (const auto& [uuid, _] : ttlCollectionInfo) {
+        auto priority = computeTTLPriority(uuid, collSubpassHistory);
+        ttlPriorityMap[uuid] = priority;
+
+        if (priority == AdmissionContext::Priority::kNormal) {
+            normalPriorityCount++;
+        }
+    }
+    return normalPriorityCount;
+}
+
+AdmissionContext::Priority getTTLPriority(
+    const UUID& uuid,
+    const stdx::unordered_map<UUID, AdmissionContext::Priority, UUID::Hash>& ttlPriorityMap) {
+    auto it = ttlPriorityMap.find(uuid);
+
+    // The 'ttlPriorityMap' should contain entries for every collection in the TTLCollectionCache at
+    // the start of a subpass. If not, something went wrong during the population of the map.
+    invariant(it != ttlPriorityMap.end());
+
+    return it->second;
+}
+
+// Given 'remainingWorkAfterSubpass', updates the 'collSubpassHistory' count for each collection
+// with more work. Removes collections from 'collSubpassHistory' with no work left after the
+// subpass.
+void updateCollSubpassHistory(stdx::unordered_map<UUID, long long, UUID::Hash>& collSubpassHistory,
+                              const TTLCollectionCache::InfoMap& remainingWorkAfterSubpass) {
+    // Remove history for collections that are caught up on TTL deletes.
+    stdx::erase_if(collSubpassHistory, [&](auto&& it) {
+        auto uuid = it.first;
+        return remainingWorkAfterSubpass.find(uuid) == remainingWorkAfterSubpass.end();
+    });
+
+    // Increment the subpass count for the unexhausted collections.
+    for (const auto& [uuid, _] : remainingWorkAfterSubpass) {
+        if (auto it = collSubpassHistory.find(uuid); it != collSubpassHistory.end()) {
+            it->second++;
+        } else {
+            collSubpassHistory[uuid] = 1;
+        }
+    }
+}
+
 // Generates an expiration date based on the user-configured expireAfterSeconds. Includes special
 // 'safe' handling for time-series collections.
 Date_t safeExpirationDate(OperationContext* opCtx,
@@ -238,6 +302,10 @@ CounterMetric ttlPasses("ttl.passes");
 CounterMetric ttlSubPasses("ttl.subPasses");
 CounterMetric ttlDeletedDocuments("ttl.deletedDocuments");
 
+// Counts the subpasses over TTL collections where the deletes on a collection are increased from
+// 'low' to 'normal' priority.
+CounterMetric ttlCollSubpassesIncreasedPriority("ttl.collSubpassesIncreasedPriority");
+
 using MtabType = TenantMigrationAccessBlocker::BlockerType;
 
 TTLMonitor* TTLMonitor::get(ServiceContext* serviceCtx) {
@@ -320,24 +388,30 @@ void TTLMonitor::shutdown() {
 void TTLMonitor::_doTTLPass() {
     const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext();
     OperationContext* opCtx = opCtxPtr.get();
-    SetAdmissionPriorityForLock priority(opCtx, AdmissionContext::Priority::kLow);
 
     hangTTLMonitorBetweenPasses.pauseWhileSet(opCtx);
 
     // Increment the metric after the TTL work has been finished.
     ON_BLOCK_EXIT([&] { ttlPasses.increment(); });
 
+    // Tracks the number of consecutive subpasses that have failed to exhaust a collection of TTL
+    // deletes. If a collection incurs 'ttlCollLowPrioritySubpassLimit', then all TTL deletes on the
+    // collection are executed at 'normal' priority until there are no TTL deletes remaining on the
+    // collection.
+    stdx::unordered_map<UUID, long long, UUID::Hash> collSubpassHistory;
+
     bool moreToDelete = true;
     while (moreToDelete) {
         // Sub-passes may not delete all documents in the interest of fairness. If a sub-pass
         // indicates that it did not delete everything possible, we continue performing sub-passes.
         // This maintains the semantic that a full TTL pass deletes everything it possibly can
         // before sleeping periodically.
-        moreToDelete = _doTTLSubPass(opCtx);
+        moreToDelete = _doTTLSubPass(opCtx, collSubpassHistory);
     }
 }
 
-bool TTLMonitor::_doTTLSubPass(OperationContext* opCtx) {
+bool TTLMonitor::_doTTLSubPass(
+    OperationContext* opCtx, stdx::unordered_map<UUID, long long, UUID::Hash>& collSubpassHistory) {
     // If part of replSet but not in a readable state (e.g. during initial sync), skip.
     if (repl::ReplicationCoordinator::get(opCtx)->getReplicationMode() ==
             repl::ReplicationCoordinator::modeReplSet &&
@@ -352,6 +426,16 @@ bool TTLMonitor::_doTTLSubPass(OperationContext* opCtx) {
     // during a long running pass.
     TTLCollectionCache::InfoMap work = ttlCollectionCache.getTTLInfos();
 
+    // Before the subpass begins work, compute the priority at which TTL deletes should be executed
+    // on each collection. By default, TTL deletes are 'low' priority. Only collections where TTL
+    // deletes have fallen behind over several subpasses are promoted to 'normal' priority TTL
+    // deletes.
+    stdx::unordered_map<UUID, AdmissionContext::Priority, UUID::Hash> ttlPriorityMap;
+    auto numNormalPriorityCollections =
+        populateTTLPriorityMap(work, collSubpassHistory, ttlPriorityMap);
+
+    ttlCollSubpassesIncreasedPriority.increment(numNormalPriorityCollections);
+
     // When batching is enabled, _doTTLIndexDelete will limit the amount of work it
     // performs in both time and the number of documents it deletes. If it reaches one
     // of these limits on an index, it will return moreToDelete as true, and we will
@@ -364,6 +448,13 @@ bool TTLMonitor::_doTTLSubPass(OperationContext* opCtx) {
     do {
         TTLCollectionCache::InfoMap moreWork;
         for (const auto& [uuid, infos] : work) {
+            // If there are multiple TTL indexes on a TTL collection, and any of those have fallen
+            // behind TTL inserts over consecutive subpasses, raising the priority to
+            // 'AdmissionContext::Priority::kNormal' for one index means the priority will be
+            // 'normal' for all indexes.
+            AdmissionContext::Priority priority = getTTLPriority(uuid, ttlPriorityMap);
+            SetAdmissionPriorityForLock priorityGuard(opCtx, priority);
+
             for (const auto& info : infos) {
                 bool moreToDelete = _doTTLIndexDelete(opCtx, &ttlCollectionCache, uuid, info);
                 if (moreToDelete) {
@@ -376,6 +467,8 @@ bool TTLMonitor::_doTTLSubPass(OperationContext* opCtx) {
     } while (!work.empty() &&
              Seconds(timer.seconds()) < Seconds(ttlMonitorSubPassTargetSecs.load()));
 
+    updateCollSubpassHistory(collSubpassHistory, work);
+
     // More work signals there may more expired documents to visit.
     return !work.empty();
 }
diff --git a/src/mongo/db/ttl.h b/src/mongo/db/ttl.h
index ccd6002983d..be0cc3145ab 100644
--- a/src/mongo/db/ttl.h
+++ b/src/mongo/db/ttl.h
@@ -89,9 +89,15 @@ private:
      * Once it is confirmed there are no more expired documents on an index, the index will not be
      * visited again for the remainder of the sub-pass.
      *
+     * The 'collSubpassHistory' tracks the number of consecutive subpasses on a collection that
+     * completed with more expired documents remaining. It is used to determine when a collection's
+     * TTL deletes should be raised from 'low' to 'normal' priority to prevent TTL deletes from
+     * falling behind on TTL inserts.
+     *
      * Returns true if there are more expired documents to delete. False otherwise.
      */
-    bool _doTTLSubPass(OperationContext* opCtx);
+    bool _doTTLSubPass(OperationContext* opCtx,
+                       stdx::unordered_map<UUID, long long, UUID::Hash>& collSubpassHistory);
 
     /**
      * Given a TTL index, attempts to delete all expired documents through the index until
diff --git a/src/mongo/db/ttl.idl b/src/mongo/db/ttl.idl
index 184cc990fd7..bfc96c06909 100644
--- a/src/mongo/db/ttl.idl
+++ b/src/mongo/db/ttl.idl
@@ -93,3 +93,15 @@ server_parameters:
         validator:
             gte: 0
 
+    ttlCollLowPrioritySubpassLimit:
+        description:
+            "By default, TTL deletes are 'low' priority. Limits the number of consecutive subpasses
+            completed without exhausting TTL deletes on a collection before the TTL deletes on the
+            collection are raised to 'normal' priority."
+        set_at: [ startup, runtime ]
+        cpp_vartype: AtomicWord<int>
+        cpp_varname: ttlCollLowPrioritySubpassLimit
+        default: 10
+        validator:
+            gt: 0
+
diff --git a/src/mongo/db/ttl_test.cpp b/src/mongo/db/ttl_test.cpp
index aaa22316077..63e2ae50524 100644
--- a/src/mongo/db/ttl_test.cpp
+++ b/src/mongo/db/ttl_test.cpp
@@ -86,7 +86,8 @@ protected:
 
     bool doTTLSubPassForTest(OperationContext* opCtx) {
         TTLMonitor* ttlMonitor = TTLMonitor::get(getGlobalServiceContext());
-        return ttlMonitor->_doTTLSubPass(opCtx);
+
+        return ttlMonitor->_doTTLSubPass(opCtx, _collSubpassHistoryPlaceHolder);
     }
 
     long long getTTLPasses() {
@@ -122,6 +123,13 @@ protected:
 
 private:
     ServiceContext::UniqueOperationContext _opCtx;
+
+    // In a given TTL Pass, there may be multiple subpasses. Between subpasses, collection delete
+    // history is tracked for collections who have remaining TTL deletes a subpass. The
+    // history is strictly for TTL delete priority purposes in a contended system. It does not
+    // impact behavior in an uncontended, isolated system such as this test suite. This is used
+    // strictly as a placeholder in order to test subpass behavior.
+    stdx::unordered_map<UUID, long long, UUID::Hash> _collSubpassHistoryPlaceHolder;
 };
 
 namespace {