SERVER-68477 include 'expireAfterSeconds' type information when registering TTL indexes with the TTLCollectionCache

(cherry picked from commit cc3ae631bce7943fbda5182ff3b9d93d1125be40)

9 files changed, 157 insertions, 39 deletions

diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp
index 8af0ba8efc9..21b5d6dc920 100644
--- a/src/mongo/db/catalog/coll_mod.cpp
+++ b/src/mongo/db/catalog/coll_mod.cpp
@@ -596,7 +596,8 @@ void _setClusteredExpireAfterSeconds(
                 if (!oldExpireAfterSeconds) {
                     auto ttlCache = &TTLCollectionCache::get(opCtx->getServiceContext());
                     opCtx->recoveryUnit()->onCommit([ttlCache, uuid = coll->uuid()](auto _) {
-                        ttlCache->registerTTLInfo(uuid, TTLCollectionCache::ClusteredId());
+                        ttlCache->registerTTLInfo(
+                            uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}});
                     });
                 }
 
diff --git a/src/mongo/db/catalog/coll_mod_index.cpp b/src/mongo/db/catalog/coll_mod_index.cpp
index d56acc639f4..2ffecf7f1e1 100644
--- a/src/mongo/db/catalog/coll_mod_index.cpp
+++ b/src/mongo/db/catalog/coll_mod_index.cpp
@@ -67,8 +67,10 @@ void _processCollModIndexRequestExpireAfterSeconds(OperationContext* opCtx,
         // Do not refer to 'idx' within this commit handler as it may be be invalidated by
         // IndexCatalog::refreshEntry().
         opCtx->recoveryUnit()->onCommit(
-            [ttlCache, uuid = coll->uuid(), indexName = idx->indexName()](auto _) {
-                ttlCache->registerTTLInfo(uuid, indexName);
+            [ttlCache, uuid = coll->uuid(), indexName = idx->indexName(), indexExpireAfterSeconds](
+                auto _) {
+                ttlCache->registerTTLInfo(
+                    uuid, TTLCollectionCache::Info{indexName, /*isExpireAfterSecondsNaN=*/false});
             });
 
         // Change the value of "expireAfterSeconds" on disk.
@@ -77,6 +79,29 @@ void _processCollModIndexRequestExpireAfterSeconds(OperationContext* opCtx,
         return;
     }
 
+    // If the current `expireAfterSeconds` is NaN, it can never be equal to
+    // 'indexExpireAfterSeconds'.
+    if (oldExpireSecsElement.isNaN()) {
+        // Setting *oldExpireSecs is mostly for informational purposes.
+        // We could also use index_key_validate::kExpireAfterSecondsForInactiveTTLIndex but
+        // 0 is more consistent with the previous safeNumberLong() behavior and avoids potential
+        // showing the same value for the new and old values in the collMod response.
+        *oldExpireSecs = 0;
+
+        // Change the value of "expireAfterSeconds" on disk.
+        autoColl->getWritableCollection(opCtx)->updateTTLSetting(
+            opCtx, idx->indexName(), indexExpireAfterSeconds);
+
+        // Keep the TTL information maintained by the TTLCollectionCache in sync so that we don't
+        // try to fix up the TTL index during the next step-up.
+        auto ttlCache = &TTLCollectionCache::get(opCtx->getServiceContext());
+        const auto& coll = autoColl->getCollection();
+        opCtx->recoveryUnit()->onCommit(
+            [ttlCache, uuid = coll->uuid(), indexName = idx->indexName(), indexExpireAfterSeconds](
+                auto _) { ttlCache->unsetTTLIndexExpireAfterSecondsNaN(uuid, indexName); });
+        return;
+    }
+
     // This collection is already TTL. Compare the requested value against the existing setting
     // before updating the catalog.
     *oldExpireSecs = oldExpireSecsElement.safeNumberLong();
diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp
index 757f653b517..d6912c7e361 100644
--- a/src/mongo/db/catalog/collection_impl.cpp
+++ b/src/mongo/db/catalog/collection_impl.cpp
@@ -541,11 +541,11 @@ void CollectionImpl::init(OperationContext* opCtx) {
             if (opCtx->lockState()->inAWriteUnitOfWork()) {
                 opCtx->recoveryUnit()->onCommit([svcCtx, uuid](auto ts) {
                     TTLCollectionCache::get(svcCtx).registerTTLInfo(
-                        uuid, TTLCollectionCache::ClusteredId{});
+                        uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}});
                 });
             } else {
-                TTLCollectionCache::get(svcCtx).registerTTLInfo(uuid,
-                                                                TTLCollectionCache::ClusteredId{});
+                TTLCollectionCache::get(svcCtx).registerTTLInfo(
+                    uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}});
             }
         }
     }
diff --git a/src/mongo/db/catalog/index_build_block.cpp b/src/mongo/db/catalog/index_build_block.cpp
index 6174e891b64..50f8331ee86 100644
--- a/src/mongo/db/catalog/index_build_block.cpp
+++ b/src/mongo/db/catalog/index_build_block.cpp
@@ -274,7 +274,10 @@ void IndexBuildBlock::success(OperationContext* opCtx, Collection* collection) {
             // Note that TTL deletion is supported on capped clustered collections via bounded
             // collection scan, which does not use an index.
             if (spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName) && !coll->isCapped()) {
-                TTLCollectionCache::get(svcCtx).registerTTLInfo(coll->uuid(), indexName);
+                TTLCollectionCache::get(svcCtx).registerTTLInfo(
+                    coll->uuid(),
+                    TTLCollectionCache::Info{
+                        indexName, spec[IndexDescriptor::kExpireAfterSecondsFieldName].isNaN()});
             }
         });
 }
diff --git a/src/mongo/db/catalog/index_catalog_impl.cpp b/src/mongo/db/catalog/index_catalog_impl.cpp
index 6a9f4c46896..35a3a74c3b0 100644
--- a/src/mongo/db/catalog/index_catalog_impl.cpp
+++ b/src/mongo/db/catalog/index_catalog_impl.cpp
@@ -211,7 +211,10 @@ Status IndexCatalogImpl::init(OperationContext* opCtx, Collection* collection) {
         if (spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName) &&
             !collection->isCapped()) {
             TTLCollectionCache::get(opCtx->getServiceContext())
-                .registerTTLInfo(collection->uuid(), indexName);
+                .registerTTLInfo(
+                    collection->uuid(),
+                    TTLCollectionCache::Info{
+                        indexName, spec[IndexDescriptor::kExpireAfterSecondsFieldName].isNaN()});
         }
 
         bool ready = collection->isIndexReady(indexName);
diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp
index ee9b801882a..32ec347d525 100644
--- a/src/mongo/db/ttl.cpp
+++ b/src/mongo/db/ttl.cpp
@@ -211,7 +211,11 @@ private:
                 // The collection was dropped.
                 auto nss = collectionCatalog->lookupNSSByUUID(opCtx, uuid);
                 if (!nss) {
-                    ttlCollectionCache.deregisterTTLInfo(uuid, info);
+                    if (info.isClustered()) {
+                        ttlCollectionCache.deregisterTTLClusteredIndex(uuid);
+                    } else {
+                        ttlCollectionCache.deregisterTTLIndexByName(uuid, info.getIndexName());
+                    }
                     continue;
                 }
 
@@ -328,15 +332,11 @@ private:
         ResourceConsumption::ScopedMetricsCollector scopedMetrics(opCtx, nss.db().toString());
 
         const auto& collection = coll.getCollection();
-        stdx::visit(
-            visit_helper::Overloaded{
-                [&](const TTLCollectionCache::ClusteredId&) {
-                    deleteExpiredWithCollscan(opCtx, ttlCollectionCache, collection);
-                },
-                [&](const TTLCollectionCache::IndexName& indexName) {
-                    deleteExpiredWithIndex(opCtx, ttlCollectionCache, collection, indexName);
-                }},
-            info);
+        if (info.isClustered()) {
+            deleteExpiredWithCollscan(opCtx, ttlCollectionCache, collection);
+        } else {
+            deleteExpiredWithIndex(opCtx, ttlCollectionCache, collection, info.getIndexName());
+        }
     }
 
     /**
@@ -369,13 +369,13 @@ private:
                                 const CollectionPtr& collection,
                                 std::string indexName) {
         if (!collection->isIndexPresent(indexName)) {
-            ttlCollectionCache->deregisterTTLInfo(collection->uuid(), indexName);
+            ttlCollectionCache->deregisterTTLIndexByName(collection->uuid(), indexName);
             return;
         }
 
         BSONObj spec = collection->getIndexSpec(indexName);
         if (!spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName)) {
-            ttlCollectionCache->deregisterTTLInfo(collection->uuid(), indexName);
+            ttlCollectionCache->deregisterTTLIndexByName(collection->uuid(), indexName);
             return;
         }
 
@@ -525,8 +525,7 @@ private:
 
         auto expireAfterSeconds = collOptions.expireAfterSeconds;
         if (!expireAfterSeconds) {
-            ttlCollectionCache->deregisterTTLInfo(collection->uuid(),
-                                                  TTLCollectionCache::ClusteredId{});
+            ttlCollectionCache->deregisterTTLClusteredIndex(collection->uuid());
             return;
         }
 
diff --git a/src/mongo/db/ttl_collection_cache.cpp b/src/mongo/db/ttl_collection_cache.cpp
index 27d1335510c..86fcc682068 100644
--- a/src/mongo/db/ttl_collection_cache.cpp
+++ b/src/mongo/db/ttl_collection_cache.cpp
@@ -63,13 +63,30 @@ void TTLCollectionCache::registerTTLInfo(UUID uuid, const Info& info) {
     }
 }
 
-void TTLCollectionCache::deregisterTTLInfo(UUID uuid, const Info& info) {
+void TTLCollectionCache::_deregisterTTLInfo(UUID uuid, const Info& info) {
     stdx::lock_guard<Latch> lock(_ttlInfosLock);
     auto infoIt = _ttlInfos.find(uuid);
     fassert(5400705, infoIt != _ttlInfos.end());
     auto& [_, infoVec] = *infoIt;
 
-    auto iter = std::find(infoVec.begin(), infoVec.end(), info);
+    auto iter = infoVec.begin();
+    if (info.isClustered()) {
+        // For clustered collections, we cannot have more than one clustered info per UUID.
+        // All we have to do here is ensure that the 'info' to search for is also 'clustered'.
+        iter = std::find_if(infoVec.begin(), infoVec.end(), [](const auto& infoVecItem) {
+            return infoVecItem.isClustered();
+        });
+    } else {
+        // For TTL indexes, we search non-clustered TTL info items on the index name only.
+        auto indexName = info.getIndexName();
+        iter = std::find_if(infoVec.begin(), infoVec.end(), [&indexName](const auto& infoVecItem) {
+            if (infoVecItem.isClustered()) {
+                return false;
+            }
+            return indexName == infoVecItem.getIndexName();
+        });
+    }
+
     fassert(40220, iter != infoVec.end());
     infoVec.erase(iter);
     if (infoVec.empty()) {
@@ -77,6 +94,31 @@ void TTLCollectionCache::deregisterTTLInfo(UUID uuid, const Info& info) {
     }
 }
 
+void TTLCollectionCache::deregisterTTLIndexByName(UUID uuid, const IndexName& indexName) {
+    _deregisterTTLInfo(std::move(uuid), TTLCollectionCache::Info{indexName, /*unusedSpec=*/{}});
+}
+
+void TTLCollectionCache::deregisterTTLClusteredIndex(UUID uuid) {
+    _deregisterTTLInfo(std::move(uuid),
+                       TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}});
+}
+
+void TTLCollectionCache::unsetTTLIndexExpireAfterSecondsNaN(UUID uuid, const IndexName& indexName) {
+    stdx::lock_guard<Latch> lock(_ttlInfosLock);
+    auto infoIt = _ttlInfos.find(uuid);
+    if (infoIt == _ttlInfos.end()) {
+        return;
+    }
+
+    auto&& infoVec = infoIt->second;
+    for (auto&& info : infoVec) {
+        if (!info.isClustered() && info.getIndexName() == indexName) {
+            info.unsetExpireAfterSecondsNaN();
+            break;
+        }
+    }
+}
+
 TTLCollectionCache::InfoMap TTLCollectionCache::getTTLInfos() {
     stdx::lock_guard<Latch> lock(_ttlInfosLock);
     return _ttlInfos;
diff --git a/src/mongo/db/ttl_collection_cache.h b/src/mongo/db/ttl_collection_cache.h
index 8c70ce9f8d8..a8f00b6c6a6 100644
--- a/src/mongo/db/ttl_collection_cache.h
+++ b/src/mongo/db/ttl_collection_cache.h
@@ -54,16 +54,52 @@ public:
     using IndexName = std::string;
 
     // Specifies how a collection should expire data with TTL.
-    using Info = stdx::variant<ClusteredId, IndexName>;
+    class Info {
+    public:
+        explicit Info(ClusteredId) : _isClustered(true), _isExpireAfterSecondsNaN(false) {}
+        Info(IndexName indexName, bool isExpireAfterSecondsNaN)
+            : _isClustered(false),
+              _indexName(std::move(indexName)),
+              _isExpireAfterSecondsNaN(isExpireAfterSecondsNaN) {}
+        bool isClustered() const {
+            return _isClustered;
+        }
+        IndexName getIndexName() const {
+            return _indexName;
+        }
+        bool isExpireAfterSecondsNaN() const {
+            return _isExpireAfterSecondsNaN;
+        }
+        void unsetExpireAfterSecondsNaN() {
+            _isExpireAfterSecondsNaN = false;
+        }
+
+    private:
+        bool _isClustered;
+        IndexName _indexName;
+        bool _isExpireAfterSecondsNaN;
+    };
 
     // Caller is responsible for ensuring no duplicates are registered.
     void registerTTLInfo(UUID uuid, const Info& info);
-    void deregisterTTLInfo(UUID uuid, const Info& info);
+    void deregisterTTLIndexByName(UUID uuid, const IndexName& indexName);
+    void deregisterTTLClusteredIndex(UUID uuid);
+
+    /**
+     * Resets expireAfterSeconds flag on TTL index.
+     * For idempotency, this has no effect if index is not found.
+     */
+    void unsetTTLIndexExpireAfterSecondsNaN(UUID uuid, const IndexName& indexName);
 
     using InfoMap = stdx::unordered_map<UUID, std::vector<Info>, UUID::Hash>;
     InfoMap getTTLInfos();
 
 private:
+    /**
+     * Shared implementation for deregistering TTL infos.
+     */
+    void _deregisterTTLInfo(UUID uuid, const Info& info);
+
     Mutex _ttlInfosLock = MONGO_MAKE_LATCH("TTLCollectionCache::_ttlInfosLock");
     InfoMap _ttlInfos;
 };
diff --git a/src/mongo/db/ttl_collection_cache_test.cpp b/src/mongo/db/ttl_collection_cache_test.cpp
index 4f598b3736c..747d1371887 100644
--- a/src/mongo/db/ttl_collection_cache_test.cpp
+++ b/src/mongo/db/ttl_collection_cache_test.cpp
@@ -39,9 +39,9 @@ TEST(TTLCollectionCacheTest, Basic) {
 
     auto uuidCollA = UUID::gen();
     auto uuidCollB = UUID::gen();
-    auto infoIndexA1 = TTLCollectionCache::Info{"collA_ttl_1"};
+    auto infoIndexA1 = TTLCollectionCache::Info{"collA_ttl_1", /*isExpireAfterSecondsNaN=*/false};
     auto infoClusteredA = TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}};
-    auto infoIndexB1 = TTLCollectionCache::Info("collB_ttl_1");
+    auto infoIndexB1 = TTLCollectionCache::Info("collB_ttl_1", /*isExpireAfterSecondsNaN=*/true);
 
     // Confirm registerTTLInfo() behavior using getTTLInfo().
     cache.registerTTLInfo(uuidCollA, infoIndexA1);
@@ -52,36 +52,45 @@ TEST(TTLCollectionCacheTest, Basic) {
     ASSERT_EQ(infos.size(), 2U);
     ASSERT_EQ(infos.count(uuidCollA), 1U);
     ASSERT_EQ(infos[uuidCollA].size(), 2U);
-    auto indexNameA = stdx::get_if<TTLCollectionCache::IndexName>(&infos[uuidCollA][0]);
-    ASSERT(indexNameA);
-    ASSERT_EQ(*indexNameA, "collA_ttl_1");
-    ASSERT(stdx::get_if<TTLCollectionCache::ClusteredId>(&infos[uuidCollA][1]));
+    ASSERT_FALSE(infos[uuidCollA][0].isClustered());
+    ASSERT_EQ(infos[uuidCollA][0].getIndexName(), "collA_ttl_1");
+    ASSERT_FALSE(infos[uuidCollA][0].isExpireAfterSecondsNaN());
+    ASSERT(infos[uuidCollA][1].isClustered());
 
     ASSERT_EQ(infos.count(uuidCollB), 1U);
     ASSERT_EQ(infos[uuidCollB].size(), 1U);
 
-    auto indexNameB = stdx::get_if<TTLCollectionCache::IndexName>(&infos[uuidCollB][0]);
-    ASSERT(indexNameB);
-    ASSERT_EQ(*indexNameB, "collB_ttl_1");
+    ASSERT_FALSE(infos[uuidCollB][0].isClustered());
+    ASSERT_EQ(infos[uuidCollB][0].getIndexName(), "collB_ttl_1");
+    ASSERT(infos[uuidCollB][0].isExpireAfterSecondsNaN());
+
+    // Check that we can reset '_isExpireAfterSecondsNaN()' on the TTL index.
+    cache.unsetTTLIndexExpireAfterSecondsNaN(uuidCollB, infoIndexB1.getIndexName());
+    infos = cache.getTTLInfos();
+    ASSERT_EQ(infos.size(), 2U);
+    ASSERT_EQ(infos.count(uuidCollB), 1U);
+    ASSERT_EQ(infos[uuidCollB].size(), 1U);
+    ASSERT_EQ(infos[uuidCollB][0].getIndexName(), "collB_ttl_1");
+    ASSERT_FALSE(infos[uuidCollB][0].isExpireAfterSecondsNaN());
 
     // Check deregisterTTLInfo(). TTLCollectionCache should clean up
     // UUIDs that no longer have any TTL infos registered.
-    cache.deregisterTTLInfo(uuidCollB, infoIndexB1);
+    cache.deregisterTTLIndexByName(uuidCollB, infoIndexB1.getIndexName());
     infos = cache.getTTLInfos();
     ASSERT_EQ(infos.size(), 1U);
     ASSERT_EQ(infos.count(uuidCollA), 1U);
     ASSERT_EQ(infos[uuidCollA].size(), 2U);
 
     // Remove info for TTL index on collection A.
-    cache.deregisterTTLInfo(uuidCollA, infoIndexA1);
+    cache.deregisterTTLIndexByName(uuidCollA, infoIndexA1.getIndexName());
     infos = cache.getTTLInfos();
     ASSERT_EQ(infos.size(), 1U);
     ASSERT_EQ(infos.count(uuidCollA), 1U);
     ASSERT_EQ(infos[uuidCollA].size(), 1U);
-    ASSERT(stdx::get_if<TTLCollectionCache::ClusteredId>(&infos[uuidCollA][0]));
+    ASSERT(infos[uuidCollA][0].isClustered());
 
     // Remove clustered info for collection A.
-    cache.deregisterTTLInfo(uuidCollA, infoClusteredA);
+    cache.deregisterTTLClusteredIndex(uuidCollA);
     infos = cache.getTTLInfos();
     ASSERT_EQ(infos.size(), 0U);
 }