SERVER-69186 Track newly created indexes for background validation again

11 files changed, 186 insertions, 25 deletions

diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp
index 03c41a60991..2df5e30f295 100644
--- a/src/mongo/db/catalog/multi_index_block.cpp
+++ b/src/mongo/db/catalog/multi_index_block.cpp
@@ -49,6 +49,7 @@
 #include "mongo/db/repl/repl_set_config.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/repl/tenant_migration_conflict_info.h"
+#include "mongo/db/storage/durable_catalog.h"
 #include "mongo/db/storage/storage_options.h"
 #include "mongo/db/storage/storage_parameters_gen.h"
 #include "mongo/db/storage/write_unit_of_work.h"
@@ -982,6 +983,25 @@ Status MultiIndexBlock::commit(OperationContext* opCtx,
         if (bulkBuilder->isMultikey()) {
             indexCatalogEntry->setMultikey(opCtx, collection, {}, bulkBuilder->getMultikeyPaths());
         }
+
+        if (opCtx->getServiceContext()->getStorageEngine()->supportsCheckpoints()) {
+            // Add the new index ident to a list so that the validate cmd with {background:true}
+            // can ignore the new index until it is regularly checkpoint'ed with the rest of the
+            // storage data.
+            //
+            // Index builds use the bulk loader, which can provoke a checkpoint of just that index.
+            // This makes the checkpoint's PIT view of the collection and indexes inconsistent until
+            // the next storage-wide checkpoint is taken, at which point the list will be reset.
+            //
+            // Note that it is okay if the index commit fails: background validation will never try
+            // to look at the index and the list will be reset by the next periodic storage-wide
+            // checkpoint.
+            auto indexIdent =
+                opCtx->getServiceContext()->getStorageEngine()->getCatalog()->getIndexIdent(
+                    opCtx, collection->getCatalogId(), _indexes[i].block->getIndexName());
+            opCtx->getServiceContext()->getStorageEngine()->addIndividuallyCheckpointedIndex(
+                indexIdent);
+        }
     }
 
     onCommit();
diff --git a/src/mongo/db/storage/control/storage_control.cpp b/src/mongo/db/storage/control/storage_control.cpp
index fc5534bec69..d47a4030afe 100644
--- a/src/mongo/db/storage/control/storage_control.cpp
+++ b/src/mongo/db/storage/control/storage_control.cpp
@@ -81,7 +81,8 @@ void startStorageControls(ServiceContext* serviceContext, bool forTestOnly) {
         JournalFlusher::set(serviceContext, std::move(journalFlusher));
     }
 
-    if (!storageEngine->isEphemeral() && !storageGlobalParams.queryableBackupMode) {
+    if (storageEngine->supportsCheckpoints() && !storageEngine->isEphemeral() &&
+        !storageGlobalParams.queryableBackupMode) {
         std::unique_ptr<Checkpointer> checkpointer =
             std::make_unique<Checkpointer>(storageEngine->getEngine());
         checkpointer->go();
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index bca7f550d94..d7a1a66323a 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -239,6 +239,28 @@ public:
                       "The current storage engine doesn't support backup mode");
     }
 
+    virtual void addIndividuallyCheckpointedIndex(const std::string& ident) {
+        uasserted(ErrorCodes::CommandNotSupported,
+                  "The current storage engine does not support checkpoints");
+    }
+
+    virtual void clearIndividuallyCheckpointedIndexes() {
+        uasserted(ErrorCodes::CommandNotSupported,
+                  "The current storage engine does not support checkpoints");
+    }
+
+    virtual bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const {
+        uasserted(ErrorCodes::CommandNotSupported,
+                  "The current storage engine does not support checkpoints");
+    }
+
+    /**
+     * Returns whether the KVEngine supports checkpoints.
+     */
+    virtual bool supportsCheckpoints() const {
+        return false;
+    }
+
     virtual void checkpoint() {}
 
     /**
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 1fe8ed0ee39..d85f19e012f 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -207,6 +207,11 @@ public:
     virtual bool supportsCappedCollections() const = 0;
 
     /**
+     * Returns whether the storage engine supports checkpoints.
+     */
+    virtual bool supportsCheckpoints() const = 0;
+
+    /**
      * Returns true if the engine does not persist data to disk; false otherwise.
      */
     virtual bool isEphemeral() const = 0;
@@ -629,6 +634,12 @@ public:
     virtual DurableCatalog* getCatalog() = 0;
     virtual const DurableCatalog* getCatalog() const = 0;
 
+    virtual void addIndividuallyCheckpointedIndex(const std::string& ident) = 0;
+
+    virtual void clearIndividuallyCheckpointedIndexes() = 0;
+
+    virtual bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const = 0;
+
     /**
      * A service that would like to pin the oldest timestamp registers its request here. If the
      * request can be satisfied, OK is returned with the oldest timestamp the caller can
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index a79d22a96b4..19e2f7622f7 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -959,6 +959,10 @@ StatusWith<std::deque<std::string>> StorageEngineImpl::extendBackupCursor(Operat
     return _engine->extendBackupCursor(opCtx);
 }
 
+bool StorageEngineImpl::supportsCheckpoints() const {
+    return _engine->supportsCheckpoints();
+}
+
 bool StorageEngineImpl::isEphemeral() const {
     return _engine->isEphemeral();
 }
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index 5a318744268..9734f77f10b 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -101,6 +101,8 @@ public:
     virtual StatusWith<std::deque<std::string>> extendBackupCursor(
         OperationContext* opCtx) override;
 
+    virtual bool supportsCheckpoints() const override;
+
     virtual bool isEphemeral() const override;
 
     virtual Status repairRecordStore(OperationContext* opCtx,
@@ -329,6 +331,18 @@ public:
 
     const DurableCatalog* getCatalog() const override;
 
+    void addIndividuallyCheckpointedIndex(const std::string& ident) override {
+        return _engine->addIndividuallyCheckpointedIndex(ident);
+    }
+
+    void clearIndividuallyCheckpointedIndexes() override {
+        return _engine->clearIndividuallyCheckpointedIndexes();
+    }
+
+    bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const override {
+        return _engine->isInIndividuallyCheckpointedIndexes(ident);
+    }
+
     /**
      * When loading after an unclean shutdown, this performs cleanup on the DurableCatalogImpl.
      */
diff --git a/src/mongo/db/storage/storage_engine_mock.h b/src/mongo/db/storage/storage_engine_mock.h
index a7a7078384a..e180fe7d5e7 100644
--- a/src/mongo/db/storage/storage_engine_mock.h
+++ b/src/mongo/db/storage/storage_engine_mock.h
@@ -47,7 +47,9 @@ public:
     bool supportsCappedCollections() const final {
         return true;
     }
-
+    bool supportsCheckpoints() const final {
+        return false;
+    }
     bool isEphemeral() const final {
         return true;
     }
@@ -194,6 +196,11 @@ public:
     const DurableCatalog* getCatalog() const final {
         return nullptr;
     }
+    void addIndividuallyCheckpointedIndex(const std::string& ident) final {}
+    void clearIndividuallyCheckpointedIndexes() final {}
+    bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const final {
+        return false;
+    }
 
     StatusWith<Timestamp> pinOldestTimestamp(OperationContext* opCtx,
                                              const std::string& requestingServiceName,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index ad6abdca8a2..65c422d2226 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -2013,6 +2013,7 @@ void WiredTigerKVEngine::_checkpoint(WT_SESSION* session) {
         // Third, stableTimestamp >= initialDataTimestamp: Take stable checkpoint. Steady state
         // case.
         if (initialDataTimestamp.asULL() <= 1) {
+            clearIndividuallyCheckpointedIndexes();
             invariantWTOK(session->checkpoint(session, "use_timestamp=false"), session);
             LOGV2_FOR_RECOVERY(5576602,
                                2,
@@ -2034,7 +2035,10 @@ void WiredTigerKVEngine::_checkpoint(WT_SESSION* session) {
                                "stableTimestamp"_attr = stableTimestamp,
                                "oplogNeededForRollback"_attr = toString(oplogNeededForRollback));
 
-            invariantWTOK(session->checkpoint(session, "use_timestamp=true"), session);
+            {
+                clearIndividuallyCheckpointedIndexes();
+                invariantWTOK(session->checkpoint(session, "use_timestamp=true"), session);
+            }
 
             if (oplogNeededForRollback.isOK()) {
                 // Now that the checkpoint is durable, publish the oplog needed to recover from it.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 748ec6c4c1d..10672661629 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -119,6 +119,13 @@ public:
 
     bool supportsDirectoryPerDB() const override;
 
+    /**
+     * WiredTiger supports checkpoints when it isn't running in memory.
+     */
+    bool supportsCheckpoints() const override {
+        return !isEphemeral();
+    }
+
     void checkpoint() override;
 
     bool isEphemeral() const override {
@@ -371,6 +378,18 @@ public:
         return _clockSource;
     }
 
+    void addIndividuallyCheckpointedIndex(const std::string& ident) override {
+        _checkpointedIndexes.insert(ident);
+    }
+
+    void clearIndividuallyCheckpointedIndexes() override {
+        _checkpointedIndexes.clear();
+    }
+
+    bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const override {
+        return _checkpointedIndexes.find(ident) != _checkpointedIndexes.end();
+    }
+
     StatusWith<Timestamp> pinOldestTimestamp(OperationContext* opCtx,
                                              const std::string& requestingServiceName,
                                              Timestamp requestedTimestamp,
@@ -514,6 +533,15 @@ private:
     // timestamp. Provided by replication layer because WT does not persist timestamps.
     AtomicWord<std::uint64_t> _initialDataTimestamp;
 
+    // Required for taking a checkpoint; and can be used to ensure multiple checkpoint cursors
+    // target the same checkpoint.
+    Lock::ResourceMutex _checkpointCursorMutex = Lock::ResourceMutex("checkpointCursorMutex");
+
+    // A set of indexes that were individually checkpoint'ed and are not consistent with the rest
+    // of the checkpoint's PIT view of the storage data. This set is reset when a storage-wide WT
+    // checkpoint is taken that makes the PIT view consistent again.
+    std::set<std::string> _checkpointedIndexes;
+
     AtomicWord<std::uint64_t> _oplogNeededForCrashRecovery;
 
     std::unique_ptr<WiredTigerEngineRuntimeConfigParameter> _runTimeConfigParam;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
index a1ad92795a8..a79e5494cdc 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
@@ -310,7 +310,10 @@ void WiredTigerSessionCache::waitUntilDurable(OperationContext* opCtx,
 
             auto config = syncType == Fsync::kCheckpointStableTimestamp ? "use_timestamp=true"
                                                                         : "use_timestamp=false";
-            invariantWTOK(s->checkpoint(s, config), s);
+            {
+                _engine->clearIndividuallyCheckpointedIndexes();
+                invariantWTOK(s->checkpoint(s, config), s);
+            }
 
             if (token) {
                 journalListener->onDurable(token.value());
@@ -362,6 +365,7 @@ void WiredTigerSessionCache::waitUntilDurable(OperationContext* opCtx,
                       _waitUntilDurableSession);
         LOGV2_DEBUG(22419, 4, "flushed journal");
     } else {
+        _engine->clearIndividuallyCheckpointedIndexes();
         invariantWTOK(_waitUntilDurableSession->checkpoint(_waitUntilDurableSession, nullptr),
                       _waitUntilDurableSession);
         LOGV2_DEBUG(22420, 4, "created checkpoint");
diff --git a/src/mongo/dbtests/validate_tests.cpp b/src/mongo/dbtests/validate_tests.cpp
index d32c99402c2..4674a21de29 100644
--- a/src/mongo/dbtests/validate_tests.cpp
+++ b/src/mongo/dbtests/validate_tests.cpp
@@ -96,6 +96,9 @@ public:
         auto coll = db->createCollection(&_opCtx, _nss, options, createIdIndex);
         ASSERT_TRUE(coll) << _nss;
         wuow.commit();
+
+        _engineSupportsCheckpoints =
+            _opCtx.getServiceContext()->getStorageEngine()->supportsCheckpoints();
     }
 
     explicit ValidateBase(bool full, bool background)
@@ -204,6 +207,7 @@ protected:
     const NamespaceString _nss;
     std::unique_ptr<AutoGetDb> _autoDb;
     Database* _db;
+    bool _engineSupportsCheckpoints;
 };
 
 template <bool full, bool background>
@@ -212,7 +216,9 @@ public:
     ValidateIdIndexCount() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -270,7 +276,9 @@ class ValidateSecondaryIndexCount : public ValidateBase {
 public:
     ValidateSecondaryIndexCount() : ValidateBase(full, background) {}
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -336,7 +344,9 @@ class ValidateSecondaryIndex : public ValidateBase {
 public:
     ValidateSecondaryIndex() : ValidateBase(full, background) {}
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -394,7 +404,9 @@ public:
     ValidateIdIndex() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -469,7 +481,9 @@ public:
     ValidateMultiKeyIndex() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -553,7 +567,9 @@ public:
     ValidateSparseIndex() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -612,7 +628,9 @@ public:
     ValidatePartialIndex() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -678,7 +696,9 @@ public:
     ValidatePartialIndexOnCollectionWithNonIndexableFields() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -738,7 +758,9 @@ public:
     ValidateCompoundIndex() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -819,7 +841,9 @@ public:
     ValidateIndexEntry() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -938,7 +962,9 @@ public:
     ValidateWildCardIndex() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -1062,7 +1088,9 @@ public:
     ValidateWildCardIndexWithProjection() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -1162,7 +1190,9 @@ public:
     ValidateMissingAndExtraIndexEntryResults() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -1253,7 +1283,9 @@ public:
     ValidateMissingIndexEntryResults() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -1371,7 +1403,9 @@ public:
     ValidateExtraIndexEntryResults() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -3094,7 +3128,9 @@ public:
     ValidateDuplicateKeysUniqueIndex() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -3283,7 +3319,9 @@ public:
     ValidateInvalidBSONResults() : ValidateBase(full, background) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -4092,7 +4130,9 @@ public:
         : ValidateBase(/*full=*/false, background, /*clustered=*/true) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -4152,7 +4192,9 @@ public:
         : ValidateBase(/*full=*/false, background, /*clustered=*/true) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -4252,7 +4294,9 @@ public:
         : ValidateBase(/*full=*/false, /*background=*/false, /*clustered=*/true) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }
 
@@ -4394,7 +4438,9 @@ public:
           _withSecondaryIndex(withSecondaryIndex) {}
 
     void run() {
-        if (_background) {
+        // Cannot run validate with {background:true} if the storage engine does not support
+        // checkpoints.
+        if (_background && !_engineSupportsCheckpoints) {
             return;
         }