SERVER-43942 track newly created indexes until they are part of a consistent WT checkpoint, and skip them in background validation

9 files changed, 111 insertions, 4 deletions

diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp
index 23fc51218d2..7a8a530731d 100644
--- a/src/mongo/db/catalog/catalog_control_test.cpp
+++ b/src/mongo/db/catalog/catalog_control_test.cpp
@@ -186,6 +186,11 @@ public:
     std::unique_ptr<CheckpointLock> getCheckpointLock(OperationContext* opCtx) final {
         return nullptr;
     }
+    void addIndividuallyCheckpointedIndexToList(const std::string& ident) final {}
+    void clearIndividuallyCheckpointedIndexesList() final {}
+    bool isInIndividuallyCheckpointedIndexesList(const std::string& ident) const final {
+        return false;
+    }
 };
 
 /**
diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp
index f206610698b..20a97d48938 100644
--- a/src/mongo/db/catalog/multi_index_block.cpp
+++ b/src/mongo/db/catalog/multi_index_block.cpp
@@ -50,6 +50,7 @@
 #include "mongo/db/query/collection_query_info.h"
 #include "mongo/db/repl/repl_set_config.h"
 #include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/db/storage/durable_catalog.h"
 #include "mongo/db/storage/storage_options.h"
 #include "mongo/db/storage/write_unit_of_work.h"
 #include "mongo/logger/redaction.h"
@@ -819,6 +820,29 @@ Status MultiIndexBlock::commit(OperationContext* opCtx,
                 _indexes[i].block->getEntry()->setMultikey(opCtx, *multikeyPaths);
             }
         }
+
+        if (opCtx->getServiceContext()->getStorageEngine()->supportsCheckpoints()) {
+            // Add the new index ident to a list so that the validate cmd with {background:true}
+            // can ignore the new index until it is regularly checkpoint'ed with the rest of the
+            // storage data.
+            //
+            // Index builds use the bulk loader, which can provoke a checkpoint of just that index.
+            // This makes the checkpoint's PIT view of the collection and indexes inconsistent until
+            // the next storage-wide checkpoint is taken, at which point the list will be reset.
+            //
+            // Note that it is okay if the index commit fails: background validation will never try
+            // to look at the index and the list will be reset by the next periodic storage-wide
+            // checkpoint.
+            //
+            // TODO (SERVER-44012): to remove this workaround.
+            auto checkpointLock =
+                opCtx->getServiceContext()->getStorageEngine()->getCheckpointLock(opCtx);
+            auto indexIdent =
+                opCtx->getServiceContext()->getStorageEngine()->getCatalog()->getIndexIdent(
+                    opCtx, collection->ns(), _indexes[i].block->getIndexName());
+            opCtx->getServiceContext()->getStorageEngine()->addIndividuallyCheckpointedIndexToList(
+                indexIdent);
+        }
     }
 
     onCommit();
diff --git a/src/mongo/db/catalog/validate_state.cpp b/src/mongo/db/catalog/validate_state.cpp
index 82123f96eaf..bf9aa1f6de7 100644
--- a/src/mongo/db/catalog/validate_state.cpp
+++ b/src/mongo/db/catalog/validate_state.cpp
@@ -234,6 +234,17 @@ void ValidateState::initializeCursors(OperationContext* opCtx) {
             continue;
         }
 
+        // Skip any newly created indexes that, because they were built with a WT bulk loader, are
+        // checkpoint'ed but not yet consistent with the rest of checkpoint's PIT view of the data.
+        if (_background &&
+            opCtx->getServiceContext()->getStorageEngine()->isInIndividuallyCheckpointedIndexesList(
+                diskIndexIdent)) {
+            _indexCursors.erase(desc->indexName());
+            log() << "Skipping validation on index '" << desc->indexName() << "' in collection '"
+                  << _nss << "' because the index data is not yet consistent in the checkpoint.";
+            continue;
+        }
+
         _indexes.push_back(indexCatalog->getEntryShared(desc));
     }
 
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 8e334d9cee8..d3da6e5533a 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -249,6 +249,21 @@ public:
                   "The current storage engine does not support checkpoints");
     }
 
+    virtual void addIndividuallyCheckpointedIndexToList(const std::string& ident) {
+        uasserted(ErrorCodes::CommandNotSupported,
+                  "The current storage engine does not support checkpoints");
+    }
+
+    virtual void clearIndividuallyCheckpointedIndexesList() {
+        uasserted(ErrorCodes::CommandNotSupported,
+                  "The current storage engine does not support checkpoints");
+    }
+
+    virtual bool isInIndividuallyCheckpointedIndexesList(const std::string& ident) const {
+        uasserted(ErrorCodes::CommandNotSupported,
+                  "The current storage engine does not support checkpoints");
+    }
+
     /**
      * Returns whether the KVEngine supports checkpoints.
      */
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 7deb670e66b..d50e2f2899a 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -562,6 +562,12 @@ public:
      * opening several checkpoint cursors to ensure the same checkpoint is targeted.
      */
     virtual std::unique_ptr<CheckpointLock> getCheckpointLock(OperationContext* opCtx) = 0;
+
+    virtual void addIndividuallyCheckpointedIndexToList(const std::string& ident) = 0;
+
+    virtual void clearIndividuallyCheckpointedIndexesList() = 0;
+
+    virtual bool isInIndividuallyCheckpointedIndexesList(const std::string& ident) const = 0;
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index c39eadf6889..764be38bfeb 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -330,6 +330,18 @@ public:
         return _engine->getCheckpointLock(opCtx);
     }
 
+    void addIndividuallyCheckpointedIndexToList(const std::string& ident) override {
+        return _engine->addIndividuallyCheckpointedIndexToList(ident);
+    }
+
+    void clearIndividuallyCheckpointedIndexesList() override {
+        return _engine->clearIndividuallyCheckpointedIndexesList();
+    }
+
+    bool isInIndividuallyCheckpointedIndexesList(const std::string& ident) const override {
+        return _engine->isInIndividuallyCheckpointedIndexesList(ident);
+    }
+
     /**
      * Drop abandoned idents. Returns a parallel list of index name, index spec pairs to rebuild.
      */
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 5c9e2632204..aa185066b2b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -375,6 +375,7 @@ public:
                     UniqueWiredTigerSession session = _sessionCache->getSession();
                     WT_SESSION* s = session->getSession();
                     auto checkpointLock = _wiredTigerKVEngine->getCheckpointLock(opCtx.get());
+                    _wiredTigerKVEngine->clearIndividuallyCheckpointedIndexesList();
                     invariantWTOK(s->checkpoint(s, "use_timestamp=false"));
                 } else if (stableTimestamp < initialDataTimestamp) {
                     LOG_FOR_RECOVERY(2)
@@ -391,8 +392,11 @@ public:
 
                     UniqueWiredTigerSession session = _sessionCache->getSession();
                     WT_SESSION* s = session->getSession();
-                    auto checkpointLock = _wiredTigerKVEngine->getCheckpointLock(opCtx.get());
-                    invariantWTOK(s->checkpoint(s, "use_timestamp=true"));
+                    {
+                        auto checkpointLock = _wiredTigerKVEngine->getCheckpointLock(opCtx.get());
+                        _wiredTigerKVEngine->clearIndividuallyCheckpointedIndexesList();
+                        invariantWTOK(s->checkpoint(s, "use_timestamp=true"));
+                    }
 
                     if (oplogNeededForRollback.isOK()) {
                         // Now that the checkpoint is durable, publish the oplog needed to recover
@@ -1980,6 +1984,15 @@ std::unique_ptr<StorageEngine::CheckpointLock> WiredTigerKVEngine::getCheckpoint
     return std::make_unique<CheckpointLockImpl>(opCtx, _checkpointMutex);
 }
 
+bool WiredTigerKVEngine::isInIndividuallyCheckpointedIndexesList(const std::string& ident) const {
+    for (auto it = _checkpointedIndexes.begin(); it != _checkpointedIndexes.end(); ++it) {
+        if (*it == ident) {
+            return true;
+        }
+    }
+    return false;
+}
+
 bool WiredTigerKVEngine::supportsReadConcernSnapshot() const {
     return true;
 }
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 53b834f53f5..87588eb6531 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -355,6 +355,16 @@ public:
     std::unique_ptr<StorageEngine::CheckpointLock> getCheckpointLock(
         OperationContext* opCtx) override;
 
+    void addIndividuallyCheckpointedIndexToList(const std::string& ident) override {
+        _checkpointedIndexes.push_back(ident);
+    }
+
+    void clearIndividuallyCheckpointedIndexesList() override {
+        _checkpointedIndexes.clear();
+    }
+
+    bool isInIndividuallyCheckpointedIndexesList(const std::string& ident) const override;
+
 private:
     class WiredTigerSessionSweeper;
     class WiredTigerJournalFlusher;
@@ -477,5 +487,12 @@ private:
     // Required for taking a checkpoint; and can be used to ensure multiple checkpoint cursors
     // target the same checkpoint.
     Lock::ResourceMutex _checkpointMutex = Lock::ResourceMutex("checkpointCursorMutex");
+
+    // A list of indexes that were individually checkpoint'ed and are not consistent with the rest
+    // of the checkpoint's PIT view of the storage data. This list is reset when a storage-wide WT
+    // checkpoint is taken that makes the PIT view consistent again.
+    //
+    // Access must be protected by the CheckpointLock.
+    std::list<std::string> _checkpointedIndexes;
 };
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
index cf04a16d550..d56d6ebbf6b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
@@ -273,8 +273,11 @@ void WiredTigerSessionCache::waitUntilDurable(OperationContext* opCtx,
             stdx::unique_lock<Latch> lk(_journalListenerMutex);
             JournalListener::Token token = _journalListener->getToken();
             auto config = stableCheckpoint ? "use_timestamp=true" : "use_timestamp=false";
-            auto checkpointLock = _engine->getCheckpointLock(opCtx);
-            invariantWTOK(s->checkpoint(s, config));
+            {
+                auto checkpointLock = _engine->getCheckpointLock(opCtx);
+                _engine->clearIndividuallyCheckpointedIndexesList();
+                invariantWTOK(s->checkpoint(s, config));
+            }
             _journalListener->onDurable(token);
         }
         LOG(4) << "created checkpoint (forced)";
@@ -311,6 +314,7 @@ void WiredTigerSessionCache::waitUntilDurable(OperationContext* opCtx,
         LOG(4) << "flushed journal";
     } else {
         auto checkpointLock = _engine->getCheckpointLock(opCtx);
+        _engine->clearIndividuallyCheckpointedIndexesList();
         invariantWTOK(_waitUntilDurableSession->checkpoint(_waitUntilDurableSession, nullptr));
         LOG(4) << "created checkpoint";
     }