SERVER-72677 Surface index validation errors arising from WT::verify

(cherry picked from commit 40c93f028e36f78c06756f4bfd358d240bdd9b34)

5 files changed, 53 insertions, 11 deletions

diff --git a/src/mongo/db/catalog/collection_validation.cpp b/src/mongo/db/catalog/collection_validation.cpp
index 42c5ac18dc7..ae6be4b3f2a 100644
--- a/src/mongo/db/catalog/collection_validation.cpp
+++ b/src/mongo/db/catalog/collection_validation.cpp
@@ -263,19 +263,20 @@ void _reportValidationResults(OperationContext* opCtx,
 
     // Report detailed index validation results gathered when using {full: true} for validated
     // indexes.
-    for (const auto& index : validateState->getIndexes()) {
-        const std::string indexName = index->descriptor()->indexName();
-        auto& indexResultsMap = results->indexResultsMap;
-        if (indexResultsMap.find(indexName) == indexResultsMap.end()) {
-            continue;
-        }
-
-        auto& vr = indexResultsMap.at(indexName);
-
+    int nIndexes = results->indexResultsMap.size();
+    for (const auto& [indexName, vr] : results->indexResultsMap) {
         if (!vr.valid) {
             results->valid = false;
         }
 
+        if (validateState->getSkippedIndexes().contains(indexName)) {
+            // Index internal state was checked and cleared, so it was reported in indexResultsMap,
+            // but we did not verify the index contents against the collection, so we should exclude
+            // it from this report.
+            --nIndexes;
+            continue;
+        }
+
         BSONObjBuilder bob(indexDetails.subobjStart(indexName));
         bob.appendBool("valid", vr.valid);
 
@@ -294,7 +295,7 @@ void _reportValidationResults(OperationContext* opCtx,
         results->errors.insert(results->errors.end(), vr.errors.begin(), vr.errors.end());
     }
 
-    output->append("nIndexes", static_cast<int>(validateState->getIndexes().size()));
+    output->append("nIndexes", nIndexes);
     output->append("keysPerIndex", keysPerIndex.done());
     output->append("indexDetails", indexDetails.done());
 }
diff --git a/src/mongo/db/catalog/collection_validation_test.cpp b/src/mongo/db/catalog/collection_validation_test.cpp
index 4ffff4f653b..c5313599053 100644
--- a/src/mongo/db/catalog/collection_validation_test.cpp
+++ b/src/mongo/db/catalog/collection_validation_test.cpp
@@ -47,6 +47,9 @@ namespace {
 const NamespaceString kNss = NamespaceString("test.t");
 
 class CollectionValidationTest : public CatalogTestFixture {
+protected:
+    CollectionValidationTest(Options options = {}) : CatalogTestFixture(std::move(options)) {}
+
 private:
     void setUp() override {
         CatalogTestFixture::setUp();
@@ -58,6 +61,11 @@ private:
     };
 };
 
+class CollectionValidationDiskTest : public CollectionValidationTest {
+protected:
+    CollectionValidationDiskTest() : CollectionValidationTest(Options{}.ephemeral(false)) {}
+};
+
 /**
  * Calls validate on collection kNss with both kValidateFull and kValidateNormal validation levels
  * and verifies the results.
@@ -270,6 +278,19 @@ TEST_F(CollectionValidationTest, ValidateEnforceFastCount) {
                        {CollectionValidation::ValidateMode::kForegroundFullEnforceFastCount});
 }
 
+TEST_F(CollectionValidationDiskTest, ValidateIndexDetailResultsSurfaceVerifyErrors) {
+    FailPointEnableBlock fp{"WTValidateIndexStructuralDamage"};
+    auto opCtx = operationContext();
+    insertDataRange(opCtx, 0, 5);  // initialize collection
+    foregroundValidate(
+        opCtx,
+        /*valid*/ false,
+        /*numRecords*/ std::numeric_limits<int32_t>::min(),           // uninitialized
+        /*numInvalidDocuments*/ std::numeric_limits<int32_t>::min(),  // uninitialized
+        /*numErrors*/ 1,
+        {CollectionValidation::ValidateMode::kForegroundFull});
+}
+
 /**
  * Waits for a parallel running collection validation operation to start and then hang at a
  * failpoint.
diff --git a/src/mongo/db/catalog/validate_state.cpp b/src/mongo/db/catalog/validate_state.cpp
index 434950b9540..9b93a1f69e6 100644
--- a/src/mongo/db/catalog/validate_state.cpp
+++ b/src/mongo/db/catalog/validate_state.cpp
@@ -245,8 +245,10 @@ void ValidateState::initializeCursors(OperationContext* opCtx) {
         const IndexDescriptor* desc = entry->descriptor();
 
         auto iam = entry->accessMethod()->asSortedData();
-        if (!iam)
+        if (!iam) {
+            _skippedIndexes.emplace(desc->indexName());
             continue;
+        }
 
         _indexCursors.emplace(
             desc->indexName(),
diff --git a/src/mongo/db/catalog/validate_state.h b/src/mongo/db/catalog/validate_state.h
index df796c686ce..72f8da3019e 100644
--- a/src/mongo/db/catalog/validate_state.h
+++ b/src/mongo/db/catalog/validate_state.h
@@ -123,6 +123,10 @@ public:
         return _indexes;
     }
 
+    const StringSet& getSkippedIndexes() const {
+        return _skippedIndexes;
+    }
+
     /**
      * Map of index names to index cursors.
      */
@@ -235,6 +239,10 @@ private:
     std::unique_ptr<SeekableRecordThrottleCursor> _traverseRecordStoreCursor;
     std::unique_ptr<SeekableRecordThrottleCursor> _seekRecordStoreCursor;
 
+    // Stores the set of indexes that will not be validated for some reason, e.g. they are not
+    // ready.
+    StringSet _skippedIndexes;
+
     RecordId _firstRecordId;
 
     DataThrottle _dataThrottle;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index 243e84608b8..19cba51532b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -83,6 +83,7 @@ namespace {
 
 MONGO_FAIL_POINT_DEFINE(WTCompactIndexEBUSY);
 MONGO_FAIL_POINT_DEFINE(WTIndexPauseAfterSearchNear);
+MONGO_FAIL_POINT_DEFINE(WTValidateIndexStructuralDamage);
 
 static const WiredTigerItem emptyItem(nullptr, 0);
 }  // namespace
@@ -329,6 +330,15 @@ void WiredTigerIndex::fullValidate(OperationContext* opCtx,
                                    IndexValidateResults* fullResults) const {
     dassert(opCtx->lockState()->isReadLocked());
     if (fullResults && !WiredTigerRecoveryUnit::get(opCtx)->getSessionCache()->isEphemeral()) {
+        if (WTValidateIndexStructuralDamage.shouldFail()) {
+            std::string msg = str::stream() << "verify() returned an error. "
+                                            << "This indicates structural damage. "
+                                            << "Not examining individual index entries.";
+            fullResults->errors.push_back(msg);
+            fullResults->valid = false;
+            return;
+        }
+
         int err = WiredTigerUtil::verifyTable(opCtx, _uri, &(fullResults->errors));
         if (err == EBUSY) {
             std::string msg = str::stream()