SERVER-40620 fassert during document fetch when there is evidence of index corruption

4 files changed, 93 insertions, 0 deletions

diff --git a/jstests/noPassthrough/query_yields_catch_index_corruption.js b/jstests/noPassthrough/query_yields_catch_index_corruption.js
new file mode 100644
index 00000000000..dd29397acc7
--- /dev/null
+++ b/jstests/noPassthrough/query_yields_catch_index_corruption.js
@@ -0,0 +1,54 @@
+// Note: This test will fail burn_in for some build variants. We expect each execution to generate a
+// core dump (because of an expected fassert failure). When the burn_in test runs the test in a
+// tight loop, the core dumps can consume all available disk space and cause writes to fail.
+// @tags: [requires_persistence, requires_journaling]
+(function() {
+    "use strict";
+
+    const name = "query_yields_catch_index_corruption";
+    const dbpath = MongoRunner.dataPath + name + "/";
+
+    resetDbpath(dbpath);
+
+    let mongod = MongoRunner.runMongod({dbpath: dbpath});
+    assert.neq(null, mongod, "mongod failed to start.");
+
+    let db = mongod.getDB("test");
+
+    assert.commandWorked(db.adminCommand(
+        {configureFailPoint: "skipUnindexingDocumentWhenDeleted", mode: "alwaysOn"}));
+
+    let coll = db.getCollection(name);
+    coll.drop();
+
+    assert.commandWorked(coll.createIndex({a: 1}));
+    assert.writeOK(coll.insert({a: 0}));
+
+    // Corrupt the index by deleting the document but not unindexing it.
+    assert.commandWorked(coll.remove({a: 0}));
+    let validateRes = assert.commandWorked(coll.validate());
+    assert.eq(false, validateRes.valid);
+
+    assert.throws(() => coll.find({a: 0}).toArray());
+
+    // fassert() calls std::abort(), which returns a different exit code for Windows vs. other
+    // platforms.
+    const exitCode = _isWindows() ? MongoRunner.EXIT_ABRUPT : MongoRunner.EXIT_ABORT;
+    MongoRunner.stopMongod(mongod, null, {allowedExitCode: exitCode});
+
+    // Test that the --repair flag rebuilds the corrupted index.
+    mongod = MongoRunner.runMongod({dbpath: dbpath, noCleanData: true, repair: ""});
+    assert.eq(null, mongod, "Expect this to exit cleanly");
+
+    // Restarting the server.
+    mongod = MongoRunner.runMongod({dbpath: dbpath, noCleanData: true});
+    assert.neq(null, mongod, "mongod failed to start after repair");
+
+    db = mongod.getDB("test");
+    coll = db.getCollection(name);
+
+    validateRes = assert.commandWorked(coll.validate());
+    assert.eq(true, validateRes.valid, tojson(validateRes));
+
+    MongoRunner.stopMongod(mongod);
+})();
\ No newline at end of file
diff --git a/src/mongo/db/catalog/index_catalog_impl.cpp b/src/mongo/db/catalog/index_catalog_impl.cpp
index a0a467bccc2..9700b125b5f 100644
--- a/src/mongo/db/catalog/index_catalog_impl.cpp
+++ b/src/mongo/db/catalog/index_catalog_impl.cpp
@@ -75,6 +75,8 @@
 
 namespace mongo {
 
+MONGO_FAIL_POINT_DEFINE(skipUnindexingDocumentWhenDeleted);
+
 using std::endl;
 using std::string;
 using std::unique_ptr;
@@ -1463,6 +1465,11 @@ void IndexCatalogImpl::_unindexRecord(OperationContext* opCtx,
     entry->accessMethod()->getKeys(
         obj, IndexAccessMethod::GetKeysMode::kRelaxConstraintsUnfiltered, &keys, nullptr, nullptr);
 
+    // Tests can enable this failpoint to produce index corruption scenarios where an index has
+    // extra keys.
+    MONGO_FAIL_POINT_BLOCK(skipUnindexingDocumentWhenDeleted, extraData) {
+        return;
+    }
     _unindexKeys(opCtx, entry, {keys.begin(), keys.end()}, obj, loc, logIfError, keysDeletedOut);
 }
 
diff --git a/src/mongo/db/exec/working_set_common.cpp b/src/mongo/db/exec/working_set_common.cpp
index c3d65fa448b..d41e21d2fb2 100644
--- a/src/mongo/db/exec/working_set_common.cpp
+++ b/src/mongo/db/exec/working_set_common.cpp
@@ -27,6 +27,8 @@
  *    it in the license file.
  */
 
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kQuery
+
 #include "mongo/platform/basic.h"
 
 #include "mongo/db/exec/working_set_common.h"
@@ -37,9 +39,27 @@
 #include "mongo/db/index/index_access_method.h"
 #include "mongo/db/query/canonical_query.h"
 #include "mongo/db/service_context.h"
+#include "mongo/util/log.h"
 
 namespace mongo {
 
+namespace {
+std::string indexKeyVectorDebugString(const std::vector<IndexKeyDatum>& keyData) {
+    StringBuilder sb;
+    sb << "[";
+    if (keyData.size() > 0) {
+        auto it = keyData.begin();
+        sb << "(key: " << it->keyData << ", index key pattern: " << it->indexKeyPattern << ")";
+        while (++it != keyData.end()) {
+            sb << ", (key: " << it->keyData << ", index key pattern: " << it->indexKeyPattern
+               << ")";
+        }
+    }
+    sb << "]";
+    return sb.str();
+}
+}  // namespace
+
 void WorkingSetCommon::prepareForSnapshotChange(WorkingSet* workingSet) {
     for (auto id : workingSet->getAndClearYieldSensitiveIds()) {
         if (workingSet->isFree(id)) {
@@ -68,6 +88,16 @@ bool WorkingSetCommon::fetch(OperationContext* opCtx,
     member->obj.reset();
     auto record = cursor->seekExact(member->recordId);
     if (!record) {
+        // During a query yield, if the member is in the RID_AND_IDX state it should have been
+        // marked as suspicious. If not, then we failed to find the keyed document within the same
+        // storage snapshot, and therefore there may be index corruption.
+        if (member->getState() == WorkingSetMember::RID_AND_IDX && !member->isSuspicious) {
+            error() << "Evidence of index corruption due to extraneous index key(s): "
+                    << indexKeyVectorDebugString(member->keyData) << " on document with record id "
+                    << member->recordId
+                    << ", see http://dochub.mongodb.org/core/data-recovery for recovery steps.";
+            fassertFailed(31138);
+        }
         return false;
     }
 
diff --git a/src/mongo/dbtests/query_stage_merge_sort.cpp b/src/mongo/dbtests/query_stage_merge_sort.cpp
index 5656174f7b3..bdc9f235bfb 100644
--- a/src/mongo/dbtests/query_stage_merge_sort.cpp
+++ b/src/mongo/dbtests/query_stage_merge_sort.cpp
@@ -576,6 +576,8 @@ public:
             ++it;
         }
 
+        WorkingSetCommon::prepareForSnapshotChange(&ws);
+
         // Delete recordIds[11]. The deleted document should be buffered inside the SORT_MERGE
         // stage, and therefore should still be returned.
         ms->saveState();