SERVER-75561 Report and log more detailed information when validate encounters multikey inconsistencies

(cherry picked from commit fc3f5184c88025330c2e729188f4e38bac7d9886)

2 files changed, 128 insertions, 19 deletions

diff --git a/jstests/noPassthrough/validate_multikey_failures.js b/jstests/noPassthrough/validate_multikey_failures.js
new file mode 100644
index 00000000000..6d21116c1d0
--- /dev/null
+++ b/jstests/noPassthrough/validate_multikey_failures.js
@@ -0,0 +1,78 @@
+/**
+ * Test that validate detects and properly reports multikey inconsistencies.
+ */
+(function() {
+"use strict";
+
+const baseName = "validate_multikey_failures";
+const dbpath = MongoRunner.dataPath + baseName + "/";
+let conn = MongoRunner.runMongod({dbpath: dbpath});
+let coll = conn.getDB("test").getCollection("corrupt");
+
+const resetCollection = () => {
+    coll.drop();
+    coll.createIndex({"a.b": 1});
+};
+
+const disableMultikeyUpdate = () => {
+    assert.commandWorked(
+        conn.adminCommand({configureFailPoint: "skipUpdateIndexMultikey", mode: "alwaysOn"}));
+};
+
+const enableMultikeyUpdate = () => {
+    assert.commandWorked(
+        conn.adminCommand({configureFailPoint: "skipUpdateIndexMultikey", mode: "off"}));
+};
+
+function checkValidateLogs() {
+    assert(checkLog.checkContainsWithAtLeastCountJson(
+        conn, 7463100, {"spec": {"v": 2, "key": {"_id": 1}, "name": "_id_"}}, 1));
+}
+
+// Test that multiple keys suggest index should be marked multikey.
+resetCollection();
+disableMultikeyUpdate();
+assert.commandWorked(coll.insert({a: {b: [1, 2]}}));
+enableMultikeyUpdate();
+let res = coll.validate();
+assert.commandWorked(res);
+assert(!res.valid);
+assert.eq(res.indexDetails["a.b_1"].errors.length, 1);
+assert(res.indexDetails["a.b_1"].errors[0].startsWith("Index a.b_1 is not multikey"));
+assert(res.indexDetails["a.b_1"].errors[0].includes("2 key(s)"));
+assert(checkLog.checkContainsWithAtLeastCountJson(conn, 7556100, {"indexName": "a.b_1"}, 1));
+assert(checkLog.checkContainsWithAtLeastCountJson(conn, 7556101, {"indexKey": {"a.b": 1}}, 1));
+assert(checkLog.checkContainsWithAtLeastCountJson(conn, 7556101, {"indexKey": {"a.b": 2}}, 1));
+
+// Test that a single-entry array suggests index should be marked multikey.
+resetCollection();
+disableMultikeyUpdate();
+assert.commandWorked(coll.insert({a: {b: [3]}}));
+enableMultikeyUpdate();
+res = coll.validate();
+assert.commandWorked(res);
+assert(!res.valid);
+assert.eq(res.indexDetails["a.b_1"].errors.length, 1);
+assert(res.indexDetails["a.b_1"].errors[0].startsWith("Index a.b_1 is not multikey"));
+assert(res.indexDetails["a.b_1"].errors[0].includes("1 key(s)"));
+assert(checkLog.checkContainsWithAtLeastCountJson(conn, 7556100, {"indexName": "a.b_1"}, 1));
+assert(checkLog.checkContainsWithAtLeastCountJson(conn, 7556101, {"indexKey": {"a.b": 3}}, 1));
+
+// Test that a mis-match in multikey paths should be marked multikey.
+resetCollection();
+assert.commandWorked(coll.insert({a: [{b: 4}, {b: 5}]}));
+disableMultikeyUpdate();
+assert.commandWorked(coll.insert({a: {b: [6]}}));
+enableMultikeyUpdate();
+res = coll.validate();
+assert.commandWorked(res);
+assert(!res.valid);
+assert.eq(res.indexDetails["a.b_1"].errors.length, 1);
+assert(res.indexDetails["a.b_1"].errors[0].startsWith(
+    "Index a.b_1 multikey paths do not cover a document"));
+assert(checkLog.checkContainsWithAtLeastCountJson(conn, 7556100, {"indexName": "a.b_1"}, 1));
+assert(checkLog.checkContainsWithAtLeastCountJson(conn, 7556101, {"indexKey": {"a.b": 6}}, 1));
+assert(checkLog.checkContainsWithAtLeastCountJson(conn, 5367500, {"index": "a.b_1"}, 1));
+
+MongoRunner.stopMongod(conn, null, {skipValidation: true});
+})();
\ No newline at end of file
diff --git a/src/mongo/db/catalog/index_consistency.cpp b/src/mongo/db/catalog/index_consistency.cpp
index 1f3b2b7b2ef..41fceb0ec8e 100644
--- a/src/mongo/db/catalog/index_consistency.cpp
+++ b/src/mongo/db/catalog/index_consistency.cpp
@@ -28,6 +28,8 @@
  */
 
 
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/storage/key_string.h"
 #include <algorithm>
 
 #include "mongo/platform/basic.h"
@@ -79,6 +81,25 @@ std::pair<std::string, std::string> _generateKeyForMap(const IndexInfo& indexInf
     return std::make_pair(indexInfo.indexName, std::string(ks.getBuffer(), ks.getSize()));
 }
 
+BSONObj _rehydrateKey(const BSONObj& keyPattern, const BSONObj& indexKey) {
+    // We need to rehydrate the indexKey for improved readability.
+    // {"": ObjectId(...)} -> {"_id": ObjectId(...)}
+    auto keysIt = keyPattern.begin();
+    auto valuesIt = indexKey.begin();
+
+    BSONObjBuilder b;
+    while (keysIt != keyPattern.end()) {
+        // keysIt and valuesIt must have the same number of elements.
+        invariant(valuesIt != indexKey.end());
+        b.appendAs(*valuesIt, keysIt->fieldName());
+        keysIt++;
+        valuesIt++;
+    }
+
+    return b.obj();
+}
+
+
 }  // namespace
 
 IndexInfo::IndexInfo(const IndexDescriptor* descriptor, IndexAccessMethod* indexAccessMethod)
@@ -934,6 +955,26 @@ void KeyStringIndexConsistency::traverseRecord(OperationContext* opCtx,
                                        {multikeyMetadataKeys->begin(), multikeyMetadataKeys->end()},
                                        *documentMultikeyPaths);
 
+    auto printMultikeyMetadata = [&]() {
+        LOGV2(7556100,
+              "Index is not multikey but document has multikey data",
+              "indexName"_attr = descriptor->indexName(),
+              "recordId"_attr = recordId,
+              "record"_attr = redact(recordBson));
+        for (auto& key : *documentKeySet) {
+            auto indexKey = KeyString::toBsonSafe(key.getBuffer(),
+                                                  key.getSize(),
+                                                  iam->getSortedDataInterface()->getOrdering(),
+                                                  key.getTypeBits());
+            const BSONObj rehydratedKey = _rehydrateKey(descriptor->keyPattern(), indexKey);
+            LOGV2(7556101,
+                  "Index key for document with multikey inconsistency",
+                  "indexName"_attr = descriptor->indexName(),
+                  "recordId"_attr = recordId,
+                  "indexKey"_attr = redact(rehydratedKey));
+        }
+    };
+
     if (!index->isMultikey(opCtx, coll) && shouldBeMultikey) {
         if (_validateState->fixErrors()) {
             writeConflictRetry(opCtx, "setIndexAsMultikey", coll->ns().ns(), [&] {
@@ -951,13 +992,16 @@ void KeyStringIndexConsistency::traverseRecord(OperationContext* opCtx,
                                                       << " set to multikey.");
             results->repaired = true;
         } else {
+            printMultikeyMetadata();
+
             auto& curRecordResults = (results->indexResultsMap)[descriptor->indexName()];
             const std::string msg = fmt::format(
-                "Index {} is not multikey but has more than one key in document with "
-                "RecordId({}) and {}",
+                "Index {} is not multikey but document with RecordId({}) and {} has multikey data, "
+                "{} key(s)",
                 descriptor->indexName(),
                 recordId.toString(),
-                recordBson.getField("_id").toString());
+                recordBson.getField("_id").toString(),
+                documentKeySet->size());
             curRecordResults.errors.push_back(msg);
             curRecordResults.valid = false;
             if (crashOnMultikeyValidateFailure.shouldFail()) {
@@ -985,6 +1029,8 @@ void KeyStringIndexConsistency::traverseRecord(OperationContext* opCtx,
                                                           << " multikey paths updated.");
                 results->repaired = true;
             } else {
+                printMultikeyMetadata();
+
                 const std::string msg = fmt::format(
                     "Index {} multikey paths do not cover a document with RecordId({}) and {}",
                     descriptor->indexName(),
@@ -1023,22 +1069,7 @@ BSONObj KeyStringIndexConsistency::_generateInfo(const std::string& indexName,
                                                  const RecordId& recordId,
                                                  const BSONObj& indexKey,
                                                  const BSONObj& idKey) {
-
-    // We need to rehydrate the indexKey for improved readability.
-    // {"": ObjectId(...)} -> {"_id": ObjectId(...)}
-    auto keysIt = keyPattern.begin();
-    auto valuesIt = indexKey.begin();
-
-    BSONObjBuilder b;
-    while (keysIt != keyPattern.end()) {
-        // keysIt and valuesIt must have the same number of elements.
-        invariant(valuesIt != indexKey.end());
-        b.appendAs(*valuesIt, keysIt->fieldName());
-        keysIt++;
-        valuesIt++;
-    }
-
-    BSONObj rehydratedKey = b.done();
+    BSONObj rehydratedKey = _rehydrateKey(keyPattern, indexKey);
 
     BSONObjBuilder infoBuilder;
     infoBuilder.append("indexName", indexName);