SERVER-60048 CheckReplDBHash should not fail for cases where we expect retryable findAndModify images to be inconsistent after a restart

(cherry picked from commit bd9939bf4f98cced5b8dcd1f7249431ed5ee688c)

SERVER-61923 Fix bug where consistency checker can mask error due to ignoring inconsistency in image collections

(cherry picked from commit e2d48665f771a82bfe7de2a112276cd3692a6007)

SERVER-61610 Fix ignore dbHash mismatches for invalidated retryable FaM images

(cherry picked from commit 1b5fee201b23fb7c88025cebbd0f113fa2756747)

1 files changed, 41 insertions, 1 deletions

diff --git a/src/mongo/shell/data_consistency_checker.js b/src/mongo/shell/data_consistency_checker.js
index 4cdfb07dd87..25092c5cf0a 100644
--- a/src/mongo/shell/data_consistency_checker.js
+++ b/src/mongo/shell/data_consistency_checker.js
@@ -200,6 +200,35 @@ var {DataConsistencyChecker} = (function() {
             };
         }
 
+        static canIgnoreCollectionDiff(sourceCollInfos, syncingCollInfos, collName) {
+            if (collName !== "image_collection") {
+                return false;
+            }
+            const sourceNode = sourceCollInfos.conn;
+            const syncingNode = syncingCollInfos.conn;
+
+            const sourceSession = sourceNode.getDB('test').getSession();
+            const syncingSession = syncingNode.getDB('test').getSession();
+            const diff = this.getCollectionDiffUsingSessions(
+                sourceSession, syncingSession, sourceCollInfos.dbName, collName);
+            for (let doc of diff.docsWithDifferentContents) {
+                const sourceDoc = doc["sourceNode"];
+                const syncingDoc = doc["syncingNode"];
+                if (!sourceDoc || !syncingDoc) {
+                    return false;
+                }
+                const hasInvalidated = sourceDoc.hasOwnProperty("invalidated") &&
+                    syncingDoc.hasOwnProperty("invalidated");
+                if (!hasInvalidated || sourceDoc["invalidated"] === syncingDoc["invalidated"]) {
+                    // We only ever expect cases where the 'invalidated' fields are mismatched.
+                    return false;
+                }
+            }
+            print(`Ignoring inconsistencies for 'image_collection' because this can be expected` +
+                  ` when images are invalidated`);
+            return true;
+        }
+
         static dumpCollectionDiff(collectionPrinted, sourceCollInfos, syncingCollInfos, collName) {
             print('Dumping collection: ' + sourceCollInfos.ns(collName));
 
@@ -299,9 +328,20 @@ var {DataConsistencyChecker} = (function() {
                 if (sourceDBHash.collections[collName] !== syncingDBHash.collections[collName]) {
                     prettyPrint(`the two nodes have a different hash for the collection ${dbName}.${
                         collName}: ${dbHashesMsg}`);
+                    // Although rare, the 'config.image_collection' table can be inconsistent after
+                    // an initial sync or after a restart (see SERVER-60048). Dump the collection
+                    // diff anyways for more visibility as a sanity check.
                     this.dumpCollectionDiff(
                         collectionPrinted, sourceCollInfos, syncingCollInfos, collName);
-                    success = false;
+                    const shouldIgnoreFailure =
+                        this.canIgnoreCollectionDiff(sourceCollInfos, syncingCollInfos, collName);
+                    if (shouldIgnoreFailure) {
+                        prettyPrint(
+                            `Collection diff in ${dbName}.${collName} can be ignored: ${dbHashesMsg}
+                            . Inconsistencies in the image collection can be expected in certain
+                            restart scenarios.`);
+                    }
+                    success = shouldIgnoreFailure && success;
                 }
             });