SERVER-60048 CheckReplDBHash should not fail for cases where we expect retryable findAndModify images to be inconsistent after a restart

2 files changed, 33 insertions, 3 deletions

diff --git a/src/mongo/shell/data_consistency_checker.js b/src/mongo/shell/data_consistency_checker.js
index 802fb60841b..185d1033058 100644
--- a/src/mongo/shell/data_consistency_checker.js
+++ b/src/mongo/shell/data_consistency_checker.js
@@ -200,6 +200,32 @@ var {DataConsistencyChecker} = (function() {
             };
         }
 
+        static canIgnoreCollectionDiff(sourceCollInfos, syncingCollInfos, collName) {
+            if (collName !== "image_collection") {
+                return false;
+            }
+            const sourceNode = sourceCollInfos.conn;
+            const syncingNode = syncingCollInfos.conn;
+
+            const sourceSession = sourceNode.getDB('test').getSession();
+            const syncingSession = syncingNode.getDB('test').getSession();
+            const diff = this.getCollectionDiffUsingSessions(
+                sourceSession, syncingSession, sourceCollInfos.dbName, collName);
+            for (let doc of diff.docsWithDifferentContents) {
+                const sourceDoc = doc["first"];
+                const syncingDoc = doc["second"];
+                const hasInvalidated = sourceDoc.hasOwnProperty("invalidated") &&
+                    syncingDoc.hasOwnProperty("invalidated");
+                if (!hasInvalidated || sourceDoc["invalidated"] === syncingDoc["invalidated"]) {
+                    // We only ever expect cases where the 'invalidated' fields are mismatched.
+                    return false;
+                }
+            }
+            print(`Ignoring inconsistencies for 'image_collection' because this can be expected` +
+                  ` when images are invalidated`);
+            return true;
+        }
+
         static dumpCollectionDiff(collectionPrinted, sourceCollInfos, syncingCollInfos, collName) {
             print('Dumping collection: ' + sourceCollInfos.ns(collName));
 
@@ -299,9 +325,14 @@ var {DataConsistencyChecker} = (function() {
                 if (sourceDBHash.collections[collName] !== syncingDBHash.collections[collName]) {
                     prettyPrint(`the two nodes have a different hash for the collection ${dbName}.${
                         collName}: ${dbHashesMsg}`);
+                    // Although rare, the 'config.image_collection' table can be inconsistent after
+                    // an initial sync or after a restart (see SERVER-60048). Dump the collection
+                    // diff anyways for more visibility as a sanity check.
                     this.dumpCollectionDiff(
                         collectionPrinted, sourceCollInfos, syncingCollInfos, collName);
-                    success = false;
+                    const shouldIgnoreFailure =
+                        this.canIgnoreCollectionDiff(sourceCollInfos, syncingCollInfos, collName);
+                    success = shouldIgnoreFailure;
                 }
             });
 
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 6c65a0fbf7f..c9ec677e1b0 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -2441,8 +2441,7 @@ var ReplSetTest = function(opts) {
                 // TODO SERVER-60238: remove system.preimages check when replication to secondaries
                 // is implemented.
                 const primaryCollections =
-                    Object.keys(primaryDBHash.collections)
-                        .filter((x) => x !== "config.image_collection" && x !== "system.preimages");
+                    Object.keys(primaryDBHash.collections).filter((x) => x !== "system.preimages");
                 assert.commandWorked(primaryDBHash);
 
                 // Filter only collections that were retrieved by the dbhash. listCollections