SERVER-32368 do not perform UUID check against a shard's config cache in the sharding_csrs_continuous_config_stepdown_WT suite

(cherry picked from commit fe5b3058c3ba27f8262fa5198cf243053ba50b9b)

3 files changed, 59 insertions, 30 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
index 41810261223..d1da1afc614 100644
--- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
@@ -155,6 +155,9 @@ selector:
 executor:
   config:
     shell_options:
+      global_vars:
+        TestData:
+          skipCheckingCatalogCacheConsistencyWithShardingCatalog: true
       eval: "load('jstests/libs/override_methods/sharding_continuous_config_stepdown.js');"
       nodb: ''
       readMode: commands
diff --git a/jstests/libs/override_methods/check_uuids_consistent_across_cluster.js b/jstests/libs/override_methods/check_uuids_consistent_across_cluster.js
index 3b6636df7ac..4fbc9add333 100644
--- a/jstests/libs/override_methods/check_uuids_consistent_across_cluster.js
+++ b/jstests/libs/override_methods/check_uuids_consistent_across_cluster.js
@@ -1,28 +1,42 @@
 /**
- * Load this file when starting a sharded cluster to provide a callback to check that collection
- * UUIDs are consistent across shards and the config server.
+ * Provides a hook to check that shards' storage catalogs and catalog caches are consistent
+ * with the sharding catalog on the config server.
  *
- * To disable this check on a specific test, add the following to the JS file:
+ * The hook currently checks that: if the sharding catalog says a shard owns chunks for a sharded
+ * collection, then the shard has an entry for the collection
+ * - in its storage catalog, with the same UUID as the collection has in the sharding catalog
+ * - in its catalog cache, with the same UUID as the collection has in the sharding catalog
  *
- *     TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
- *
- * Common test actions that can cause the UUID check to fail are:
- * - Causing a shard not to have a primary. The UUID check must run commands against primaries.
- * - Restarting shard replica sets / primaries. The UUID consistency check uses shard primary
- *   connections cached in ShardingTest._connections, which may become out of date if the primary
- *   changes and the shell doesn't detect the new primary before this check occurs.
- * - No config primary. The check queries the config collections.
- * - Blackholing mongos from the config servers. The UUID check uses the mongos to query the config.
+ * TODO (SERVER-33252): extend the hook to add consistency checks for databases
+ * TODO (SERVER-33253): extend the hook to add consistency checks for collection indexes and options
  */
-
 "use strict";
 
 ShardingTest.prototype.checkUUIDsConsistentAcrossCluster = function() {
     if (jsTest.options().skipCheckingUUIDsConsistentAcrossCluster) {
-        print("Skipping checking UUID consistency across the cluster");
+        // A test may want to skip the consistency checks for a few reasons:
+        // 1)  The checks are performed against shard and config primaries, and the connections
+        //     cached on ShardingTest are used. So, tests that end with a different (or no) shard or
+        //     config primary should skip the checks.
+        // 2) The sharding catalog is read from the config server via mongos, so tests that cause
+        //    the config primary to be unreachable from mongos should skip the checks.
+        print(
+            "Skipping checking consistency of the sharding catalog with shards' storage catalogs and catalog caches");
         return;
     }
-    print("Checking UUID consistency across the cluster");
+
+    if (jsTest.options().skipCheckingCatalogCacheConsistencyWithShardingCatalog) {
+        // When a shard takes or loses ownership of a chunk (through shardCollection, moveChunk, or
+        // dropCollection), a best-effort is made to make the shard refresh its routing table cache.
+        // But since sharding catalog changes are not transactional, it's possible the shard's
+        // catalog cache will be stale. A test or suite that induces stepdowns or otherwise makes it
+        // likely that this "best-effort" will fail should skip checks for only the catalog caches.
+        print(
+            "Checking consistency of the sharding catalog with shards' storage catalogs, but not with shards' catalog caches");
+    } else {
+        print(
+            "Checking consistency of the sharding catalog with shards' storage catalogs and catalog caches");
+    }
 
     function parseNs(dbDotColl) {
         assert.gt(dbDotColl.indexOf('.'),
@@ -88,8 +102,10 @@ ShardingTest.prototype.checkUUIDsConsistentAcrossCluster = function() {
                 }
                 let shardConn = shardConnStringToConn[shardConnString];
 
-                print("checking that the UUID in the storage catalog for " + ns + " on " +
-                      shardConn + " is consistent with the config server");
+                print("Checking that the UUID for " + ns + " returned by listCollections on " +
+                      shardConn +
+                      " is consistent with the UUID in config.collections on the config server");
+
                 const actualCollMetadata =
                     shardConn.getDB(dbName).getCollectionInfos({name: collName})[0];
                 assert.eq(authoritativeCollMetadata.collInfo.uuid,
@@ -99,19 +115,27 @@ ShardingTest.prototype.checkUUIDsConsistentAcrossCluster = function() {
                               ", actual collection info on shard " + shardConnString + ": " +
                               tojson(actualCollMetadata));
 
-                print("checking that the UUID in config.cache.collections for " + ns + " on " +
-                      shardConn + " is consistent with the config server");
-                assert.commandWorked(shardConn.adminCommand(
-                    {_flushRoutingTableCacheUpdates: ns, syncFromConfig: false}));
-                const actualConfigMetadata =
-                    shardConn.getDB("config").getCollection("cache.collections").find({"_id":
-                                                                                           ns})[0];
-                assert.eq(authoritativeCollMetadata.collInfo.uuid,
-                          actualConfigMetadata.uuid,
-                          "authoritative collection info on config server: " +
-                              tojson(authoritativeCollMetadata.collInfo) +
-                              ", actual config info on shard " + shardConnString + ": " +
-                              tojson(actualConfigMetadata));
+                if (!jsTest.options().skipCheckingCatalogCacheConsistencyWithShardingCatalog) {
+                    print(
+                        "Checking that the UUID for " + ns + " in config.cache.collections on " +
+                        shardConn +
+                        " is consistent with the UUID in config.collections on the config server");
+
+                    // Wait for the shard to finish writing its last refresh to disk.
+                    assert.commandWorked(shardConn.adminCommand(
+                        {_flushRoutingTableCacheUpdates: ns, syncFromConfig: false}));
+
+                    const actualConfigMetadata =
+                        shardConn.getDB("config").getCollection("cache.collections").find({
+                            "_id": ns
+                        })[0];
+                    assert.eq(authoritativeCollMetadata.collInfo.uuid,
+                              actualConfigMetadata.uuid,
+                              "authoritative collection info on config server: " +
+                                  tojson(authoritativeCollMetadata.collInfo) +
+                                  ", actual config info on shard " + shardConnString + ": " +
+                                  tojson(actualConfigMetadata));
+                }
             }
         }
     } catch (e) {
diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js
index c7bd0552599..a6b8ddaa37b 100644
--- a/src/mongo/shell/utils.js
+++ b/src/mongo/shell/utils.js
@@ -257,6 +257,8 @@ jsTestOptions = function() {
             skipValidationNamespaces: TestData.skipValidationNamespaces || [],
             skipCheckingUUIDsConsistentAcrossCluster:
                 TestData.skipCheckingUUIDsConsistentAcrossCluster || false,
+            skipCheckingCatalogCacheConsistencyWithShardingCatalog:
+                TestData.skipCheckingCatalogCacheConsistencyWithShardingCatalog || false,
             jsonSchemaTestFile: TestData.jsonSchemaTestFile,
             excludedDBsFromDBHash: TestData.excludedDBsFromDBHash,
             alwaysInjectTransactionNumber: TestData.alwaysInjectTransactionNumber,