SERVER-65035 Create new JS hook to check routing table consistency

31 files changed, 232 insertions, 2 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml
index 83467144c4c..8cfd7dbc3ae 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml
@@ -114,6 +114,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml
index 3bfbf5175bd..4375552f6b2 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml
@@ -121,6 +121,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_clusterwide_ops_add_remove_shards.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_clusterwide_ops_add_remove_shards.yml
index 04cdc819806..7fdb5318933 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_clusterwide_ops_add_remove_shards.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_clusterwide_ops_add_remove_shards.yml
@@ -22,6 +22,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
index 18452678c0c..5dd83068144 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
@@ -201,6 +201,7 @@ executor:
     kill: true
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
     shell_options:
       global_vars:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml
index bda88a5e705..8badd363b1a 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml
@@ -172,6 +172,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml
index 1976dc1f531..4fa7dea97ba 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml
@@ -172,6 +172,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml
index eebb48b0492..9ea1bf7c2cc 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml
@@ -165,6 +165,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml
index 0dded08fdbf..8b743d9e35f 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml
@@ -252,6 +252,7 @@ executor:
     wait_for_mongos_retarget: true
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
     shell_options:
       global_vars:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml
index 630db00d785..5dbf97cc666 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml
@@ -252,6 +252,7 @@ executor:
     wait_for_mongos_retarget: true
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml
index fca9621e84e..cb38a7af3d3 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml
@@ -170,6 +170,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml
index de7ca513f81..cc369071517 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml
@@ -224,6 +224,7 @@ executor:
     wait_for_mongos_retarget: true
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml
index f2e8e9a06b1..6f68345793f 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml
@@ -127,6 +127,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml
index 0d104100f9d..c1a4682bf68 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml
@@ -119,6 +119,7 @@ executor:
   - class: CheckReplDBHashInBackground
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
index 41e0539bbc3..a52256fd970 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
@@ -201,6 +201,7 @@ executor:
     terminate: true
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
index fe148c981c7..49a119dca0a 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
@@ -193,6 +193,7 @@ executor:
     wait_for_mongos_retarget: true
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
index f4e9fad96fc..c657d29e781 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
@@ -199,6 +199,7 @@ executor:
     wait_for_mongos_retarget: true
   - class: CheckReplDBHash
   - class: CheckOrphansDeleted
+  - class: CheckRoutingTableConsistency
   - class: ValidateCollections # Validation can interfere with other operations, so this goes last.
   - class: CleanupConcurrencyWorkloads
   fixture:
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index 0f47f2ff5e4..2904d9064aa 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -261,6 +261,10 @@ def mongo_shell_program(  # pylint: disable=too-many-arguments,too-many-branches
     # Load a callback to check that all orphans are deleted before shutting down a ShardingTest.
     eval_sb.append("load('jstests/libs/override_methods/check_orphans_are_deleted.js');")
 
+    # Load a callback to check that the info stored in config.collections and config.chunks is
+    # semantically correct before shutting down a ShardingTest.
+    eval_sb.append("load('jstests/libs/override_methods/check_routing_table_consistency.js');")
+
     # Load this file to retry operations that fail due to in-progress background operations.
     eval_sb.append(
         "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');")
diff --git a/buildscripts/resmokelib/testing/hooks/routing_table_consistency.py b/buildscripts/resmokelib/testing/hooks/routing_table_consistency.py
new file mode 100644
index 00000000000..23aa04adbd6
--- /dev/null
+++ b/buildscripts/resmokelib/testing/hooks/routing_table_consistency.py
@@ -0,0 +1,24 @@
+"""Test hook for verifying the consistency of sharded collections metadata stored in the config server of a sharded cluster."""
+
+import os.path
+
+from buildscripts.resmokelib.testing.fixtures import shardedcluster
+from buildscripts.resmokelib.testing.hooks import jsfile
+
+
+class CheckRoutingTableConsistency(jsfile.DataConsistencyHook):
+    """Verifies the absence of corrupted entries in config.chunks and config.collections."""
+
+    IS_BACKGROUND = False
+
+    def __init__(self, hook_logger, fixture, shell_options=None):
+        """Initialize CheckRoutingTableConsistency."""
+
+        if not isinstance(fixture, shardedcluster.ShardedClusterFixture):
+            raise ValueError(f"'fixture' must be an instance of ShardedClusterFixture, but got"
+                             f" {fixture.__class__.__name__}")
+
+        description = "Inspect collection and chunk metadata in config server"
+        js_filename = os.path.join("jstests", "hooks", "run_check_routing_table_consistency.js")
+        super().__init__(hook_logger, fixture, js_filename, description,
+                         shell_options=shell_options)
diff --git a/jstests/hooks/run_check_routing_table_consistency.js b/jstests/hooks/run_check_routing_table_consistency.js
new file mode 100644
index 00000000000..305ac9550bc
--- /dev/null
+++ b/jstests/hooks/run_check_routing_table_consistency.js
@@ -0,0 +1,18 @@
+'use strict';
+
+(function() {
+load('jstests/libs/check_routing_table_consistency_helpers.js');  // For check implementation.
+load('jstests/libs/discover_topology.js');  // For Topology and DiscoverTopology.
+
+assert.neq(typeof db, 'undefined', 'No `db` object, is the shell connected to a server?');
+
+const conn = db.getMongo();
+const topology = DiscoverTopology.findConnectedNodes(conn);
+
+if (topology.type !== Topology.kShardedCluster) {
+    throw new Error(
+        'Routing table consistency check must be run against a sharded cluster, but got: ' +
+        tojson(topology));
+}
+RoutingTableConsistencyChecker.run(db.getMongo());
+})();
diff --git a/jstests/libs/check_routing_table_consistency_helpers.js b/jstests/libs/check_routing_table_consistency_helpers.js
new file mode 100644
index 00000000000..ac2631e62f4
--- /dev/null
+++ b/jstests/libs/check_routing_table_consistency_helpers.js
@@ -0,0 +1,125 @@
+'use strict';
+
+var RoutingTableConsistencyChecker = (function() {
+    const sameObjectFields = (lhsObjFields, rhsObjFields) => {
+        if (lhsObjFields.length !== rhsObjFields.length) {
+            return false;
+        }
+
+        for (let i = 0; i != lhsObjFields.length; ++i) {
+            if (lhsObjFields[i] !== rhsObjFields[i]) {
+                return false;
+            }
+        }
+
+        return true;
+    };
+
+    const fetchRoutingTableData = (mongos) => {
+        // Group docs in config.chunks by coll UUID (sorting by minKey),
+        // then join with docs in config.collections.
+        // NOTE: the query may throw an exception if the collected data size is bigger than 16MB.
+        // To skip the execution of the hook, set the `TestData.skipCheckRoutingTableConsistency`
+        // flag in the failing test.
+        return mongos.getDB('config')
+                                .chunks
+                                .aggregate([
+                                    {$sort: {min: 1}},
+                                    {
+                                        $group: {
+                                            _id: '$uuid',
+                                            routingTable: {$push: '$$ROOT'},
+                                        }
+                                    },
+                                    {$lookup: {
+                                        from: 'collections',
+                                        localField: '_id',
+                                        foreignField: 'uuid',
+                                        as: 'details'
+                                    },
+                                    }
+                                ])
+                                .toArray();
+    };
+
+    const checkCollRoutingTable = (nss, shardKeyPattern, routingTable) => {
+        if (!routingTable) {
+            jsTest.log(`config.collections entry '${nss}' has no chunks`);
+            return false;
+        }
+
+        const shardKeyfields = Object.keys(shardKeyPattern);
+        let lastLowerBound = {};
+        for (const key of shardKeyfields) {
+            lastLowerBound[key] = MinKey;
+        }
+        let expectedLastBound = {};
+        for (const key of shardKeyfields) {
+            expectedLastBound[key] = MaxKey;
+        }
+        for (const chunk of routingTable) {
+            if (!sameObjectFields(Object.keys(chunk.min), shardKeyfields) ||
+                !sameObjectFields(Object.keys(chunk.max), shardKeyfields)) {
+                jsTest.log(`Shard key pattern violation found in config.chunks for ${
+                    nss}! Expected: ${tojson(shardKeyPattern)}, found chunk ${tojson(chunk)}`);
+                return false;
+            }
+
+            if (bsonWoCompare(chunk.min, lastLowerBound) !== 0) {
+                jsTest.log(`Found gap or range overlap in config.chunks for collection ${
+                    nss}, chunk ${tojson(chunk._id)}! Expected ${tojson(lastLowerBound)}, found ${
+                    tojson(chunk.min)}`);
+                return false;
+            }
+            lastLowerBound = chunk.max;
+        }
+
+        if (bsonWoCompare(routingTable[routingTable.length - 1].max, expectedLastBound) !== 0) {
+            jsTest.log(
+                `Incomplete range key detected in config.chunks for ${nss} (MaxKey missing)`);
+            return false;
+        }
+
+        jsTest.log(`${nss} with ${routingTable.length} chunks passed the check`);
+        return true;
+    };
+
+    const run = (mongos) => {
+        try {
+            jsTest.log('Checking routing table consistency');
+            // Group docs in config.chunks by coll UUID (sorting by minKey),
+            // then join with docs in config.collections.
+            const testCollectionsWithRoutingTable = fetchRoutingTableData(mongos);
+
+            for (const collData of testCollectionsWithRoutingTable) {
+                // Test invariant
+                assert.lte(
+                    1,
+                    collData.details.length,
+                    'Possible bug in aggregation generating testCollectionsWithRoutingTable');
+
+                // Test checks
+                assert.eq(
+                    1,
+                    collData.details.length,
+                    `Found entries in config.chunks with no match in config.collections! Details: ${
+                        tojson(collData)}`);
+                assert(checkCollRoutingTable(
+                           collData.details[0]._id, collData.details[0].key, collData.routingTable),
+                       `Corrupted routing table detected for ${collData._id}! Details: ${
+                           tojson(collData)}`);
+            }
+        } catch (e) {
+            if (e.code !== ErrorCodes.Unauthorized) {
+                throw e;
+            }
+            jsTest.log(
+                'Skipping check of routing table consistency - access to admin collections is not authorized');
+        }
+        jsTest.log('Routing table consistency check completed');
+    };
+
+    return {
+        run: run,
+    };
+})();
diff --git a/jstests/libs/override_methods/check_routing_table_consistency.js b/jstests/libs/override_methods/check_routing_table_consistency.js
new file mode 100644
index 00000000000..c9a47714132
--- /dev/null
+++ b/jstests/libs/override_methods/check_routing_table_consistency.js
@@ -0,0 +1,16 @@
+'use strict';
+
+load('jstests/libs/check_routing_table_consistency_helpers.js');
+
+ShardingTest.prototype.checkRoutingTableConsistency = function() {
+    if (jsTest.options().skipCheckRoutingTableConsistency) {
+        jsTest.log("Skipping routing table consistency check");
+        return;
+    }
+
+    const mongos = new Mongo(this.s.host);
+    mongos.fullOptions = this.s.fullOptions || {};
+    mongos.setReadPref('primaryPreferred');
+
+    RoutingTableConsistencyChecker.run(mongos);
+};
diff --git a/jstests/noPassthrough/unix_socket.js b/jstests/noPassthrough/unix_socket.js
index 32c36ed330b..0b24d0e9739 100644
--- a/jstests/noPassthrough/unix_socket.js
+++ b/jstests/noPassthrough/unix_socket.js
@@ -22,6 +22,7 @@ if (_isWindows()) {
 // Checking index consistency involves reconnecting to the mongos.
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 TestData.skipCheckOrphans = true;
+TestData.skipCheckRoutingTableConsistency = true;
 
 // Do not fail if this test leaves unterminated processes because testSockOptions
 // is expected to throw before it calls stopMongod.
diff --git a/jstests/sharding/all_config_servers_blackholed_from_mongos.js b/jstests/sharding/all_config_servers_blackholed_from_mongos.js
index 9279bc94df4..a0368f56bff 100644
--- a/jstests/sharding/all_config_servers_blackholed_from_mongos.js
+++ b/jstests/sharding/all_config_servers_blackholed_from_mongos.js
@@ -47,5 +47,10 @@ assert.writeError(
     st.s.getDB('NonExistentDB')
         .TestColl.insert({_id: 0, value: 'This value will never be inserted'}, {maxTimeMS: 15000}));
 
+jsTest.log('Resuming communication between mongos and config servers before teardown');
+st.forEachConfigServer((configSvr) => {
+    configSvr.discardMessagesFrom(st.s, 0.0);
+});
+
 st.stop();
 }());
diff --git a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
index a2ed9d5b025..85340c5436f 100644
--- a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
+++ b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
@@ -80,5 +80,7 @@ for (var i = 0; i < 2; i++) {
     }
 }
 
+// Restart one config server node to ensure that teardown checks may be executed
+st.restartConfigServer(0);
 st.stop();
 }());
diff --git a/jstests/sharding/convert_to_and_from_sharded.js b/jstests/sharding/convert_to_and_from_sharded.js
index 02e29bcef0e..a401b47b5fe 100644
--- a/jstests/sharding/convert_to_and_from_sharded.js
+++ b/jstests/sharding/convert_to_and_from_sharded.js
@@ -10,6 +10,9 @@
 // This test calls removeShard which can leave docs in config.rangeDeletions in state "pending",
 // therefore preventing orphans from being cleaned up.
 TestData.skipCheckOrphans = true;
+// Disable the check, since it causes the test to hang when run within the
+// 'sharding_batched_deletes_passthrough' suite
+TestData.skipCheckRoutingTableConsistency = true;
 
 var NUM_NODES = 3;
 
@@ -125,6 +128,6 @@ replShard.awaitNodesAgreeOnPrimary();
 priConn = replShard.getPrimary();
 checkBasicCRUD(priConn.getDB('test').unsharded);
 checkBasicCRUD(priConn.getDB('test').sharded);
-
+jsTest.log("TROLL!!");
 replShard.stopSet();
 })();
diff --git a/jstests/sharding/lagged_config_secondary.js b/jstests/sharding/lagged_config_secondary.js
index 196ca38f537..b0b22022401 100644
--- a/jstests/sharding/lagged_config_secondary.js
+++ b/jstests/sharding/lagged_config_secondary.js
@@ -11,6 +11,7 @@ load("jstests/libs/write_concern_util.js");
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 TestData.skipCheckOrphans = true;
+TestData.skipCheckRoutingTableConsistency = true;
 
 (function() {
 
diff --git a/jstests/sharding/mongos_get_shard_version.js b/jstests/sharding/mongos_get_shard_version.js
index a62d67585a3..6314d6c706d 100644
--- a/jstests/sharding/mongos_get_shard_version.js
+++ b/jstests/sharding/mongos_get_shard_version.js
@@ -17,6 +17,10 @@ if (isStepdownSuite && isCodeCoverageEnabled) {
     return;
 }
 
+// Disable the check (the size of the chunk boundaries generated by this test causes the aggregation
+// query collecting the routing table metadata to fail due to max document size exceeded).
+TestData.skipCheckRoutingTableConsistency = true;
+
 const st = new ShardingTest({
     shards: 2,
     other: {mongosOptions: {setParameter: {enableFinerGrainedCatalogCacheRefresh: true}}}
diff --git a/jstests/sharding/primary_config_server_blackholed_from_mongos.js b/jstests/sharding/primary_config_server_blackholed_from_mongos.js
index df08df55db7..ac3be0782f4 100644
--- a/jstests/sharding/primary_config_server_blackholed_from_mongos.js
+++ b/jstests/sharding/primary_config_server_blackholed_from_mongos.js
@@ -83,5 +83,9 @@ assert.lt(0, configDB.chunks.find().itcount());
 assert.lt(0, configDB.chunks.find().count());
 assert.lt(0, configDB.chunks.aggregate().itcount());
 
+jsTest.log('Remove network partition before tearing down');
+configPrimary.discardMessagesFrom(st.s, 0.0);
+st.s.discardMessagesFrom(configPrimary, 0.0);
+
 st.stop();
 }());
diff --git a/jstests/ssl/sni_name_advertisement.js b/jstests/ssl/sni_name_advertisement.js
index 938f7ceb230..a2251e8c5e2 100644
--- a/jstests/ssl/sni_name_advertisement.js
+++ b/jstests/ssl/sni_name_advertisement.js
@@ -9,6 +9,7 @@ load('jstests/ssl/libs/ssl_helpers.js');
 // Checking index consistency involves reconnecting to the mongos.
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 TestData.skipCheckOrphans = true;
+TestData.skipCheckRoutingTableConsistency = true;
 
 let path = "jstests/libs/";
 let pemKeyFile = path + "server.pem";
@@ -80,4 +81,4 @@ jsTestLog("Testing sharded configuration bound to IP " + testIP);
 assert.eq(desiredOutput,
           getSNISharded(ipParams),
           "IP address is advertised as SNI name in sharded mongod");
-})();
\ No newline at end of file
+})();
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index 63dd41857c2..e505085c731 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -380,6 +380,7 @@ var ShardingTest = function(params) {
         this.checkUUIDsConsistentAcrossCluster();
         this.checkIndexesConsistentAcrossCluster();
         this.checkOrphansAreDeleted();
+        this.checkRoutingTableConsistency();
 
         if (jsTestOptions().alwaysUseLogFiles) {
             if (opts.noCleanData === false) {
@@ -1753,3 +1754,7 @@ ShardingTest.prototype.checkIndexesConsistentAcrossCluster = function() {};
 ShardingTest.prototype.checkOrphansAreDeleted = function() {
     print("Unhooked function");
 };
+
+ShardingTest.prototype.checkRoutingTableConsistency = function() {
+    print("Unhooked checkRoutingTableConsistency function");
+};
diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js
index ebfa03f7268..15bcd84f5c7 100644
--- a/src/mongo/shell/utils.js
+++ b/src/mongo/shell/utils.js
@@ -413,6 +413,7 @@ jsTestOptions = function() {
             // is shut down.
             alwaysUseLogFiles: TestData.alwaysUseLogFiles || false,
             skipCheckOrphans: TestData.skipCheckOrphans || false,
+            skipCheckRoutingTableConsistency: TestData.skipCheckRoutingTableConsistency || false,
             inEvergreen: TestData.inEvergreen || false,
 
             undoRecorderPath: TestData.undoRecorderPath,