SERVER-44917 Add ShardingTest hook to validate indexes are consistent across shards on shutdown

35 files changed, 231 insertions, 51 deletions

diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index 0b231dbd4ae..45a13e64525 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -335,6 +335,10 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
     eval_sb.append(
         "load('jstests/libs/override_methods/check_uuids_consistent_across_cluster.js');")
 
+    # Load a callback to check index consistency before shutting down a ShardingTest.
+    eval_sb.append(
+        "load('jstests/libs/override_methods/check_indexes_consistent_across_cluster.js');")
+
     # Load this file to retry operations that fail due to in-progress background operations.
     eval_sb.append(
         "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');")
diff --git a/jstests/aggregation/sharded_agg_cleanup_on_error.js b/jstests/aggregation/sharded_agg_cleanup_on_error.js
index fb646819d06..1a57fffe018 100644
--- a/jstests/aggregation/sharded_agg_cleanup_on_error.js
+++ b/jstests/aggregation/sharded_agg_cleanup_on_error.js
@@ -80,7 +80,7 @@ try {
         errCode: kDivideByZeroErrCode
     });
 } finally {
-    assert.commandWorked(shard0DB.adminCommand({configureFailPoint: kFailPointName, mode: "off"}));
+    assert.commandWorked(shard1DB.adminCommand({configureFailPoint: kFailPointName, mode: "off"}));
 }
 
 // Test that aggregations which fail to establish a merging shard cursor also cleanup the open
diff --git a/jstests/auth/access_control_with_unreachable_configs.js b/jstests/auth/access_control_with_unreachable_configs.js
index 69cb9140720..285cb9a9cee 100644
--- a/jstests/auth/access_control_with_unreachable_configs.js
+++ b/jstests/auth/access_control_with_unreachable_configs.js
@@ -4,6 +4,9 @@
 // there are.
 // @tags: [requires_sharding]
 
+// The config servers are not reachable at shutdown.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 var st = new ShardingTest({
     shards: 1,
     mongos: 1,
diff --git a/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js b/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js
index 148e9d1dbfc..83eb8802076 100644
--- a/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js
+++ b/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js
@@ -7,6 +7,10 @@ load('jstests/ssl/libs/ssl_helpers.js');
 (function() {
 'use strict';
 
+// TODO (SERVER-45108): This test doesn't specify authentication mode through 'options' and
+// authutil.asCluster() only checks jsTest.options().clusterAuthMode.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 // Disable auth explicitly
 var noAuthOptions = {noauth: ''};
 var transitionToAuthOptions = {clusterAuthMode: 'keyFile', keyFile: KEYFILE, transitionToAuth: ''};
diff --git a/jstests/gle/gle_sharded_wc.js b/jstests/gle/gle_sharded_wc.js
index 6e3cc919212..b5c468a045f 100644
--- a/jstests/gle/gle_sharded_wc.js
+++ b/jstests/gle/gle_sharded_wc.js
@@ -5,9 +5,10 @@
 // so cannot be run on the ephemeralForTest storage engine, as it accepts all journaled writes.
 // @tags: [SERVER-21420]
 
-// Checking UUID consistency involves talking to the shard primaries, but by the end of this test,
-// one shard does not have a primary.
+// Checking UUID and index consistency involves talking to the shard primaries, but by the end of
+// this test, one shard does not have a primary.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 TestData.skipCheckDBHashes = true;
 
 (function() {
diff --git a/jstests/gle/gle_sharded_write.js b/jstests/gle/gle_sharded_write.js
index 537787aa1fe..b67ab34ec1a 100644
--- a/jstests/gle/gle_sharded_write.js
+++ b/jstests/gle/gle_sharded_write.js
@@ -3,8 +3,10 @@
 // Note that test should work correctly with and without write commands.
 //
 
-// Checking UUID consistency involves talking to shards, but this test shuts down one shard.
+// Checking UUID and index consistency involves talking to shards, but this test shuts down one
+// shard.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/libs/override_methods/check_indexes_consistent_across_cluster.js b/jstests/libs/override_methods/check_indexes_consistent_across_cluster.js
new file mode 100644
index 00000000000..228d62e6bab
--- /dev/null
+++ b/jstests/libs/override_methods/check_indexes_consistent_across_cluster.js
@@ -0,0 +1,111 @@
+/**
+ * Provides a hook to check that indexes are consistent across the sharded cluster.
+ *
+ * The hook checks that for every collection, all the shards that own chunks for the
+ * collection have the same indexes.
+ */
+"use strict";
+
+ShardingTest.prototype.checkIndexesConsistentAcrossCluster = function() {
+    if (jsTest.options().skipCheckingIndexesConsistentAcrossCluster) {
+        print("Skipping index consistency check across the cluster");
+        return;
+    }
+
+    /**
+     * Returns true if the array contains the given BSON object.
+     */
+    function containsBSON(arr, targetObj) {
+        for (const obj of arr) {
+            if (bsonWoCompare(obj, targetObj) === 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    print("Checking consistency of indexes across the cluster");
+
+    const mongos = new Mongo(this.s.host);
+    const keyFile = this.keyFile;
+
+    // TODO (SERVER-45017): Remove this check when v4.4 becomes last-stable.
+    const isMixedVersion = this.isMixedVersionCluster();
+
+    /**
+     * Returns an array of config.collections docs for undropped collections.
+     */
+    function getCollDocs() {
+        return mongos.getDB("config").collections.find({dropped: false}).toArray();
+    }
+
+    /**
+     * Returns a function that returns an array of index docs for the namespace grouped
+     * by shard.
+     */
+    function makeGetIndexDocsFunc(ns) {
+        return () => {
+            mongos.setReadPref("primary");
+            if (isMixedVersion) {
+                return mongos.getCollection(ns)
+                    .aggregate([
+                        {$indexStats: {}},
+                        {$group: {_id: "$host", indexes: {$push: {key: "$key", name: "$name"}}}},
+                        {$project: {_id: 0, host: "$_id", indexes: 1}}
+                    ])
+                    .toArray();
+            }
+            return mongos.getCollection(ns)
+                .aggregate([
+                    {$indexStats: {}},
+                    {$group: {_id: "$shard", indexes: {$push: {spec: "$spec"}}}},
+                    {$project: {_id: 0, shard: "$_id", indexes: 1}}
+                ])
+                .toArray();
+        };
+    }
+
+    const collDocs = keyFile ? authutil.asCluster(mongos, keyFile, getCollDocs) : getCollDocs();
+    for (const collDoc of collDocs) {
+        const ns = collDoc._id;
+        const getIndexDocsForNs = makeGetIndexDocsFunc(ns);
+        print(`Checking that the indexes for ${ns} are consistent across shards...`);
+
+        // Find the indexes on each shard. For example:
+        // [{"shard" : "rs0",
+        //   "indexes" : [{"spec" : {"v" : 2, "key" : {"_id" : 1}, "name" : "_id_"}},
+        //                {"spec" : {"v" : 2, "key" : {"x" : 1}, "name" : "x_1"}}]},
+        //  {"shard" : "rs1",
+        //   "indexes" : [{"spec" : {"v" : 2, "key" : {"_id" :1}, "name" : "_id_"}}]}];
+        const indexDocs =
+            keyFile ? authutil.asCluster(mongos, keyFile, getIndexDocsForNs) : getIndexDocsForNs();
+
+        if (indexDocs.length == 0) {
+            print(`Found no indexes for ${ns}, skipping index consistency check`);
+            continue;
+        }
+
+        // Find indexes that exist on all shards. For the example above:
+        // [{"spec" : {"v" : 2, "key" : {"_id" : 1}, "name" : "_id_"}}];
+        let consistentIndexes = indexDocs[0].indexes;
+        for (let i = 1; i < indexDocs.length; i++) {
+            consistentIndexes =
+                consistentIndexes.filter(index => containsBSON(indexDocs[i].indexes, index));
+        }
+
+        // Find inconsistent indexes. For the example above:
+        // {"rs0": [{"spec" : {"v" : 2, "key" : {"_id" : 1}, "name" : "_id_"}}], "rs1" : []};
+        const inconsistentIndexesOnShard = {};
+        let isConsistent = true;
+        for (const indexDoc of indexDocs) {
+            const inconsistentIndexes =
+                indexDoc.indexes.filter(index => !containsBSON(consistentIndexes, index));
+            inconsistentIndexesOnShard[isMixedVersion ? indexDoc.host : indexDoc.shard] =
+                inconsistentIndexes;
+            isConsistent = inconsistentIndexes.length === 0;
+        }
+
+        assert(isConsistent,
+               `found inconsistent indexes for ${ns}: ${tojson(inconsistentIndexesOnShard)}`);
+    }
+};
diff --git a/jstests/noPassthrough/currentop_two_phase_coordinator_metrics.js b/jstests/noPassthrough/currentop_two_phase_coordinator_metrics.js
index f5cdc047a48..69375b3818c 100644
--- a/jstests/noPassthrough/currentop_two_phase_coordinator_metrics.js
+++ b/jstests/noPassthrough/currentop_two_phase_coordinator_metrics.js
@@ -309,6 +309,8 @@ results = adminDB.aggregate([{$currentOp: {'allUsers': true}}, {$match: filter}]
 jsTest.log(`Curop result(s): ${tojson(results)}`);
 assert.eq(1, results.length);
 assertCuropFields(coordinator, commitStartCutoff, 'inactive', [], null, null, results[0]);
+
+adminDB.logout();
 })();
 
 st.stop();
diff --git a/jstests/noPassthrough/unix_socket.js b/jstests/noPassthrough/unix_socket.js
index fc1ad2abf58..a6f0b1d6bf0 100644
--- a/jstests/noPassthrough/unix_socket.js
+++ b/jstests/noPassthrough/unix_socket.js
@@ -16,6 +16,9 @@ if (_isWindows()) {
     return;
 }
 
+// Checking index consistency involves reconnecting to the mongos.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 // Do not fail if this test leaves unterminated processes because testSockOptions
 // is expected to throw before it calls stopMongod.
 TestData.failIfUnterminatedProcesses = false;
diff --git a/jstests/sharding/all_config_servers_blackholed_from_mongos.js b/jstests/sharding/all_config_servers_blackholed_from_mongos.js
index 8fb040e60f7..1acb63f4408 100644
--- a/jstests/sharding/all_config_servers_blackholed_from_mongos.js
+++ b/jstests/sharding/all_config_servers_blackholed_from_mongos.js
@@ -2,11 +2,12 @@
  * Ensures that if the config servers are blackholed from the point of view of MongoS, metadata
  * operations do not get stuck forever.
  *
- * Checking UUID consistency involves talking to config servers through mongos, but mongos is
- * blackholed from the config servers in this test.
+ * Checking UUID and index consistency involves talking to config servers through mongos, but mongos
+ * is blackholed from the config servers in this test.
  */
 
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
index 19b19c724d3..4e9ae887bdc 100644
--- a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
+++ b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
@@ -2,9 +2,10 @@
  * Shuts down config server and shard replica set nodes one by one and ensures correct behaviour.
  */
 
-// Checking UUID consistency involves talking to the config servers, which are shut down in this
-// test.
+// Checking UUID and index consistency involves talking to the config servers, which are shut down
+// in this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/allow_partial_results.js b/jstests/sharding/allow_partial_results.js
index 7a52a15e81a..bef3c9083a6 100644
--- a/jstests/sharding/allow_partial_results.js
+++ b/jstests/sharding/allow_partial_results.js
@@ -5,6 +5,7 @@
 
 // This test shuts down a shard.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/allow_partial_results_nshards.js b/jstests/sharding/allow_partial_results_nshards.js
index d87a3f71934..26aa7198500 100644
--- a/jstests/sharding/allow_partial_results_nshards.js
+++ b/jstests/sharding/allow_partial_results_nshards.js
@@ -15,8 +15,10 @@ TestData.disableImplicitSessions = true;
 // check would be lost.
 TestData.skipGossipingClusterTime = true;
 
-// Don't check for consistency across the cluster at the end, since the test shuts down a shard.
+// Don't check for UUID and index consistency across the cluster at the end, since the test shuts
+// down a shard.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 // Set up a 2-shard single-node replicaset cluster.
 const st = new ShardingTest({name: jsTestName(), shards: 2, rs: {nodes: 1}});
diff --git a/jstests/sharding/auth_no_config_primary.js b/jstests/sharding/auth_no_config_primary.js
index a9ed9a5c21a..7e865eeb154 100644
--- a/jstests/sharding/auth_no_config_primary.js
+++ b/jstests/sharding/auth_no_config_primary.js
@@ -6,9 +6,10 @@
  * @tags: [requires_persistence]
  */
 
-// Checking UUID consistency involves talking to the config server primary, but there is no config
-// server primary by the end of this test.
+// Checking UUID and index consistency involves talking to the config server primary, but there is
+// no config server primary by the end of this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 TestData.skipCheckDBHashes = true;
 
 (function() {
diff --git a/jstests/sharding/batch_write_command_sharded.js b/jstests/sharding/batch_write_command_sharded.js
index 854e1364531..f3f260c1d83 100644
--- a/jstests/sharding/batch_write_command_sharded.js
+++ b/jstests/sharding/batch_write_command_sharded.js
@@ -4,9 +4,10 @@
 // *only* mongos-specific tests.
 //
 
-// Checking UUID consistency involves talking to the config server primary, but there is no config
-// server primary by the end of this test.
+// Checking UUID and index consistency involves talking to the config server primary, but there is
+// no config server primary by the end of this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/config_rs_no_primary.js b/jstests/sharding/config_rs_no_primary.js
index 41d74869930..e4a90569c87 100644
--- a/jstests/sharding/config_rs_no_primary.js
+++ b/jstests/sharding/config_rs_no_primary.js
@@ -1,9 +1,10 @@
 // Tests operation of the cluster when the config servers have no primary and thus the cluster
 // metadata is in read-only mode.
 
-// Checking UUID consistency involves talking to the config server primary, but there is no config
-// server primary by the end of this test.
+// Checking UUID and index consistency involves talking to the config server primary, but there is
+// no config server primary by the end of this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/count_config_servers.js b/jstests/sharding/count_config_servers.js
index 7df3c4b8843..4d358191c59 100644
--- a/jstests/sharding/count_config_servers.js
+++ b/jstests/sharding/count_config_servers.js
@@ -3,9 +3,10 @@
  * This test fails when run with authentication due to SERVER-6327
  */
 
-// Checking UUID consistency requires querying the config primary, but this test shuts down 2 out of
-// the 3 config servers. Therefore, we cannot do the check on this test.
+// Checking UUID and index consistency requires querying the config primary, but this test
+// shuts down 2 out of the 3 config servers. Therefore, we cannot do the check on this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/lagged_config_secondary.js b/jstests/sharding/lagged_config_secondary.js
index 7fecb777de5..06b186c8666 100644
--- a/jstests/sharding/lagged_config_secondary.js
+++ b/jstests/sharding/lagged_config_secondary.js
@@ -3,9 +3,10 @@
  * are behind the majority opTime.
  */
 
-// Checking UUID consistency involves mongos being able to do a read from the config server, but
-// this test is designed to make mongos time out when reading from the config server.
+// Checking UUID and index consistency involves mongos being able to do a read from the config
+// server, but this test is designed to make mongos time out when reading from the config server.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 var st =
diff --git a/jstests/sharding/list_indexes_shard_targeting.js b/jstests/sharding/list_indexes_shard_targeting.js
index 6571679596e..f52748819a6 100644
--- a/jstests/sharding/list_indexes_shard_targeting.js
+++ b/jstests/sharding/list_indexes_shard_targeting.js
@@ -9,6 +9,9 @@
 
 load("jstests/sharding/libs/shard_versioning_util.js");
 
+// This test makes shards have inconsistent indexes.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 const st = new ShardingTest({shards: 3});
 const dbName = "test";
 const collName = "user";
diff --git a/jstests/sharding/localhostAuthBypass.js b/jstests/sharding/localhostAuthBypass.js
index 6dd8a9d6d62..31103df7fe4 100644
--- a/jstests/sharding/localhostAuthBypass.js
+++ b/jstests/sharding/localhostAuthBypass.js
@@ -3,10 +3,11 @@
 // This test is to ensure that localhost authentication works correctly against a sharded
 // cluster whether they are hosted with "localhost" or a hostname.
 
-// Checking UUID consistency, which occurs on ShardingTest.stop, involves using a mongos to read
-// data on the config server, but this test uses a special shutdown function which stops the
-// mongoses before calling ShardingTest.stop.
+// Checking UUID and index consistency, which occurs on ShardingTest.stop, involves using a
+// mongos to read data on the config server, but this test uses a special shutdown function
+// which stops the mongoses before calling ShardingTest.stop.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
index 226548fa88e..dde1f029ad4 100644
--- a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
+++ b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
@@ -11,9 +11,10 @@
 // (connection connected after shard change).
 //
 
-// Checking UUID consistency involves talking to shard primaries, but by the end of this test, one
-// shard does not have a primary.
+// Checking UUID and index consistency involves talking to shard primaries, but by the end of this
+// test, one shard does not have a primary.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 // Replica set nodes started with --shardsvr do not enable key generation until they are added to a
 // sharded cluster and reject commands with gossiped clusterTime from users without the
diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js
index b6fffb409e5..b58774eb510 100644
--- a/jstests/sharding/mongos_rs_shard_failure_tolerance.js
+++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js
@@ -11,9 +11,10 @@
 // (connection connected after shard change).
 //
 
-// Checking UUID consistency involves talking to shard primaries, but by the end of this test, one
-// shard does not have a primary.
+// Checking UUID and index consistency involves talking to shard primaries, but by the end of this
+// test, one shard does not have a primary.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/mongos_shard_failure_tolerance.js b/jstests/sharding/mongos_shard_failure_tolerance.js
index 479ca437f97..9707d8f1ce9 100644
--- a/jstests/sharding/mongos_shard_failure_tolerance.js
+++ b/jstests/sharding/mongos_shard_failure_tolerance.js
@@ -10,8 +10,9 @@
 // sequence), idle (connection is connected but not used before a shard change), and new
 // (connection connected after shard change).
 //
-// Checking UUID consistency involves talking to shards, but this test shuts down shards.
+// Checking UUID and index consistency involves talking to shards, but this test shuts down shards.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/move_chunk_missing_idx.js b/jstests/sharding/move_chunk_missing_idx.js
index e52d95f722a..16cc64ddddd 100644
--- a/jstests/sharding/move_chunk_missing_idx.js
+++ b/jstests/sharding/move_chunk_missing_idx.js
@@ -3,6 +3,9 @@
  * have the index and is not empty.
  */
 
+// This test runs dropIndex on one of the shards.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 var st = new ShardingTest({shards: 2});
 
 var testDB = st.s.getDB('test');
diff --git a/jstests/sharding/primary_config_server_blackholed_from_mongos.js b/jstests/sharding/primary_config_server_blackholed_from_mongos.js
index 1b2dfbc15b5..06afeb2abfc 100644
--- a/jstests/sharding/primary_config_server_blackholed_from_mongos.js
+++ b/jstests/sharding/primary_config_server_blackholed_from_mongos.js
@@ -3,6 +3,10 @@
 (function() {
 'use strict';
 
+// Checking index consistency involves talking to the primary config server which is blackholed from
+// the mongos in this test.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 var st = new ShardingTest({shards: 2, mongos: 1, useBridge: true});
 
 var testDB = st.s.getDB('BlackHoleDB');
diff --git a/jstests/sharding/read_pref_cmd.js b/jstests/sharding/read_pref_cmd.js
index 47b88405dab..8f3139b2de6 100644
--- a/jstests/sharding/read_pref_cmd.js
+++ b/jstests/sharding/read_pref_cmd.js
@@ -10,9 +10,10 @@ load("jstests/replsets/rslib.js");
 
 var NODE_COUNT = 2;
 
-// Checking UUID consistency involves reading from the config server through mongos, but this test
-// sets an invalid readPreference on the connection to the mongos.
+// Checking UUID and index consistency involves reading from the config server through mongos, but
+// this test sets an invalid readPreference on the connection to the mongos.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 /**
  * Prepare to call testReadPreference() or assertFailure().
diff --git a/jstests/sharding/return_partial_shards_down.js b/jstests/sharding/return_partial_shards_down.js
index 11a28bf6364..145daa1bc0c 100644
--- a/jstests/sharding/return_partial_shards_down.js
+++ b/jstests/sharding/return_partial_shards_down.js
@@ -2,8 +2,9 @@
 // Tests that zero results are correctly returned with returnPartial and shards down
 //
 
-// Checking UUID consistency involves talking to shards, but this test shuts down shards.
+// Checking UUID and index consistency involves talking to shards, but this test shuts down shards.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 var checkDocCount = function(coll, returnPartialFlag, shardsDown, expectedCount) {
     if (jsTestOptions().mongosBinVersion == "last-stable" && shardsDown) {
diff --git a/jstests/sharding/shard_insert_getlasterror_w2.js b/jstests/sharding/shard_insert_getlasterror_w2.js
index d5ee0becf29..38bb2db5fa3 100644
--- a/jstests/sharding/shard_insert_getlasterror_w2.js
+++ b/jstests/sharding/shard_insert_getlasterror_w2.js
@@ -1,9 +1,10 @@
 // replica set as solo shard
 // TODO: Add assertion code that catches hang
 
-// The UUID check must be able to contact the shard primaries, but this test manually stops 2/3
-// nodes of a replica set.
+// The UUID and index check must be able to contact the shard primaries, but this test manually
+// stops 2/3 nodes of a replica set.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/warm_up_connection_pool.js b/jstests/sharding/warm_up_connection_pool.js
index dc3f67f6940..ff034bd5523 100644
--- a/jstests/sharding/warm_up_connection_pool.js
+++ b/jstests/sharding/warm_up_connection_pool.js
@@ -2,8 +2,10 @@
  * @tags: [requires_fcv_44, multiversion_incompatible]
  */
 
-// Checking UUID consistency involves talking to a shard node, which in this test is shutdown
+// Checking UUID and index consistency involves talking to a shard node, which in this
+// test is shutdown.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/ssl/mixed_mode_sharded_transition.js b/jstests/ssl/mixed_mode_sharded_transition.js
index 8f9136b3e39..5835491bd4f 100644
--- a/jstests/ssl/mixed_mode_sharded_transition.js
+++ b/jstests/ssl/mixed_mode_sharded_transition.js
@@ -11,6 +11,10 @@ load('jstests/ssl/libs/ssl_helpers.js');
 (function() {
 'use strict';
 
+// TODO (SERVER-45108): authutil.asCluster() only works with a keyFile and we are not
+// currently threading the x509 options through to the check indexes hook.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 var transitionToX509AllowSSL =
     Object.merge(allowSSL, {transitionToAuth: '', clusterAuthMode: 'x509'});
 var transitionToX509PreferSSL =
diff --git a/jstests/ssl/sharding_with_x509.js b/jstests/ssl/sharding_with_x509.js
index d00417c24f1..58bce31aeea 100644
--- a/jstests/ssl/sharding_with_x509.js
+++ b/jstests/ssl/sharding_with_x509.js
@@ -3,6 +3,10 @@
 (function() {
 'use strict';
 
+// TODO (SERVER-45108): authutil.asCluster() only works with a keyFile and we are not
+// currently threading the x509 options through to the check indexes hook.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 var x509_options = {
     sslMode: "requireSSL",
     sslPEMKeyFile: "jstests/libs/server.pem",
diff --git a/jstests/ssl/sni_name_advertisement.js b/jstests/ssl/sni_name_advertisement.js
index 72c2fd90d56..d945703b4aa 100644
--- a/jstests/ssl/sni_name_advertisement.js
+++ b/jstests/ssl/sni_name_advertisement.js
@@ -6,6 +6,9 @@
 'use strict';
 load('jstests/ssl/libs/ssl_helpers.js');
 
+// Checking index consistency involves reconnecting to the mongos.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 let path = "jstests/libs/";
 let pemKeyFile = path + "server.pem";
 let caFile = path + "ca.pem";
diff --git a/jstests/ssl/ssl_get_more.js b/jstests/ssl/ssl_get_more.js
index 45a98b7ff75..ef0e4922167 100644
--- a/jstests/ssl/ssl_get_more.js
+++ b/jstests/ssl/ssl_get_more.js
@@ -1,6 +1,10 @@
 (function() {
 "use strict";
 
+// TODO (SERVER-45108): authutil.asCluster() only works with a keyFile and we are not
+// currently threading the x509 options through to the check indexes hook.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 var x509_options = {
     sslMode: "requireSSL",
     sslPEMKeyFile: "jstests/libs/server.pem",
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index 3e9bc205ba9..a5368bf7d58 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -388,6 +388,7 @@ var ShardingTest = function(params) {
 
     this.stop = function(opts) {
         this.checkUUIDsConsistentAcrossCluster();
+        this.checkIndexesConsistentAcrossCluster();
 
         this.stopAllMongos(opts);
 
@@ -977,7 +978,7 @@ var ShardingTest = function(params) {
      *     otherParams.configOptions.binVersion, otherParams.shardOptions.binVersion,
      *     otherParams.mongosOptions.binVersion
      */
-    function _isMixedVersionCluster() {
+    this.isMixedVersionCluster = function() {
         var lastStableBinVersion = MongoRunner.getBinVersionFor('last-stable');
 
         // Must check shardMixedBinVersion because it causes shardOptions.binVersion to be an object
@@ -1039,7 +1040,7 @@ var ShardingTest = function(params) {
         }
 
         return false;
-    }
+    };
 
     /**
      * Returns if there is a new feature compatibility version for the "latest" version. This must
@@ -1150,7 +1151,7 @@ var ShardingTest = function(params) {
             jsTestOptions().networkMessageCompressors;
     }
 
-    var keyFile = otherParams.keyFile;
+    this.keyFile = otherParams.keyFile;
     var hostName = otherParams.host === undefined ? getHostName() : otherParams.host;
 
     this._testName = testName;
@@ -1303,7 +1304,7 @@ var ShardingTest = function(params) {
                 useHostName: otherParams.useHostname,
                 useBridge: otherParams.useBridge,
                 bridgeOptions: otherParams.bridgeOptions,
-                keyFile: keyFile,
+                keyFile: this.keyFile,
                 protocolVersion: protocolVersion,
                 waitForKeys: false,
                 settings: rsSettings,
@@ -1322,7 +1323,7 @@ var ShardingTest = function(params) {
                 pathOpts: Object.merge(pathOpts, {shard: i}),
                 dbpath: "$testName$shard",
                 shardsvr: '',
-                keyFile: keyFile
+                keyFile: this.keyFile
             };
 
             options.setParameter = options.setParameter || {};
@@ -1400,7 +1401,7 @@ var ShardingTest = function(params) {
         host: hostName,
         useBridge: otherParams.useBridge,
         bridgeOptions: otherParams.bridgeOptions,
-        keyFile: keyFile,
+        keyFile: this.keyFile,
         waitForKeys: false,
         name: testName + "-configRS",
         seedRandomNumberGenerator: !randomSeedAlreadySet,
@@ -1492,8 +1493,8 @@ var ShardingTest = function(params) {
         var rs = this._rs[i].test;
         rs.getPrimary().getDB("admin").foo.save({x: 1});
 
-        if (keyFile) {
-            authutil.asCluster(rs.nodes, keyFile, function() {
+        if (this.keyFile) {
+            authutil.asCluster(rs.nodes, this.keyFile, function() {
                 rs.awaitReplication();
             });
         }
@@ -1527,7 +1528,7 @@ var ShardingTest = function(params) {
             useHostname: otherParams.useHostname,
             pathOpts: Object.merge(pathOpts, {mongos: i}),
             verbose: mongosVerboseLevel,
-            keyFile: keyFile,
+            keyFile: this.keyFile,
         };
 
         if (otherParams.mongosOptions && otherParams.mongosOptions.binVersion) {
@@ -1544,7 +1545,7 @@ var ShardingTest = function(params) {
     }
 
     const configRS = this.configRS;
-    if (_hasNewFeatureCompatibilityVersion() && _isMixedVersionCluster()) {
+    if (_hasNewFeatureCompatibilityVersion() && this.isMixedVersionCluster()) {
         function setFeatureCompatibilityVersion() {
             assert.commandWorked(
                 csrsPrimary.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
@@ -1554,8 +1555,8 @@ var ShardingTest = function(params) {
             configRS.awaitReplication();
         }
 
-        if (keyFile) {
-            authutil.asCluster(this.configRS.nodes, keyFile, setFeatureCompatibilityVersion);
+        if (this.keyFile) {
+            authutil.asCluster(this.configRS.nodes, this.keyFile, setFeatureCompatibilityVersion);
         } else {
             setFeatureCompatibilityVersion();
         }
@@ -1572,8 +1573,8 @@ var ShardingTest = function(params) {
             configRS.awaitLastOpCommitted();
         }
 
-        if (keyFile) {
-            authutil.asCluster(csrsPrimary, keyFile, setChunkSize);
+        if (this.keyFile) {
+            authutil.asCluster(csrsPrimary, this.keyFile, setChunkSize);
         } else {
             setChunkSize();
         }
@@ -1645,8 +1646,8 @@ var ShardingTest = function(params) {
 
     // If auth is enabled for the test, login the mongos connections as system in order to configure
     // the instances and then log them out again.
-    if (keyFile) {
-        authutil.asCluster(this._mongos, keyFile, _configureCluster);
+    if (this.keyFile) {
+        authutil.asCluster(this._mongos, this.keyFile, _configureCluster);
     } else if (mongosOptions[0] && mongosOptions[0].keyFile) {
         authutil.asCluster(this._mongos, mongosOptions[0].keyFile, _configureCluster);
     } else {
@@ -1730,13 +1731,13 @@ var ShardingTest = function(params) {
             for (let i = 0; i < numShards; i++) {
                 if (otherParams.rs || otherParams["rs" + i] || startShardsAsRS) {
                     const rs = this._rs[i].test;
-                    flushRT(rs.getPrimary(), rs.nodes, keyFile);
+                    flushRT(rs.getPrimary(), rs.nodes, this.keyFile);
                 } else {
                     // If specified, use the keyFile for the standalone shard.
                     const keyFileLocal = (otherParams.shards && otherParams.shards[i] &&
                                           otherParams.shards[i].keyFile)
                         ? otherParams.shards[i].keyFile
-                        : keyFile;
+                        : this.keyFile;
                     flushRT(this["shard" + i], this["shard" + i], keyFileLocal);
                 }
             }
@@ -1747,3 +1748,6 @@ var ShardingTest = function(params) {
 // Stub for a hook to check that collection UUIDs are consistent across shards and the config
 // server.
 ShardingTest.prototype.checkUUIDsConsistentAcrossCluster = function() {};
+
+// Stub for a hook to check that indexes are consistent across shards.
+ShardingTest.prototype.checkIndexesConsistentAcrossCluster = function() {};
diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js
index cb4d04d351e..66ceedfc951 100644
--- a/src/mongo/shell/utils.js
+++ b/src/mongo/shell/utils.js
@@ -315,6 +315,8 @@ jsTestOptions = function() {
             skipValidationNamespaces: TestData.skipValidationNamespaces || [],
             skipCheckingUUIDsConsistentAcrossCluster:
                 TestData.skipCheckingUUIDsConsistentAcrossCluster || false,
+            skipCheckingIndexesConsistentAcrossCluster:
+                TestData.skipCheckingIndexesConsistentAcrossCluster || false,
             skipCheckingCatalogCacheConsistencyWithShardingCatalog:
                 TestData.skipCheckingCatalogCacheConsistencyWithShardingCatalog || false,
             skipAwaitingReplicationOnShardsBeforeCheckingUUIDs: