SERVER-44917 Add ShardingTest hook to validate indexes are consistent across shards on shutdown

20 files changed, 58 insertions, 29 deletions

diff --git a/jstests/sharding/all_config_servers_blackholed_from_mongos.js b/jstests/sharding/all_config_servers_blackholed_from_mongos.js
index 8fb040e60f7..1acb63f4408 100644
--- a/jstests/sharding/all_config_servers_blackholed_from_mongos.js
+++ b/jstests/sharding/all_config_servers_blackholed_from_mongos.js
@@ -2,11 +2,12 @@
  * Ensures that if the config servers are blackholed from the point of view of MongoS, metadata
  * operations do not get stuck forever.
  *
- * Checking UUID consistency involves talking to config servers through mongos, but mongos is
- * blackholed from the config servers in this test.
+ * Checking UUID and index consistency involves talking to config servers through mongos, but mongos
+ * is blackholed from the config servers in this test.
  */
 
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
index 19b19c724d3..4e9ae887bdc 100644
--- a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
+++ b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
@@ -2,9 +2,10 @@
  * Shuts down config server and shard replica set nodes one by one and ensures correct behaviour.
  */
 
-// Checking UUID consistency involves talking to the config servers, which are shut down in this
-// test.
+// Checking UUID and index consistency involves talking to the config servers, which are shut down
+// in this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/allow_partial_results.js b/jstests/sharding/allow_partial_results.js
index 7a52a15e81a..bef3c9083a6 100644
--- a/jstests/sharding/allow_partial_results.js
+++ b/jstests/sharding/allow_partial_results.js
@@ -5,6 +5,7 @@
 
 // This test shuts down a shard.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/allow_partial_results_nshards.js b/jstests/sharding/allow_partial_results_nshards.js
index d87a3f71934..26aa7198500 100644
--- a/jstests/sharding/allow_partial_results_nshards.js
+++ b/jstests/sharding/allow_partial_results_nshards.js
@@ -15,8 +15,10 @@ TestData.disableImplicitSessions = true;
 // check would be lost.
 TestData.skipGossipingClusterTime = true;
 
-// Don't check for consistency across the cluster at the end, since the test shuts down a shard.
+// Don't check for UUID and index consistency across the cluster at the end, since the test shuts
+// down a shard.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 // Set up a 2-shard single-node replicaset cluster.
 const st = new ShardingTest({name: jsTestName(), shards: 2, rs: {nodes: 1}});
diff --git a/jstests/sharding/auth_no_config_primary.js b/jstests/sharding/auth_no_config_primary.js
index a9ed9a5c21a..7e865eeb154 100644
--- a/jstests/sharding/auth_no_config_primary.js
+++ b/jstests/sharding/auth_no_config_primary.js
@@ -6,9 +6,10 @@
  * @tags: [requires_persistence]
  */
 
-// Checking UUID consistency involves talking to the config server primary, but there is no config
-// server primary by the end of this test.
+// Checking UUID and index consistency involves talking to the config server primary, but there is
+// no config server primary by the end of this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 TestData.skipCheckDBHashes = true;
 
 (function() {
diff --git a/jstests/sharding/batch_write_command_sharded.js b/jstests/sharding/batch_write_command_sharded.js
index 854e1364531..f3f260c1d83 100644
--- a/jstests/sharding/batch_write_command_sharded.js
+++ b/jstests/sharding/batch_write_command_sharded.js
@@ -4,9 +4,10 @@
 // *only* mongos-specific tests.
 //
 
-// Checking UUID consistency involves talking to the config server primary, but there is no config
-// server primary by the end of this test.
+// Checking UUID and index consistency involves talking to the config server primary, but there is
+// no config server primary by the end of this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/config_rs_no_primary.js b/jstests/sharding/config_rs_no_primary.js
index 41d74869930..e4a90569c87 100644
--- a/jstests/sharding/config_rs_no_primary.js
+++ b/jstests/sharding/config_rs_no_primary.js
@@ -1,9 +1,10 @@
 // Tests operation of the cluster when the config servers have no primary and thus the cluster
 // metadata is in read-only mode.
 
-// Checking UUID consistency involves talking to the config server primary, but there is no config
-// server primary by the end of this test.
+// Checking UUID and index consistency involves talking to the config server primary, but there is
+// no config server primary by the end of this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/count_config_servers.js b/jstests/sharding/count_config_servers.js
index 7df3c4b8843..4d358191c59 100644
--- a/jstests/sharding/count_config_servers.js
+++ b/jstests/sharding/count_config_servers.js
@@ -3,9 +3,10 @@
  * This test fails when run with authentication due to SERVER-6327
  */
 
-// Checking UUID consistency requires querying the config primary, but this test shuts down 2 out of
-// the 3 config servers. Therefore, we cannot do the check on this test.
+// Checking UUID and index consistency requires querying the config primary, but this test
+// shuts down 2 out of the 3 config servers. Therefore, we cannot do the check on this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/lagged_config_secondary.js b/jstests/sharding/lagged_config_secondary.js
index 7fecb777de5..06b186c8666 100644
--- a/jstests/sharding/lagged_config_secondary.js
+++ b/jstests/sharding/lagged_config_secondary.js
@@ -3,9 +3,10 @@
  * are behind the majority opTime.
  */
 
-// Checking UUID consistency involves mongos being able to do a read from the config server, but
-// this test is designed to make mongos time out when reading from the config server.
+// Checking UUID and index consistency involves mongos being able to do a read from the config
+// server, but this test is designed to make mongos time out when reading from the config server.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 var st =
diff --git a/jstests/sharding/list_indexes_shard_targeting.js b/jstests/sharding/list_indexes_shard_targeting.js
index 6571679596e..f52748819a6 100644
--- a/jstests/sharding/list_indexes_shard_targeting.js
+++ b/jstests/sharding/list_indexes_shard_targeting.js
@@ -9,6 +9,9 @@
 
 load("jstests/sharding/libs/shard_versioning_util.js");
 
+// This test makes shards have inconsistent indexes.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 const st = new ShardingTest({shards: 3});
 const dbName = "test";
 const collName = "user";
diff --git a/jstests/sharding/localhostAuthBypass.js b/jstests/sharding/localhostAuthBypass.js
index 6dd8a9d6d62..31103df7fe4 100644
--- a/jstests/sharding/localhostAuthBypass.js
+++ b/jstests/sharding/localhostAuthBypass.js
@@ -3,10 +3,11 @@
 // This test is to ensure that localhost authentication works correctly against a sharded
 // cluster whether they are hosted with "localhost" or a hostname.
 
-// Checking UUID consistency, which occurs on ShardingTest.stop, involves using a mongos to read
-// data on the config server, but this test uses a special shutdown function which stops the
-// mongoses before calling ShardingTest.stop.
+// Checking UUID and index consistency, which occurs on ShardingTest.stop, involves using a
+// mongos to read data on the config server, but this test uses a special shutdown function
+// which stops the mongoses before calling ShardingTest.stop.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
index 226548fa88e..dde1f029ad4 100644
--- a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
+++ b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
@@ -11,9 +11,10 @@
 // (connection connected after shard change).
 //
 
-// Checking UUID consistency involves talking to shard primaries, but by the end of this test, one
-// shard does not have a primary.
+// Checking UUID and index consistency involves talking to shard primaries, but by the end of this
+// test, one shard does not have a primary.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 // Replica set nodes started with --shardsvr do not enable key generation until they are added to a
 // sharded cluster and reject commands with gossiped clusterTime from users without the
diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js
index b6fffb409e5..b58774eb510 100644
--- a/jstests/sharding/mongos_rs_shard_failure_tolerance.js
+++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js
@@ -11,9 +11,10 @@
 // (connection connected after shard change).
 //
 
-// Checking UUID consistency involves talking to shard primaries, but by the end of this test, one
-// shard does not have a primary.
+// Checking UUID and index consistency involves talking to shard primaries, but by the end of this
+// test, one shard does not have a primary.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/mongos_shard_failure_tolerance.js b/jstests/sharding/mongos_shard_failure_tolerance.js
index 479ca437f97..9707d8f1ce9 100644
--- a/jstests/sharding/mongos_shard_failure_tolerance.js
+++ b/jstests/sharding/mongos_shard_failure_tolerance.js
@@ -10,8 +10,9 @@
 // sequence), idle (connection is connected but not used before a shard change), and new
 // (connection connected after shard change).
 //
-// Checking UUID consistency involves talking to shards, but this test shuts down shards.
+// Checking UUID and index consistency involves talking to shards, but this test shuts down shards.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/move_chunk_missing_idx.js b/jstests/sharding/move_chunk_missing_idx.js
index e52d95f722a..16cc64ddddd 100644
--- a/jstests/sharding/move_chunk_missing_idx.js
+++ b/jstests/sharding/move_chunk_missing_idx.js
@@ -3,6 +3,9 @@
  * have the index and is not empty.
  */
 
+// This test runs dropIndex on one of the shards.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 var st = new ShardingTest({shards: 2});
 
 var testDB = st.s.getDB('test');
diff --git a/jstests/sharding/primary_config_server_blackholed_from_mongos.js b/jstests/sharding/primary_config_server_blackholed_from_mongos.js
index 1b2dfbc15b5..06afeb2abfc 100644
--- a/jstests/sharding/primary_config_server_blackholed_from_mongos.js
+++ b/jstests/sharding/primary_config_server_blackholed_from_mongos.js
@@ -3,6 +3,10 @@
 (function() {
 'use strict';
 
+// Checking index consistency involves talking to the primary config server which is blackholed from
+// the mongos in this test.
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+
 var st = new ShardingTest({shards: 2, mongos: 1, useBridge: true});
 
 var testDB = st.s.getDB('BlackHoleDB');
diff --git a/jstests/sharding/read_pref_cmd.js b/jstests/sharding/read_pref_cmd.js
index 47b88405dab..8f3139b2de6 100644
--- a/jstests/sharding/read_pref_cmd.js
+++ b/jstests/sharding/read_pref_cmd.js
@@ -10,9 +10,10 @@ load("jstests/replsets/rslib.js");
 
 var NODE_COUNT = 2;
 
-// Checking UUID consistency involves reading from the config server through mongos, but this test
-// sets an invalid readPreference on the connection to the mongos.
+// Checking UUID and index consistency involves reading from the config server through mongos, but
+// this test sets an invalid readPreference on the connection to the mongos.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 /**
  * Prepare to call testReadPreference() or assertFailure().
diff --git a/jstests/sharding/return_partial_shards_down.js b/jstests/sharding/return_partial_shards_down.js
index 11a28bf6364..145daa1bc0c 100644
--- a/jstests/sharding/return_partial_shards_down.js
+++ b/jstests/sharding/return_partial_shards_down.js
@@ -2,8 +2,9 @@
 // Tests that zero results are correctly returned with returnPartial and shards down
 //
 
-// Checking UUID consistency involves talking to shards, but this test shuts down shards.
+// Checking UUID and index consistency involves talking to shards, but this test shuts down shards.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 var checkDocCount = function(coll, returnPartialFlag, shardsDown, expectedCount) {
     if (jsTestOptions().mongosBinVersion == "last-stable" && shardsDown) {
diff --git a/jstests/sharding/shard_insert_getlasterror_w2.js b/jstests/sharding/shard_insert_getlasterror_w2.js
index d5ee0becf29..38bb2db5fa3 100644
--- a/jstests/sharding/shard_insert_getlasterror_w2.js
+++ b/jstests/sharding/shard_insert_getlasterror_w2.js
@@ -1,9 +1,10 @@
 // replica set as solo shard
 // TODO: Add assertion code that catches hang
 
-// The UUID check must be able to contact the shard primaries, but this test manually stops 2/3
-// nodes of a replica set.
+// The UUID and index check must be able to contact the shard primaries, but this test manually
+// stops 2/3 nodes of a replica set.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/warm_up_connection_pool.js b/jstests/sharding/warm_up_connection_pool.js
index dc3f67f6940..ff034bd5523 100644
--- a/jstests/sharding/warm_up_connection_pool.js
+++ b/jstests/sharding/warm_up_connection_pool.js
@@ -2,8 +2,10 @@
  * @tags: [requires_fcv_44, multiversion_incompatible]
  */
 
-// Checking UUID consistency involves talking to a shard node, which in this test is shutdown
+// Checking UUID and index consistency involves talking to a shard node, which in this
+// test is shutdown.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 
 (function() {
 'use strict';