SERVER-42191 Add a new test hook that checks that all orphaned documents are eventually deleted

 create mode 100644 jstests/libs/override_methods/check_orphans_are_deleted.js

50 files changed, 225 insertions, 0 deletions

diff --git a/buildscripts/resmokeconfig/suites/multiversion.yml b/buildscripts/resmokeconfig/suites/multiversion.yml
index eb8b8e458cd..88efab7d5a2 100644
--- a/buildscripts/resmokeconfig/suites/multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/multiversion.yml
@@ -23,3 +23,7 @@ executor:
   config:
     shell_options:
       nodb: ''
+      global_vars:
+        TestData:
+          # TODO: SERVER-45959 remove after v4.4
+          skipCheckOrphans: true
diff --git a/buildscripts/resmokeconfig/suites/multiversion_auth.yml b/buildscripts/resmokeconfig/suites/multiversion_auth.yml
index 2dfeaa609bd..df662863282 100644
--- a/buildscripts/resmokeconfig/suites/multiversion_auth.yml
+++ b/buildscripts/resmokeconfig/suites/multiversion_auth.yml
@@ -33,6 +33,8 @@ executor:
           authMechanism: SCRAM-SHA-1
           keyFile: *keyFile
           keyFileData: *keyFileData
+          # TODO: SERVER-45959 remove after v4.4
+          skipCheckOrphans: true
 
       authenticationDatabase: local
       authenticationMechanism: SCRAM-SHA-1
diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
index 6d5b596da65..c63b20551fe 100644
--- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
@@ -218,7 +218,9 @@ executor:
     shell_options:
       global_vars:
         TestData:
+          # TODO: SERVER-45994 remove
           skipCheckingCatalogCacheConsistencyWithShardingCatalog: true
+          skipCheckOrphans: true
       eval: "load('jstests/libs/override_methods/sharding_continuous_config_stepdown.js');"
       nodb: ''
       readMode: commands
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index 1b8f00e0984..28ae7ed1343 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -36,4 +36,6 @@ executor:
           mongosBinVersion: 'last-stable'
           shardMixedBinVersions: true
           skipCheckingUUIDsConsistentAcrossCluster: true
+          # TODO: SERVER-45959 remove after v4.4
+          skipCheckOrphans: true
       nodb: ''
diff --git a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
index b12aecbecd5..a84a90efa19 100644
--- a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
@@ -15,3 +15,5 @@ executor:
         TestData:
           useRandomBinVersionsWithinReplicaSet: true
           mongosBinVersion: 'last-stable'
+          # TODO: SERVER-45959 remove after v4.4
+          skipCheckOrphans: true
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index 78c67534a30..a63dd1952ce 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -339,6 +339,9 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
     eval_sb.append(
         "load('jstests/libs/override_methods/check_indexes_consistent_across_cluster.js');")
 
+    # Load a callback to check that all orphans are deleted before shutting down a ShardingTest.
+    eval_sb.append("load('jstests/libs/override_methods/check_orphans_are_deleted.js');")
+
     # Load this file to retry operations that fail due to in-progress background operations.
     eval_sb.append(
         "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');")
diff --git a/jstests/auth/access_control_with_unreachable_configs.js b/jstests/auth/access_control_with_unreachable_configs.js
index 285cb9a9cee..153b569bf62 100644
--- a/jstests/auth/access_control_with_unreachable_configs.js
+++ b/jstests/auth/access_control_with_unreachable_configs.js
@@ -6,6 +6,7 @@
 
 // The config servers are not reachable at shutdown.
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 var st = new ShardingTest({
     shards: 1,
diff --git a/jstests/auth/upgrade_noauth_to_keyfile.js b/jstests/auth/upgrade_noauth_to_keyfile.js
index 41eef5612c8..add604ec8f2 100644
--- a/jstests/auth/upgrade_noauth_to_keyfile.js
+++ b/jstests/auth/upgrade_noauth_to_keyfile.js
@@ -11,6 +11,7 @@ load('jstests/multiVersion/libs/multi_rs.js');
 // and sharded clusters as a user other than __system. Attempting to advance the clusterTime while
 // it has been signed with a dummy key results in an authorization error.
 TestData.skipGossipingClusterTime = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js b/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js
index 2f69ccbb3f2..9047c2cddef 100644
--- a/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js
+++ b/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js
@@ -9,6 +9,7 @@ load('jstests/ssl/libs/ssl_helpers.js');
 
 // IndexConsistencyCheck requires auth which ttA/ttA fails at.
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 // Disable auth explicitly
 var noAuthOptions = {noauth: ''};
diff --git a/jstests/libs/override_methods/check_orphans_are_deleted.js b/jstests/libs/override_methods/check_orphans_are_deleted.js
new file mode 100644
index 00000000000..f2d329df80b
--- /dev/null
+++ b/jstests/libs/override_methods/check_orphans_are_deleted.js
@@ -0,0 +1,119 @@
+/**
+ * Asserts that all shards in the sharded cluster doesn't own any orphan documents.
+ * Requires all shards and config server to have primary that is reachable.
+ *
+ * Note: Doesn't catch documents in the shard that doesn't have the full shard key.
+ * Assumes that all shards have the index that matches the shard key.
+ */
+ShardingTest.prototype.checkOrphansAreDeleted = function() {
+    if (jsTest.options().skipCheckOrphans) {
+        print("Skipping orphan check across the cluster");
+        return;
+    }
+
+    print('Running check orphans against cluster with mongos: ' + this.s.host);
+
+    let runCheck = function(mongosConn, shardConn, shardId) {
+        let configDB = shardConn.getDB('config');
+
+        let migrationCoordinatorDocs = [];
+        assert.soon(
+            () => {
+                try {
+                    migrationCoordinatorDocs = configDB.migrationCoordinators.find().toArray();
+                    return migrationCoordinatorDocs.length == 0;
+                } catch (exp) {
+                    // Primary purpose is to stabilize shell repl set monitor to recognize the
+                    // current primary.
+                    print('caught exception while checking migration coordinators, ' +
+                          'will retry again unless timed out: ' + tojson(exp));
+                }
+            },
+            () => {
+                return 'timed out waiting for migrationCoordinators to be empty @ ' + shardId +
+                    ', last known contents: ' + tojson(migrationCoordinatorDocs);
+            },
+            5 * 60 * 1000,
+            1000);
+
+        let rangeDeletions = [];
+        assert.soon(
+            () => {
+                rangeDeletions = configDB.rangeDeletions.find().toArray();
+                return rangeDeletions.length == 0;
+            },
+            () => {
+                return 'timed out waiting for rangeDeletions to be empty @ ' + shardId +
+                    ', last known contents: ' + tojson(rangeDeletions);
+            });
+
+        mongosConn.getDB('config').collections.find({dropped: false}).forEach(collDoc => {
+            let tempNsArray = collDoc._id.split('.');
+            let dbName = tempNsArray.shift();
+            let collName = tempNsArray.join('.');
+
+            let coll = shardConn.getDB(dbName)[collName];
+            mongosConn.getDB('config')
+                .chunks.find({ns: collDoc._id, shard: {$ne: shardId}})
+                .forEach(chunkDoc => {
+                    // Use $min/$max so this will also work with hashed and compound shard keys.
+                    let orphans = coll.find({})
+                                      .hint(collDoc.key)
+                                      .min(chunkDoc.min)
+                                      .max(chunkDoc.max)
+                                      .toArray();
+                    assert.eq(0,
+                              orphans.length,
+                              'found orphans @ ' + shardId + ' within chunk: ' + tojson(chunkDoc) +
+                                  ', orphans: ' + tojson(orphans));
+                });
+        });
+    };
+
+    let getConn = function(connStr) {
+        try {
+            return new Mongo(connStr);
+        } catch (exp) {
+            jsTest.log('Unable to connect to ' + connStr + ' while trying to check for orphans');
+            return null;
+        }
+    };
+
+    // Use a new connection so we don't have to worry about existing users logged in to the
+    // connection.
+    let mongosConn = new Mongo(this.s.host);
+    mongosConn.fullOptions = Object.merge(this.s.fullOptions, {});
+
+    const keyFile = this.keyFile;
+    if (keyFile || mongosConn.fullOptions.clusterAuthMode == 'x509') {
+        authutil.asCluster(mongosConn, keyFile, () => {
+            assert.commandWorked(mongosConn.adminCommand({balancerStop: 1}));
+
+            // Use config.shards so we will not miss shards added outside of ShardingTest.
+            mongosConn.getDB('config').shards.find().forEach(shardDoc => {
+                let shardConn = getConn(shardDoc.host);
+
+                // Inherit connection options from mongos connection.
+                shardConn.fullOptions = Object.merge(this.s.fullOptions, {});
+
+                if (shardConn != null) {
+                    authutil.asCluster(shardConn, keyFile, () => {
+                        runCheck(mongosConn, shardConn, shardDoc._id);
+                    });
+                }
+            });
+        });
+    } else {
+        assert.commandWorked(mongosConn.adminCommand({balancerStop: 1}));
+
+        // Use config.shards so we will not miss shards added outside of ShardingTest.
+        mongosConn.getDB('config').shards.find().forEach(shardDoc => {
+            let shardConn = getConn(shardDoc.host);
+
+            if (shardConn != null) {
+                shardConn.host = shardDoc.host;
+                runCheck(mongosConn, shardConn, shardDoc._id);
+            }
+        });
+    }
+};
diff --git a/jstests/noPassthrough/optimize_sharded_sample_with_orphaned_docs.js b/jstests/noPassthrough/optimize_sharded_sample_with_orphaned_docs.js
index 22fe3b12276..84b6c33e29f 100644
--- a/jstests/noPassthrough/optimize_sharded_sample_with_orphaned_docs.js
+++ b/jstests/noPassthrough/optimize_sharded_sample_with_orphaned_docs.js
@@ -7,6 +7,9 @@
 (function() {
 "use strict";
 
+// Deliberately inserts orphans outside of migration.
+TestData.skipCheckOrphans = true;
+
 load('jstests/libs/analyze_plan.js');  // For aggPlanHasStage().
 
 // Set up a 2-shard cluster.
diff --git a/jstests/noPassthrough/stepdown_query.js b/jstests/noPassthrough/stepdown_query.js
index f52bd3c875c..4809e471f7a 100644
--- a/jstests/noPassthrough/stepdown_query.js
+++ b/jstests/noPassthrough/stepdown_query.js
@@ -6,6 +6,7 @@
 
 // Checking UUID consistency involves talking to a shard node, which in this test is shutdown
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/noPassthrough/unix_socket.js b/jstests/noPassthrough/unix_socket.js
index a6f0b1d6bf0..58c2976bbfa 100644
--- a/jstests/noPassthrough/unix_socket.js
+++ b/jstests/noPassthrough/unix_socket.js
@@ -18,6 +18,7 @@ if (_isWindows()) {
 
 // Checking index consistency involves reconnecting to the mongos.
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 // Do not fail if this test leaves unterminated processes because testSockOptions
 // is expected to throw before it calls stopMongod.
diff --git a/jstests/sharding/accurate_count_with_predicate.js b/jstests/sharding/accurate_count_with_predicate.js
index 4b5fcef4e2e..9c7d4c22593 100644
--- a/jstests/sharding/accurate_count_with_predicate.js
+++ b/jstests/sharding/accurate_count_with_predicate.js
@@ -9,6 +9,9 @@
 (function() {
 "use strict";
 
+// Deliberately inserts orphans outside of migration.
+TestData.skipCheckOrphans = true;
+
 const st = new ShardingTest({shards: 2});
 const shard0Coll = st.shard0.getCollection("test.slowcount");
 const num = 10;
diff --git a/jstests/sharding/aggregation_currentop.js b/jstests/sharding/aggregation_currentop.js
index f3b6ac3be5c..692366a8a37 100644
--- a/jstests/sharding/aggregation_currentop.js
+++ b/jstests/sharding/aggregation_currentop.js
@@ -18,6 +18,8 @@
 
 // Restarts cause issues with authentication for awaiting replication.
 TestData.skipAwaitingReplicationOnShardsBeforeCheckingUUIDs = true;
+// Restarts shard nodes with no keyFile.
+TestData.skipCheckOrphans = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/all_config_servers_blackholed_from_mongos.js b/jstests/sharding/all_config_servers_blackholed_from_mongos.js
index 1acb63f4408..b9010f5055e 100644
--- a/jstests/sharding/all_config_servers_blackholed_from_mongos.js
+++ b/jstests/sharding/all_config_servers_blackholed_from_mongos.js
@@ -8,6 +8,7 @@
 
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
index 4e9ae887bdc..b0bd0f59e8c 100644
--- a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
+++ b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js
@@ -6,6 +6,7 @@
 // in this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/auth_no_config_primary.js b/jstests/sharding/auth_no_config_primary.js
index 7e865eeb154..a7fad6186a4 100644
--- a/jstests/sharding/auth_no_config_primary.js
+++ b/jstests/sharding/auth_no_config_primary.js
@@ -11,6 +11,7 @@
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
 TestData.skipCheckDBHashes = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/batch_write_command_sharded.js b/jstests/sharding/batch_write_command_sharded.js
index f3f260c1d83..cd088de4dd1 100644
--- a/jstests/sharding/batch_write_command_sharded.js
+++ b/jstests/sharding/batch_write_command_sharded.js
@@ -8,6 +8,7 @@
 // no config server primary by the end of this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/change_stream_update_lookup_collation.js b/jstests/sharding/change_stream_update_lookup_collation.js
index 3cd5e0b593f..86e6cbb3ac7 100644
--- a/jstests/sharding/change_stream_update_lookup_collation.js
+++ b/jstests/sharding/change_stream_update_lookup_collation.js
@@ -14,6 +14,9 @@ if (!supportsMajorityReadConcern()) {
     return;
 }
 
+// Shard key index has collation, which is not compatible with $min/$max
+TestData.skipCheckOrphans = true;
+
 const st = new ShardingTest({
     shards: 2,
     config: 1,
diff --git a/jstests/sharding/collation_lookup.js b/jstests/sharding/collation_lookup.js
index ae3a15d59e6..dcb693265e0 100644
--- a/jstests/sharding/collation_lookup.js
+++ b/jstests/sharding/collation_lookup.js
@@ -13,6 +13,9 @@ load("jstests/aggregation/extras/utils.js");                     // for arrayEq
 load("jstests/noPassthrough/libs/server_parameter_helpers.js");  // For setParameterOnAllHosts.
 load("jstests/libs/discover_topology.js");                       // For findDataBearingNodes.
 
+// Shard key index has collation, which is not compatible with $min/$max
+TestData.skipCheckOrphans = true;
+
 function runTests(withDefaultCollationColl, withoutDefaultCollationColl, collation) {
     // Test that the $lookup stage respects the inherited collation.
     let res = withDefaultCollationColl
diff --git a/jstests/sharding/collation_targeting.js b/jstests/sharding/collation_targeting.js
index 5485a9cc428..7b60eafc68d 100644
--- a/jstests/sharding/collation_targeting.js
+++ b/jstests/sharding/collation_targeting.js
@@ -3,6 +3,9 @@
 (function() {
 "use strict";
 
+// Shard key index has collation, which is not compatible with $min/$max
+TestData.skipCheckOrphans = true;
+
 const caseInsensitive = {
     locale: "en_US",
     strength: 2
diff --git a/jstests/sharding/collation_targeting_inherited.js b/jstests/sharding/collation_targeting_inherited.js
index ab3a94ef6d5..c86198adbcb 100644
--- a/jstests/sharding/collation_targeting_inherited.js
+++ b/jstests/sharding/collation_targeting_inherited.js
@@ -3,6 +3,9 @@
 (function() {
 "use strict";
 
+// Shard key index has collation, which is not compatible with $min/$max
+TestData.skipCheckOrphans = true;
+
 const caseInsensitive = {
     locale: "en_US",
     strength: 2
diff --git a/jstests/sharding/config_rs_no_primary.js b/jstests/sharding/config_rs_no_primary.js
index e4a90569c87..91ce74de45d 100644
--- a/jstests/sharding/config_rs_no_primary.js
+++ b/jstests/sharding/config_rs_no_primary.js
@@ -5,6 +5,7 @@
 // no config server primary by the end of this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/count_config_servers.js b/jstests/sharding/count_config_servers.js
index 4d358191c59..ded75607cd0 100644
--- a/jstests/sharding/count_config_servers.js
+++ b/jstests/sharding/count_config_servers.js
@@ -7,6 +7,7 @@
 // shuts down 2 out of the 3 config servers. Therefore, we cannot do the check on this test.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/count_slaveok.js b/jstests/sharding/count_slaveok.js
index a0357de1f81..aa8bd2237af 100644
--- a/jstests/sharding/count_slaveok.js
+++ b/jstests/sharding/count_slaveok.js
@@ -5,6 +5,7 @@
 
 // Checking UUID consistency involves talking to a shard node, which in this test is shutdown
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/cursor_valid_after_shard_stepdown.js b/jstests/sharding/cursor_valid_after_shard_stepdown.js
index 52452809915..c96ce21ab3d 100644
--- a/jstests/sharding/cursor_valid_after_shard_stepdown.js
+++ b/jstests/sharding/cursor_valid_after_shard_stepdown.js
@@ -2,6 +2,7 @@
 // Checking UUID consistency involves talking to the shard primary, but by the end of this test, the
 // shard does not have a primary.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/index_commands_during_initial_split.js b/jstests/sharding/index_commands_during_initial_split.js
index 7b5ab80b728..27ba9c5fe39 100644
--- a/jstests/sharding/index_commands_during_initial_split.js
+++ b/jstests/sharding/index_commands_during_initial_split.js
@@ -10,6 +10,9 @@ load("jstests/libs/fail_point_util.js");
 load("jstests/libs/parallelTester.js");
 load("jstests/sharding/libs/sharded_index_util.js");
 
+// Test intentionally inserts orphans outside of migration.
+TestData.skipCheckOrphans = true;
+
 /*
  * Shards the given collection.
  */
diff --git a/jstests/sharding/index_commands_shard_targeting.js b/jstests/sharding/index_commands_shard_targeting.js
index 38012c7baf4..e3b3f8d6a40 100644
--- a/jstests/sharding/index_commands_shard_targeting.js
+++ b/jstests/sharding/index_commands_shard_targeting.js
@@ -12,6 +12,9 @@ load("jstests/libs/fail_point_util.js");
 load("jstests/sharding/libs/sharded_index_util.js");
 load("jstests/sharding/libs/shard_versioning_util.js");
 
+// Test deliberately inserts orphans outside of migration.
+TestData.skipCheckOrphans = true;
+
 /*
  * Runs the command after performing chunk operations to make the primary shard (shard0) not own
  * any chunks for the collection, and the subset of non-primary shards (shard1 and shard2) that
diff --git a/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js b/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js
index 6a350b90693..e7508fc58c1 100644
--- a/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js
+++ b/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js
@@ -9,6 +9,9 @@ load('jstests/libs/chunk_manipulation_util.js');
 load("jstests/libs/parallelTester.js");
 load("jstests/sharding/libs/sharded_index_util.js");
 
+// Test deliberately inserts orphans outside of migration.
+TestData.skipCheckOrphans = true;
+
 /*
  * Runs moveChunk on the host to move the chunk to the given shard.
  */
diff --git a/jstests/sharding/lagged_config_secondary.js b/jstests/sharding/lagged_config_secondary.js
index 06b186c8666..5cc0504deca 100644
--- a/jstests/sharding/lagged_config_secondary.js
+++ b/jstests/sharding/lagged_config_secondary.js
@@ -7,6 +7,7 @@
 // server, but this test is designed to make mongos time out when reading from the config server.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 var st =
diff --git a/jstests/sharding/localhostAuthBypass.js b/jstests/sharding/localhostAuthBypass.js
index 31103df7fe4..eb9f3f771c8 100644
--- a/jstests/sharding/localhostAuthBypass.js
+++ b/jstests/sharding/localhostAuthBypass.js
@@ -8,6 +8,7 @@
 // which stops the mongoses before calling ShardingTest.stop.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/migration_fails_if_exists_in_rangedeletions.js b/jstests/sharding/migration_fails_if_exists_in_rangedeletions.js
index ecad310035e..101fc0bf0f8 100644
--- a/jstests/sharding/migration_fails_if_exists_in_rangedeletions.js
+++ b/jstests/sharding/migration_fails_if_exists_in_rangedeletions.js
@@ -8,6 +8,9 @@
 
 load("jstests/libs/uuid_util.js");
 
+// TODO: remove after SERVER-45338 is fixed.
+TestData.skipCheckOrphans = true;
+
 const dbName = "test";
 const collName = "foo";
 const ns = dbName + "." + collName;
diff --git a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
index dde1f029ad4..30b81200c38 100644
--- a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
+++ b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js
@@ -15,6 +15,7 @@
 // test, one shard does not have a primary.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 // Replica set nodes started with --shardsvr do not enable key generation until they are added to a
 // sharded cluster and reject commands with gossiped clusterTime from users without the
diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js
index b58774eb510..34d68c45f6e 100644
--- a/jstests/sharding/mongos_rs_shard_failure_tolerance.js
+++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js
@@ -15,6 +15,7 @@
 // test, one shard does not have a primary.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/mr_and_agg_versioning.js b/jstests/sharding/mr_and_agg_versioning.js
index 552c43aadc3..e53eab11e2e 100644
--- a/jstests/sharding/mr_and_agg_versioning.js
+++ b/jstests/sharding/mr_and_agg_versioning.js
@@ -2,6 +2,9 @@
 (function() {
 "use strict";
 
+// Test delibarately inserts orphaned data outside of migrations.
+TestData.skipCheckOrphans = true;
+
 const st = new ShardingTest({shards: 2, mongos: 3});
 
 const dbName = jsTest.name();
diff --git a/jstests/sharding/prefix_shard_key.js b/jstests/sharding/prefix_shard_key.js
index b10c8fb79f4..2c33ff77fd5 100644
--- a/jstests/sharding/prefix_shard_key.js
+++ b/jstests/sharding/prefix_shard_key.js
@@ -10,6 +10,9 @@
 (function() {
 'use strict';
 
+// Shard key index does not exactly match shard key, so it is not compatible with $min/$max.
+TestData.skipCheckOrphans = true;
+
 var checkDocCounts = function(expectedShardCount) {
     for (var shardName in expectedShardCount) {
         var shard = (shardName == s.shard0.shardName ? s.shard0 : s.shard1);
diff --git a/jstests/sharding/primary_config_server_blackholed_from_mongos.js b/jstests/sharding/primary_config_server_blackholed_from_mongos.js
index 0424a307621..72985bf36f3 100644
--- a/jstests/sharding/primary_config_server_blackholed_from_mongos.js
+++ b/jstests/sharding/primary_config_server_blackholed_from_mongos.js
@@ -6,6 +6,7 @@
 // Checking index consistency involves talking to the primary config server which is blackholed from
 // the mongos in this test.
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 var st = new ShardingTest({shards: 2, mongos: 1, useBridge: true});
 
diff --git a/jstests/sharding/read_pref.js b/jstests/sharding/read_pref.js
index 454cc2fb6dd..b3e4f835ea7 100644
--- a/jstests/sharding/read_pref.js
+++ b/jstests/sharding/read_pref.js
@@ -5,6 +5,7 @@
 
 // Checking UUID consistency involves talking to a shard node, which in this test is shutdown
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 'use strict';
diff --git a/jstests/sharding/resubmit_rangedeletions_on_stepup.js b/jstests/sharding/resubmit_rangedeletions_on_stepup.js
index 71a8c79c7a2..03f68ae7558 100644
--- a/jstests/sharding/resubmit_rangedeletions_on_stepup.js
+++ b/jstests/sharding/resubmit_rangedeletions_on_stepup.js
@@ -9,6 +9,8 @@
 load("jstests/libs/uuid_util.js");
 
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
+// Test deliberately keeps range deletion in pending state.
+TestData.skipCheckOrphans = true;
 
 const dbName = "test";
 const collName = "foo";
diff --git a/jstests/sharding/shard_insert_getlasterror_w2.js b/jstests/sharding/shard_insert_getlasterror_w2.js
index 38bb2db5fa3..a85126c1c04 100644
--- a/jstests/sharding/shard_insert_getlasterror_w2.js
+++ b/jstests/sharding/shard_insert_getlasterror_w2.js
@@ -5,6 +5,7 @@
 // stops 2/3 nodes of a replica set.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 (function() {
 "use strict";
diff --git a/jstests/sharding/unowned_doc_filtering.js b/jstests/sharding/unowned_doc_filtering.js
index e61e1247ee9..07c42453ca9 100644
--- a/jstests/sharding/unowned_doc_filtering.js
+++ b/jstests/sharding/unowned_doc_filtering.js
@@ -10,6 +10,9 @@
 // This test shuts down shards.
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 
+// Test deliberately inserts orphans.
+TestData.skipCheckOrphans = true;
+
 (function() {
 "use strict";
 
diff --git a/jstests/sharding/update_replace_id.js b/jstests/sharding/update_replace_id.js
index 8c6f8e29a99..1b5ab8f43e0 100644
--- a/jstests/sharding/update_replace_id.js
+++ b/jstests/sharding/update_replace_id.js
@@ -16,6 +16,9 @@
 (function() {
 load("jstests/libs/profiler.js");  // For profilerHas*OrThrow helper functions.
 
+// Test deliberately inserts orphans outside of migrations.
+TestData.skipCheckOrphans = true;
+
 const st = new ShardingTest({shards: 2, mongos: 1, config: 1, other: {enableBalancer: false}});
 
 const mongosDB = st.s0.getDB(jsTestName());
diff --git a/jstests/ssl/libs/ssl_helpers.js b/jstests/ssl/libs/ssl_helpers.js
index 65ba385b321..969a812c8ed 100644
--- a/jstests/ssl/libs/ssl_helpers.js
+++ b/jstests/ssl/libs/ssl_helpers.js
@@ -204,6 +204,13 @@ function mixedShardTest(options1, options2, shouldSucceed, disableResumableRange
                     'CN=client,OU=KernelUser,O=MongoDB,L=New York City,ST=New York,C=US';
                 st.s.getDB('$external')
                     .createUser({user: x509User, roles: [{role: '__system', db: 'admin'}]});
+
+                // Check orphan hook needs a privileged user to auth as.
+                // Works only for stand alone shards.
+                st._connections.forEach((shardConn) => {
+                    shardConn.getDB('$external')
+                        .createUser({user: x509User, roles: [{role: '__system', db: 'admin'}]});
+                });
             }
 
             st.stop();
diff --git a/jstests/ssl/sharding_with_x509.js b/jstests/ssl/sharding_with_x509.js
index df071ce3b97..7e0a0903296 100644
--- a/jstests/ssl/sharding_with_x509.js
+++ b/jstests/ssl/sharding_with_x509.js
@@ -82,5 +82,9 @@ if (st.configRS) {
 const x509User = 'CN=client,OU=KernelUser,O=MongoDB,L=New York City,ST=New York,C=US';
 st.s.getDB('$external').createUser({user: x509User, roles: [{role: '__system', db: 'admin'}]});
 
+// Orphan checks needs a privileged user to auth as.
+st.shard0.getDB('$external').createUser({user: x509User, roles: [{role: '__system', db: 'admin'}]});
+st.shard1.getDB('$external').createUser({user: x509User, roles: [{role: '__system', db: 'admin'}]});
+
 st.stop();
 })();
diff --git a/jstests/ssl/sni_name_advertisement.js b/jstests/ssl/sni_name_advertisement.js
index d945703b4aa..938f7ceb230 100644
--- a/jstests/ssl/sni_name_advertisement.js
+++ b/jstests/ssl/sni_name_advertisement.js
@@ -8,6 +8,7 @@ load('jstests/ssl/libs/ssl_helpers.js');
 
 // Checking index consistency involves reconnecting to the mongos.
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
 
 let path = "jstests/libs/";
 let pemKeyFile = path + "server.pem";
diff --git a/jstests/ssl/ssl_get_more.js b/jstests/ssl/ssl_get_more.js
index 37e4e2acb1d..f7cf987d660 100644
--- a/jstests/ssl/ssl_get_more.js
+++ b/jstests/ssl/ssl_get_more.js
@@ -62,5 +62,8 @@ if (st.configRS) {
 const x509User = 'CN=client,OU=KernelUser,O=MongoDB,L=New York City,ST=New York,C=US';
 st.s.getDB('$external').createUser({user: x509User, roles: [{role: '__system', db: 'admin'}]});
 
+// Orphan checks needs a privileged user to auth as.
+st.shard0.getDB('$external').createUser({user: x509User, roles: [{role: '__system', db: 'admin'}]});
+
 st.stop();
 }());
diff --git a/jstests/sslSpecial/mixed_mode_sharded_transition_nossl.js b/jstests/sslSpecial/mixed_mode_sharded_transition_nossl.js
index 03ee4e5fb09..59f2f90b4e5 100644
--- a/jstests/sslSpecial/mixed_mode_sharded_transition_nossl.js
+++ b/jstests/sslSpecial/mixed_mode_sharded_transition_nossl.js
@@ -6,7 +6,11 @@
  * test suite. This suite does not use ssl so it cannot test modes with ssl.
  */
 
+// Test setup randomly have auth/no auth setting on shards, which make hooks targetting shard
+// directly more complicated. Skip the hooks since this test doesn't really do migrations.
 TestData.skipCheckingIndexesConsistentAcrossCluster = true;
+TestData.skipCheckOrphans = true;
+
 load('jstests/ssl/libs/ssl_helpers.js');
 
 (function() {
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index 8f13e2fbc9f..42c5a6590ff 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -389,6 +389,7 @@ var ShardingTest = function(params) {
     this.stop = function(opts = {}) {
         this.checkUUIDsConsistentAcrossCluster();
         this.checkIndexesConsistentAcrossCluster();
+        this.checkOrphansAreDeleted();
 
         if (jsTestOptions().alwaysUseLogFiles) {
             if (opts.noCleanData === false) {
@@ -1750,3 +1751,7 @@ ShardingTest.prototype.checkUUIDsConsistentAcrossCluster = function() {};
 
 // Stub for a hook to check that indexes are consistent across shards.
 ShardingTest.prototype.checkIndexesConsistentAcrossCluster = function() {};
+
+ShardingTest.prototype.checkOrphansAreDeleted = function() {
+    print("Unhooked function");
+};
diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js
index 23d8a0bc627..cdd4fa11c9e 100644
--- a/src/mongo/shell/utils.js
+++ b/src/mongo/shell/utils.js
@@ -353,6 +353,7 @@ jsTestOptions = function() {
             // in dbpath; additionally, prevent the dbpath from being cleared after a node
             // is shut down.
             alwaysUseLogFiles: TestData.alwaysUseLogFiles || false,
+            skipCheckOrphans: TestData.skipCheckOrphans || false,
         });
     }
     return _jsTestOptions;