diff options
85 files changed, 398 insertions, 11 deletions
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml index 082fe9dd0d2..3689a0ec9b6 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml @@ -116,6 +116,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml index dd12bb8d457..1d550cc3495 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml @@ -124,6 +124,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_clusterwide_ops_add_remove_shards.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_clusterwide_ops_add_remove_shards.yml index 7fdb5318933..b45c1329060 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_clusterwide_ops_add_remove_shards.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_clusterwide_ops_add_remove_shards.yml @@ -23,6 +23,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_initial_sync.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_initial_sync.yml index fafd5aca0d2..03a212b98f7 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_initial_sync.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_initial_sync.yml @@ -229,6 +229,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: CheckClusterIndexConsistency - class: ValidateCollections # Validation can interfere with other operations, so this goes last. shell_options: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml index ff84cf8706f..96dac71e666 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml @@ -203,6 +203,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml index e08a742a485..068307ca876 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml @@ -174,6 +174,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml index 37ae21f5cb8..a272c4fa07b 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml @@ -171,6 +171,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml index 8d5eeb89cd8..d81e8d6c9ef 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml @@ -164,6 +164,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml index 3de242f746d..0be3040cc69 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml @@ -254,6 +254,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml index 607916ec0ec..fbf73c0febb 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml @@ -254,6 +254,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml index 75ec530b3a9..e576c32de5e 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml @@ -166,6 +166,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml index 3eb119b82f5..f75311dda8e 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml @@ -226,6 +226,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml index 64e01124b23..4417cdf28e2 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml @@ -126,6 +126,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml index e1e5b952a68..70cf404cc5d 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml @@ -118,6 +118,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml index 5d31436d074..3bff1625e86 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml @@ -203,6 +203,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml index 945313ff40e..4581401cf1d 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml @@ -193,6 +193,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml index 4c0792175ac..99cce89d9c2 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml @@ -196,6 +196,7 @@ executor: - class: CheckReplDBHash - class: CheckOrphansDeleted - class: CheckRoutingTableConsistency + - class: CheckShardFilteringMetadata - class: ValidateCollections # Validation can interfere with other operations, so this goes last. - class: CleanupConcurrencyWorkloads fixture: diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index a87276322d8..696a8d2e922 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -256,6 +256,10 @@ def mongo_shell_program(logger, executable=None, connection_string=None, filenam # semantically correct before shutting down a ShardingTest. eval_sb.append("load('jstests/libs/override_methods/check_routing_table_consistency.js');") + # Load a callback to check that all shards have correct filtering information before shutting + # down a ShardingTest. + eval_sb.append("load('jstests/libs/override_methods/check_shard_filtering_metadata.js');") + if config.FUZZ_MONGOD_CONFIGS is not None and config.FUZZ_MONGOD_CONFIGS is not False: # Prevent commands from running with the config fuzzer. eval_sb.append( diff --git a/buildscripts/resmokelib/testing/hooks/shard_filtering_metadata.py b/buildscripts/resmokelib/testing/hooks/shard_filtering_metadata.py new file mode 100644 index 00000000000..5c197b69ae5 --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/shard_filtering_metadata.py @@ -0,0 +1,22 @@ +"""Test hook for verifying the correctness of shard filtering metadata on shards of a sharded cluster.""" + +import os.path + +from buildscripts.resmokelib.testing.fixtures import shardedcluster +from buildscripts.resmokelib.testing.hooks import jsfile + + +class CheckShardFilteringMetadata(jsfile.DataConsistencyHook): + IS_BACKGROUND = False + + def __init__(self, hook_logger, fixture, shell_options=None): + """Initialize CheckShardFilteringMetadata.""" + + if not isinstance(fixture, shardedcluster.ShardedClusterFixture): + raise ValueError(f"'fixture' must be an instance of ShardedClusterFixture, but got" + f" {fixture.__class__.__name__}") + + description = "Inspect filtering metadata on shards" + js_filename = os.path.join("jstests", "hooks", "run_check_shard_filtering_metadata.js") + super().__init__(hook_logger, fixture, js_filename, description, + shell_options=shell_options) diff --git a/jstests/auth/access_control_with_unreachable_configs.js b/jstests/auth/access_control_with_unreachable_configs.js index 153b569bf62..5ff2e143b15 100644 --- a/jstests/auth/access_control_with_unreachable_configs.js +++ b/jstests/auth/access_control_with_unreachable_configs.js @@ -7,6 +7,7 @@ // The config servers are not reachable at shutdown. TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; var st = new ShardingTest({ shards: 1, diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js index d3f69ed3dc6..7a019f71333 100644 --- a/jstests/auth/lib/commands_lib.js +++ b/jstests/auth/lib/commands_lib.js @@ -91,6 +91,12 @@ one argument, the connection object. */ +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + import { isShardMergeEnabled, makeMigrationCertificatesForTest diff --git a/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js b/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js index 9047c2cddef..4c0ff0c8518 100644 --- a/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js +++ b/jstests/auth/upgrade_noauth_to_keyfile_with_sharding.js @@ -10,6 +10,7 @@ load('jstests/ssl/libs/ssl_helpers.js'); // IndexConsistencyCheck requires auth which ttA/ttA fails at. TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; // Disable auth explicitly var noAuthOptions = {noauth: ''}; diff --git a/jstests/change_streams/refine_collection_shard_key_event.js b/jstests/change_streams/refine_collection_shard_key_event.js index 1b06e06fc0b..5acde86c27b 100644 --- a/jstests/change_streams/refine_collection_shard_key_event.js +++ b/jstests/change_streams/refine_collection_shard_key_event.js @@ -12,6 +12,9 @@ * ] */ +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { "use strict"; diff --git a/jstests/hooks/run_check_shard_filtering_metadata.js b/jstests/hooks/run_check_shard_filtering_metadata.js new file mode 100644 index 00000000000..a78160ae51c --- /dev/null +++ b/jstests/hooks/run_check_shard_filtering_metadata.js @@ -0,0 +1,38 @@ +'use strict'; + +/** + * Checks the filtering metadata on the shards matches the one in the configsvr. + */ +(function() { +load('jstests/libs/check_shard_filtering_metadata_helpers.js'); +load('jstests/libs/discover_topology.js'); + +assert.neq(typeof db, 'undefined', 'No `db` object, is the shell connected to a server?'); + +const conn = db.getMongo(); +const topology = DiscoverTopology.findConnectedNodes(conn); + +if (topology.type !== Topology.kShardedCluster) { + throw new Error( + 'Filtering metadata check can only be run against a sharded cluster, but got: ' + + tojson(topology)); +} + +for (let shardName of Object.keys(topology.shards)) { + const shard = topology.shards[shardName]; + + if (shard.type !== Topology.kReplicaSet) { + throw new Error('Unexpected topology: ' + tojson(topology)); + } + + // Await replication to ensure that metadata on secondary nodes is up-to-date. + new ReplSetTest(shard.nodes[0]).awaitSecondaryNodes(); + + // Skipping checking sharded collection metadata because any workload on the suite could perform + // an operation that is known to leave incorrect metadata (such as refineCollectionShardKey). + shard.nodes.forEach(node => { + CheckShardFilteringMetadataHelpers.run( + db.getMongo(), new Mongo(node), shardName, true /* skipCheckShardedCollections */); + }); +} +})(); diff --git a/jstests/libs/check_shard_filtering_metadata_helpers.js b/jstests/libs/check_shard_filtering_metadata_helpers.js new file mode 100644 index 00000000000..ca096e3c5fd --- /dev/null +++ b/jstests/libs/check_shard_filtering_metadata_helpers.js @@ -0,0 +1,111 @@ +'use strict'; + +var CheckShardFilteringMetadataHelpers = (function() { + function run(mongosConn, nodeConn, shardId, skipCheckShardedCollections = false) { + function checkDatabase(configDatabasesEntry) { + const dbName = configDatabasesEntry._id; + print(`CheckShardFilteringMetadata: checking database '${dbName}' on node '${ + nodeConn.host}' of shard '${shardId}'`); + + const nodeMetadata = + assert.commandWorked(nodeConn.adminCommand({getDatabaseVersion: dbName})); + + if (nodeMetadata.dbVersion.timestamp === undefined) { + // Shards are allowed to not know the dbVersion. + return; + } + + assert.eq(nodeMetadata.dbVersion.uuid, + configDatabasesEntry.version.uuid, + `Unexpected dbVersion.uuid for db '${dbName}' on node '${nodeConn.host}'`); + assert.eq(timestampCmp(nodeMetadata.dbVersion.timestamp, + configDatabasesEntry.version.timestamp), + 0, + `Unexpected dbVersion timestamp for db '${dbName}' on node '${ + nodeConn.host}'. Found '${ + tojson(nodeMetadata.dbVersion.timestamp)}'; expected '${ + tojson(configDatabasesEntry.version.timestamp)}'`); + assert.eq(nodeMetadata.dbVersion.lastMod, + configDatabasesEntry.version.lastMod, + `Unexpected dbVersion lastMod for db '${dbName}' on node '${nodeConn.host}'`); + + print(`CheckShardFilteringMetadata: Database '${dbName}' on '${nodeConn.host}' OK`); + } + + function checkShardedCollection(coll, nodeShardingState) { + const ns = coll._id; + print(`CheckShardFilteringMetadata: checking collection '${ns} ' on node '${ + nodeConn.host}' of shard '${shardId}'`); + + const configDB = mongosConn.getDB('config'); + + const highestChunkOnShard = configDB.chunks.find({uuid: coll.uuid, shard: shardId}) + .sort({lastmod: -1}) + .limit(1) + .toArray()[0]; + + const expectedShardVersion = + highestChunkOnShard ? highestChunkOnShard.lastmod : Timestamp(0, 0); + const expectedTimestamp = coll.timestamp; + + const collectionMetadataOnNode = nodeShardingState.versions[ns]; + if (collectionMetadataOnNode === undefined) { + // Shards are not authoritative. It is okay that they don't know their filtering + // info. + return; + } + + if (collectionMetadataOnNode.timestamp === undefined) { + // Versions earlier than v6.3 did not report the timestamp on shardingState command + // (SERVER-70790). This early exit can be removed after v6.0 is no longer tested in + // multiversion suites. + return; + } + + if (timestampCmp(collectionMetadataOnNode.timestamp, Timestamp(0, 0)) === 0) { + // The metadata reflects an unsharded collection. It is okay for a node to have this + // stale metadata, as long as the node knows the correct dbVersion. + return; + } + + // If the node knows its filtering info, then assert that it is correct. + assert.eq(timestampCmp(collectionMetadataOnNode.timestamp, expectedTimestamp), + 0, + `Unexpected timestamp for ns '${ns}' on node '${nodeConn.host}'. Found '${ + tojson(collectionMetadataOnNode.timestamp)}', expected '${ + tojson(expectedTimestamp)}'`); + // Only check the major version because some operations (such as resharding or + // setAllowMigrations) bump the minor version without the shards knowing. This does not + // affect placement, so it is okay. + assert.eq(collectionMetadataOnNode.placementVersion.t, + expectedShardVersion.t, + `Unexpected shardVersion for ns '${ns}' on node '${nodeConn.host}'`); + + print(`CheckShardFilteringMetadata: ns '${ns}' on '${nodeConn.host}' OK`); + } + + const configDB = mongosConn.getDB('config'); + + // Check shards know correct database versions. + // Note: We can only check the dbVersion on the primary shards because non-primary shards + // can have stale dbVersions cached (for versions on which they were not primary.). TODO: + // Once shards became authoritative for their dbVersions, we can check all shards. + configDB.databases.find({primary: shardId}).forEach(configDatabasesEntry => { + checkDatabase(configDatabasesEntry); + }); + + // Check that shards have correct filtering metadata for sharded collections. + if (!skipCheckShardedCollections) { + const nodeShardingState = nodeConn.adminCommand({shardingState: 1}); + configDB.collections.find().forEach(coll => { + checkShardedCollection(coll, nodeShardingState); + }); + } + + print("CheckShardFilteringMetadata: finished"); + } + + return { + run: run, + }; +})(); diff --git a/jstests/libs/override_methods/check_shard_filtering_metadata.js b/jstests/libs/override_methods/check_shard_filtering_metadata.js new file mode 100644 index 00000000000..d02abf9516f --- /dev/null +++ b/jstests/libs/override_methods/check_shard_filtering_metadata.js @@ -0,0 +1,74 @@ +'use strict'; + +load('jstests/libs/check_shard_filtering_metadata_helpers.js'); + +ShardingTest.prototype.checkShardFilteringMetadata = function() { + if (jsTest.options().skipCheckShardFilteringMetadata) { + jsTest.log("Skipping shard filtering metadata check"); + return; + } + + // Use a new connection so we don't have to worry about existing users logged in to the + // connection. + let mongosConn = new Mongo(this.s.host); + mongosConn.fullOptions = Object.merge(this.s.fullOptions, {}); + + const keyFile = this.keyFile; + const useAuth = keyFile || mongosConn.fullOptions.clusterAuthMode === 'x509'; + + let getConn = function(connStr) { + try { + return new Mongo(connStr); + } catch (exp) { + jsTest.log('CheckShardFilteringMetadata: Unable to connect to ' + connStr); + return null; + } + }; + + function executeAuthenticatedIfNeeded(conn, fn) { + if (useAuth) { + return authutil.asCluster(conn, keyFile, fn); + } else { + return fn(); + } + } + + executeAuthenticatedIfNeeded(mongosConn, () => { + // For each shard + mongosConn.getDB('config').shards.find().forEach(shardDoc => { + const shardName = shardDoc._id; + const shardConn = getConn(shardDoc.host); + if (shardConn === null) { + jsTest.log('CheckShardFilteringMetadata: Skipping check on shard' + shardDoc.host); + return; + } + shardConn.fullOptions = Object.merge(this.s.fullOptions, {}); + + // Await replication to ensure that metadata on secondary nodes is up-to-date. + this.awaitReplicationOnShards(); + + // Get nodes for this shard + shardConn.setSecondaryOk(); + const shardNodesHosts = executeAuthenticatedIfNeeded(shardConn, () => { + return shardConn.adminCommand({replSetGetConfig: 1}) + .config.members.filter(member => member.arbiterOnly === false) + .map(member => member.host); + }); + + // Run check on each node + shardNodesHosts.forEach(host => { + const shardNodeConn = getConn(host); + if (shardNodeConn === null) { + jsTest.log('CheckShardFilteringMetadata: Skipping check on node' + + shardDoc.host); + return; + } + shardNodeConn.fullOptions = Object.merge(this.s.fullOptions, {}); + + executeAuthenticatedIfNeeded(shardNodeConn, () => { + CheckShardFilteringMetadataHelpers.run(mongosConn, shardNodeConn, shardName); + }); + }); + }); + }); +}; diff --git a/jstests/noPassthrough/unix_socket.js b/jstests/noPassthrough/unix_socket.js index 0b24d0e9739..b18b78a2a96 100644 --- a/jstests/noPassthrough/unix_socket.js +++ b/jstests/noPassthrough/unix_socket.js @@ -23,6 +23,7 @@ if (_isWindows()) { TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; TestData.skipCheckRoutingTableConsistency = true; +TestData.skipCheckShardFilteringMetadata = true; // Do not fail if this test leaves unterminated processes because testSockOptions // is expected to throw before it calls stopMongod. diff --git a/jstests/sharding/all_config_servers_blackholed_from_mongos.js b/jstests/sharding/all_config_servers_blackholed_from_mongos.js index 689bbe5b079..f5bf02d0886 100644 --- a/jstests/sharding/all_config_servers_blackholed_from_mongos.js +++ b/jstests/sharding/all_config_servers_blackholed_from_mongos.js @@ -10,6 +10,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; TestData.skipCheckRoutingTableConsistency = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js index 85340c5436f..e60c35ee15b 100644 --- a/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js +++ b/jstests/sharding/all_shard_and_config_hosts_brought_down_one_by_one.js @@ -7,6 +7,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/allow_partial_results.js b/jstests/sharding/allow_partial_results.js index 55d083537db..d8c8d8aeb82 100644 --- a/jstests/sharding/allow_partial_results.js +++ b/jstests/sharding/allow_partial_results.js @@ -6,6 +6,7 @@ // This test shuts down a shard. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/allow_partial_results_nshards.js b/jstests/sharding/allow_partial_results_nshards.js index 769b2cd30eb..74b1b72c91b 100644 --- a/jstests/sharding/allow_partial_results_nshards.js +++ b/jstests/sharding/allow_partial_results_nshards.js @@ -19,10 +19,10 @@ TestData.disableImplicitSessions = true; // check would be lost. TestData.skipGossipingClusterTime = true; -// Don't check for UUID and index consistency across the cluster at the end, since the test shuts -// down a shard. +// Skip the following checks across the cluster at the end, since the test shuts down a shard. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; // Set up a 2-shard single-node replicaset cluster. const st = new ShardingTest({name: jsTestName(), shards: 2, rs: {nodes: 1}}); diff --git a/jstests/sharding/auth_no_config_primary.js b/jstests/sharding/auth_no_config_primary.js index a7fad6186a4..f421317f33f 100644 --- a/jstests/sharding/auth_no_config_primary.js +++ b/jstests/sharding/auth_no_config_primary.js @@ -12,6 +12,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckDBHashes = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/awaitable_hello_primary_failures.js b/jstests/sharding/awaitable_hello_primary_failures.js index b6a387920c3..09ff7aa8f5a 100644 --- a/jstests/sharding/awaitable_hello_primary_failures.js +++ b/jstests/sharding/awaitable_hello_primary_failures.js @@ -8,6 +8,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/balancer_defragmentation_merge_chunks.js b/jstests/sharding/balancer_defragmentation_merge_chunks.js index bb53e575997..653067f7458 100644 --- a/jstests/sharding/balancer_defragmentation_merge_chunks.js +++ b/jstests/sharding/balancer_defragmentation_merge_chunks.js @@ -9,6 +9,9 @@ * ] */ +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/batch_write_command_sharded.js b/jstests/sharding/batch_write_command_sharded.js index 2cdbf151f96..d2b0a2ddfd9 100644 --- a/jstests/sharding/batch_write_command_sharded.js +++ b/jstests/sharding/batch_write_command_sharded.js @@ -9,6 +9,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/change_stream_error_label.js b/jstests/sharding/change_stream_error_label.js index 7c72b3475db..d087152ca54 100644 --- a/jstests/sharding/change_stream_error_label.js +++ b/jstests/sharding/change_stream_error_label.js @@ -12,6 +12,7 @@ // Skip cross-cluster consistency checks, since this test prematurely shuts down a shard. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; // Create a two-shard cluster so that we can stop one shard to test connection interruptions. const st = new ShardingTest({ diff --git a/jstests/sharding/change_stream_resume_shard_key_change.js b/jstests/sharding/change_stream_resume_shard_key_change.js index c0aaf7dc889..295cc18384c 100644 --- a/jstests/sharding/change_stream_resume_shard_key_change.js +++ b/jstests/sharding/change_stream_resume_shard_key_change.js @@ -6,6 +6,9 @@ // requires_fcv_53, // ] +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { "use strict"; diff --git a/jstests/sharding/clear_jumbo.js b/jstests/sharding/clear_jumbo.js index 30190abac8d..5b2eb77b2d3 100644 --- a/jstests/sharding/clear_jumbo.js +++ b/jstests/sharding/clear_jumbo.js @@ -2,6 +2,9 @@ // hence the balancer is not triggered and the chunk is not marked as jumbo // @tags: [requires_fcv_61] +// Cannot run the filtering metadata check on tests that run clearJumboFlag. +TestData.skipCheckShardFilteringMetadata = true; + (function() { "use strict"; diff --git a/jstests/sharding/collection_uuid_refine_collection_shard_key.js b/jstests/sharding/collection_uuid_refine_collection_shard_key.js index 040942c98e9..615fa69fa85 100644 --- a/jstests/sharding/collection_uuid_refine_collection_shard_key.js +++ b/jstests/sharding/collection_uuid_refine_collection_shard_key.js @@ -5,6 +5,10 @@ * requires_fcv_60, * ] */ + +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/config_rs_no_primary.js b/jstests/sharding/config_rs_no_primary.js index 8bcf7e54cd4..135249ee3e1 100644 --- a/jstests/sharding/config_rs_no_primary.js +++ b/jstests/sharding/config_rs_no_primary.js @@ -6,6 +6,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/count_config_servers.js b/jstests/sharding/count_config_servers.js index 0904a873e52..d73a4198e7e 100644 --- a/jstests/sharding/count_config_servers.js +++ b/jstests/sharding/count_config_servers.js @@ -8,6 +8,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/count_secondaryok.js b/jstests/sharding/count_secondaryok.js index c5fcedf941b..64181787f1d 100644 --- a/jstests/sharding/count_secondaryok.js +++ b/jstests/sharding/count_secondaryok.js @@ -8,6 +8,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/db_and_coll_timestamps_test.js b/jstests/sharding/db_and_coll_timestamps_test.js index 82c65281e02..1cc42374552 100644 --- a/jstests/sharding/db_and_coll_timestamps_test.js +++ b/jstests/sharding/db_and_coll_timestamps_test.js @@ -8,6 +8,9 @@ * collection, dropping and creating a collection, or refining the sharding key. */ +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/ddl_ops_reported_on_current_op_command.js b/jstests/sharding/ddl_ops_reported_on_current_op_command.js index 7327df50ccd..38f5c79f61c 100644 --- a/jstests/sharding/ddl_ops_reported_on_current_op_command.js +++ b/jstests/sharding/ddl_ops_reported_on_current_op_command.js @@ -2,6 +2,10 @@ * Checks that DDL command that use step-down resilient coordinators are shown when calling the * currentOp command. */ + +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/exhaust_hello_topology_changes.js b/jstests/sharding/exhaust_hello_topology_changes.js index a2288ab3f10..b3ba45d8861 100644 --- a/jstests/sharding/exhaust_hello_topology_changes.js +++ b/jstests/sharding/exhaust_hello_topology_changes.js @@ -14,6 +14,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/extract_shard_key_values.js b/jstests/sharding/extract_shard_key_values.js index c0532ddcfe7..15f9c936658 100644 --- a/jstests/sharding/extract_shard_key_values.js +++ b/jstests/sharding/extract_shard_key_values.js @@ -7,6 +7,9 @@ // uses_transactions, // ] +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/health_monitor/config_server_health_observer_crash.js b/jstests/sharding/health_monitor/config_server_health_observer_crash.js index be4f05c68cd..51d20521b2f 100644 --- a/jstests/sharding/health_monitor/config_server_health_observer_crash.js +++ b/jstests/sharding/health_monitor/config_server_health_observer_crash.js @@ -18,6 +18,7 @@ TestData.failIfUnterminatedProcesses = false; // the mongos in this test. TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; const mongosParams = { setParameter: { diff --git a/jstests/sharding/index_and_collection_option_propagation.js b/jstests/sharding/index_and_collection_option_propagation.js index d58d5dba446..0b37e7ecd82 100644 --- a/jstests/sharding/index_and_collection_option_propagation.js +++ b/jstests/sharding/index_and_collection_option_propagation.js @@ -11,6 +11,7 @@ // cannot be performed on that node, which causes the consistency checker to fail. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { // Helper function that runs listIndexes against shards to check for the existence of an index. diff --git a/jstests/sharding/index_commands_shard_targeting.js b/jstests/sharding/index_commands_shard_targeting.js index b45dd2dccce..7568b1e6216 100644 --- a/jstests/sharding/index_commands_shard_targeting.js +++ b/jstests/sharding/index_commands_shard_targeting.js @@ -15,6 +15,9 @@ load("jstests/libs/parallelTester.js"); // For Thread. // Test deliberately inserts orphans outside of migration. TestData.skipCheckOrphans = true; +// This test connects directly to shards and creates collections. +TestData.skipCheckShardFilteringMetadata = true; + /* * Runs the command after performing chunk operations to make the primary shard (shard0) not own * any chunks for the collection, and the subset of non-primary shards (shard1 and shard2) that diff --git a/jstests/sharding/invalidate_plan_cache_entries_when_collection_generation_changes.js b/jstests/sharding/invalidate_plan_cache_entries_when_collection_generation_changes.js index 8d1188d6835..6aafb4c44ae 100644 --- a/jstests/sharding/invalidate_plan_cache_entries_when_collection_generation_changes.js +++ b/jstests/sharding/invalidate_plan_cache_entries_when_collection_generation_changes.js @@ -2,6 +2,10 @@ * Tests that plan cache entries are deleted after shard key refining, resharding and renaming * operations. */ + +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; load("jstests/libs/sbe_util.js"); diff --git a/jstests/sharding/lagged_config_secondary.js b/jstests/sharding/lagged_config_secondary.js index db0ba1f9e9c..b17f32e1fb2 100644 --- a/jstests/sharding/lagged_config_secondary.js +++ b/jstests/sharding/lagged_config_secondary.js @@ -11,6 +11,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; TestData.skipCheckRoutingTableConsistency = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { diff --git a/jstests/sharding/libs/mongos_api_params_util.js b/jstests/sharding/libs/mongos_api_params_util.js index 25766f22373..24b249a8fab 100644 --- a/jstests/sharding/libs/mongos_api_params_util.js +++ b/jstests/sharding/libs/mongos_api_params_util.js @@ -17,6 +17,9 @@ let MongosAPIParametersUtil = (function() { // therefore preventing orphans from being cleaned up. TestData.skipCheckOrphans = true; + // Cannot run the filtering metadata check on tests that run refineCollectionShardKey. + TestData.skipCheckShardFilteringMetadata = true; + function validateTestCase(testCase) { assert(testCase.skip || testCase.run, "must specify exactly one of 'skip' or 'run' for test case " + tojson(testCase)); diff --git a/jstests/sharding/linearizable_read_concern.js b/jstests/sharding/linearizable_read_concern.js index a5720681aeb..bfd1563fd3d 100644 --- a/jstests/sharding/linearizable_read_concern.js +++ b/jstests/sharding/linearizable_read_concern.js @@ -27,10 +27,10 @@ load("jstests/libs/write_concern_util.js"); (function() { "use strict"; -// Skip db hash check and shard replication since this test leaves a replica set shard -// partitioned. +// Skip the following checks since this test leaves a replica set shard partitioned. TestData.skipCheckDBHashes = true; TestData.skipAwaitingReplicationOnShardsBeforeCheckingUUIDs = true; +TestData.skipCheckShardFilteringMetadata = true; var testName = "linearizable_read_concern"; diff --git a/jstests/sharding/localhostAuthBypass.js b/jstests/sharding/localhostAuthBypass.js index 449c928e467..3576e68c517 100644 --- a/jstests/sharding/localhostAuthBypass.js +++ b/jstests/sharding/localhostAuthBypass.js @@ -9,6 +9,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/migration_coordinator_shutdown_in_critical_section.js b/jstests/sharding/migration_coordinator_shutdown_in_critical_section.js index 2cef54da249..43f00e19537 100644 --- a/jstests/sharding/migration_coordinator_shutdown_in_critical_section.js +++ b/jstests/sharding/migration_coordinator_shutdown_in_critical_section.js @@ -17,6 +17,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; load('jstests/libs/parallel_shell_helpers.js'); load('jstests/libs/fail_point_util.js'); diff --git a/jstests/sharding/mongos_insert_fails_with_shutdown.js b/jstests/sharding/mongos_insert_fails_with_shutdown.js index df02f6f54e6..6a71c6338dd 100644 --- a/jstests/sharding/mongos_insert_fails_with_shutdown.js +++ b/jstests/sharding/mongos_insert_fails_with_shutdown.js @@ -8,6 +8,8 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; TestData.skipCheckRoutingTableConsistency = true; +TestData.skipCheckShardFilteringMetadata = true; + (function() { "use strict"; diff --git a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js index eb442525cd8..db9bdfc0396 100644 --- a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js +++ b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js @@ -16,6 +16,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; // Replica set nodes started with --shardsvr do not enable key generation until they are added to a // sharded cluster and reject commands with gossiped clusterTime from users without the diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js index 66c2133c90a..af6b50af74d 100644 --- a/jstests/sharding/mongos_rs_shard_failure_tolerance.js +++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js @@ -15,6 +15,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/mongos_shard_failure_tolerance.js b/jstests/sharding/mongos_shard_failure_tolerance.js index 9707d8f1ce9..410777d7cf5 100644 --- a/jstests/sharding/mongos_shard_failure_tolerance.js +++ b/jstests/sharding/mongos_shard_failure_tolerance.js @@ -10,9 +10,10 @@ // sequence), idle (connection is connected but not used before a shard change), and new // (connection connected after shard change). // -// Checking UUID and index consistency involves talking to shards, but this test shuts down shards. +// The following checks involve talking to shards, but this test shuts down shards. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/primary_config_server_blackholed_from_mongos.js b/jstests/sharding/primary_config_server_blackholed_from_mongos.js index ac3be0782f4..080bc63057e 100644 --- a/jstests/sharding/primary_config_server_blackholed_from_mongos.js +++ b/jstests/sharding/primary_config_server_blackholed_from_mongos.js @@ -9,6 +9,7 @@ load('jstests/replsets/rslib.js'); // the mongos in this test. TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; var st = new ShardingTest({shards: 2, mongos: 1, useBridge: true}); diff --git a/jstests/sharding/query/aggregation_currentop.js b/jstests/sharding/query/aggregation_currentop.js index ec74ee2cf5d..e9dc5f04328 100644 --- a/jstests/sharding/query/aggregation_currentop.js +++ b/jstests/sharding/query/aggregation_currentop.js @@ -20,6 +20,7 @@ TestData.skipAwaitingReplicationOnShardsBeforeCheckingUUIDs = true; // Restarts shard nodes with no keyFile. TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js b/jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js index 42b191fb01a..518ffb6ee39 100644 --- a/jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js +++ b/jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js @@ -1,6 +1,10 @@ /** * Tests the interaction of the refineCollectionShardKey command with the range deleter. */ + +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { "use strict"; diff --git a/jstests/sharding/read_pref.js b/jstests/sharding/read_pref.js index 3a0c928f92f..37396c4acaf 100644 --- a/jstests/sharding/read_pref.js +++ b/jstests/sharding/read_pref.js @@ -7,6 +7,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/refine_collection_shard_key_atomic.js b/jstests/sharding/refine_collection_shard_key_atomic.js index 75781576c50..2534ebdd1a7 100644 --- a/jstests/sharding/refine_collection_shard_key_atomic.js +++ b/jstests/sharding/refine_collection_shard_key_atomic.js @@ -3,6 +3,9 @@ // config.chunks, and config.tags. // +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; load('jstests/libs/fail_point_util.js'); diff --git a/jstests/sharding/refine_collection_shard_key_basic.js b/jstests/sharding/refine_collection_shard_key_basic.js index 99a6f8761ef..6ce6d05137d 100644 --- a/jstests/sharding/refine_collection_shard_key_basic.js +++ b/jstests/sharding/refine_collection_shard_key_basic.js @@ -2,6 +2,9 @@ // Basic tests for refineCollectionShardKey. // +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; load('jstests/libs/fail_point_util.js'); diff --git a/jstests/sharding/refine_collection_shard_key_changelog.js b/jstests/sharding/refine_collection_shard_key_changelog.js index ae4e50f91d0..db44d2d2d37 100644 --- a/jstests/sharding/refine_collection_shard_key_changelog.js +++ b/jstests/sharding/refine_collection_shard_key_changelog.js @@ -2,6 +2,9 @@ // Basic tests for refineCollectionShardKey. // +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/refine_collection_shard_key_drop_chunks.js b/jstests/sharding/refine_collection_shard_key_drop_chunks.js index 0fb36efdc7d..1f329298dff 100644 --- a/jstests/sharding/refine_collection_shard_key_drop_chunks.js +++ b/jstests/sharding/refine_collection_shard_key_drop_chunks.js @@ -3,6 +3,9 @@ // cache. // +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/refine_collection_shard_key_jumbo.js b/jstests/sharding/refine_collection_shard_key_jumbo.js index 8485046d2b3..ad688fa3f5d 100644 --- a/jstests/sharding/refine_collection_shard_key_jumbo.js +++ b/jstests/sharding/refine_collection_shard_key_jumbo.js @@ -2,6 +2,9 @@ // Jumbo tests for refineCollectionShardKey. // +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/refine_collection_shard_key_primary_without_chunks.js b/jstests/sharding/refine_collection_shard_key_primary_without_chunks.js index 67bde6479c2..b53e7476688 100644 --- a/jstests/sharding/refine_collection_shard_key_primary_without_chunks.js +++ b/jstests/sharding/refine_collection_shard_key_primary_without_chunks.js @@ -1,5 +1,9 @@ // Verifies refining a shard key checks for the presence of a compatible shard key index on a shard // with chunks, not the primary shard. + +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { "use strict"; diff --git a/jstests/sharding/refine_shardkey_config_cache_refresh.js b/jstests/sharding/refine_shardkey_config_cache_refresh.js index 5c0c813d35b..4f3cf710d05 100644 --- a/jstests/sharding/refine_shardkey_config_cache_refresh.js +++ b/jstests/sharding/refine_shardkey_config_cache_refresh.js @@ -5,6 +5,9 @@ * sure that the shard will be able to eventually reach the valid state on config.cache. */ +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/rename.js b/jstests/sharding/rename.js index 0fe30d13475..5e44f760796 100644 --- a/jstests/sharding/rename.js +++ b/jstests/sharding/rename.js @@ -1,5 +1,6 @@ -// Checking UUID consistency involves talking to a shard node, which in this test is shutdown +// The following checks involve talking to a shard node, which in this test is shutdown. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/repl_monitor_refresh.js b/jstests/sharding/repl_monitor_refresh.js index f65a495d3b7..90041649f12 100644 --- a/jstests/sharding/repl_monitor_refresh.js +++ b/jstests/sharding/repl_monitor_refresh.js @@ -8,10 +8,10 @@ load("jstests/replsets/rslib.js"); (function() { "use strict"; -// Skip db hash check and shard replication since the removed node has wrong config and is still -// alive. +// Skip the following checks since the removed node has wrong config and is still alive. TestData.skipCheckDBHashes = true; TestData.skipAwaitingReplicationOnShardsBeforeCheckingUUIDs = true; +TestData.skipCheckShardFilteringMetadata = true; var NODE_COUNT = 3; var st = new ShardingTest({shards: {rs0: {nodes: NODE_COUNT, oplogSize: 10}}}); diff --git a/jstests/sharding/shard_encrypted_collection.js b/jstests/sharding/shard_encrypted_collection.js index d4c4de4f642..c5e8cd70ac6 100644 --- a/jstests/sharding/shard_encrypted_collection.js +++ b/jstests/sharding/shard_encrypted_collection.js @@ -5,6 +5,10 @@ * requires_fcv_60, * ] */ + +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { 'use strict'; diff --git a/jstests/sharding/shard_insert_getlasterror_w2.js b/jstests/sharding/shard_insert_getlasterror_w2.js index 629b97d321a..4aa2afdc524 100644 --- a/jstests/sharding/shard_insert_getlasterror_w2.js +++ b/jstests/sharding/shard_insert_getlasterror_w2.js @@ -6,6 +6,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/shard_removal_triggers_catalog_cache_invalidation.js b/jstests/sharding/shard_removal_triggers_catalog_cache_invalidation.js index 3fe72624b03..1a07396ddeb 100644 --- a/jstests/sharding/shard_removal_triggers_catalog_cache_invalidation.js +++ b/jstests/sharding/shard_removal_triggers_catalog_cache_invalidation.js @@ -9,6 +9,7 @@ load('jstests/sharding/libs/remove_shard_util.js'); // Checking UUID consistency involves talking to shards, but this test shuts down shards. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; // TODO SERVER-50144 Remove this and allow orphan checking. // This test calls removeShard which can leave docs in config.rangeDeletions in state "pending", diff --git a/jstests/sharding/sharding_multiple_ns_rs.js b/jstests/sharding/sharding_multiple_ns_rs.js index f41762b158e..4db3e4280f9 100644 --- a/jstests/sharding/sharding_multiple_ns_rs.js +++ b/jstests/sharding/sharding_multiple_ns_rs.js @@ -3,6 +3,7 @@ // not the shell detects the new primary before issuing the command is nondeterministic, skip the // consistency check for this test. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/sharding/startup_with_all_configs_down.js b/jstests/sharding/startup_with_all_configs_down.js index 06d2ceefb00..02607ecf8b6 100644 --- a/jstests/sharding/startup_with_all_configs_down.js +++ b/jstests/sharding/startup_with_all_configs_down.js @@ -6,9 +6,10 @@ // A restarted standalone will lose all data when using an ephemeral storage engine. // @tags: [requires_persistence] -// The UUID consistency check uses connections to shards cached on the ShardingTest object, but this -// test restarts a shard, so the cached connection is not usable. +// The following checks use connections to shards cached on the ShardingTest object, but this test +// restarts a shard, so the cached connection is not usable. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { "use strict"; diff --git a/jstests/sharding/timeseries_sharding_admin_commands.js b/jstests/sharding/timeseries_sharding_admin_commands.js index 954a3847d35..4f43152f548 100644 --- a/jstests/sharding/timeseries_sharding_admin_commands.js +++ b/jstests/sharding/timeseries_sharding_admin_commands.js @@ -6,6 +6,9 @@ * ] */ +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { "use strict"; diff --git a/jstests/sharding/verify_refine_collection_shard_key_opmessage.js b/jstests/sharding/verify_refine_collection_shard_key_opmessage.js index 91ca24d5deb..0da36485918 100644 --- a/jstests/sharding/verify_refine_collection_shard_key_opmessage.js +++ b/jstests/sharding/verify_refine_collection_shard_key_opmessage.js @@ -3,6 +3,10 @@ * * @tags: [requires_fcv_61] */ + +// Cannot run the filtering metadata check on tests that run refineCollectionShardKey. +TestData.skipCheckShardFilteringMetadata = true; + (function() { "use strict"; diff --git a/jstests/sharding/warm_up_connection_pool.js b/jstests/sharding/warm_up_connection_pool.js index ad87cbe21c6..a1b8e0b862a 100644 --- a/jstests/sharding/warm_up_connection_pool.js +++ b/jstests/sharding/warm_up_connection_pool.js @@ -9,6 +9,7 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true; TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; (function() { 'use strict'; diff --git a/jstests/ssl/sni_name_advertisement.js b/jstests/ssl/sni_name_advertisement.js index a2251e8c5e2..b9ead5566a6 100644 --- a/jstests/ssl/sni_name_advertisement.js +++ b/jstests/ssl/sni_name_advertisement.js @@ -10,6 +10,7 @@ load('jstests/ssl/libs/ssl_helpers.js'); TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; TestData.skipCheckRoutingTableConsistency = true; +TestData.skipCheckShardFilteringMetadata = true; let path = "jstests/libs/"; let pemKeyFile = path + "server.pem"; diff --git a/jstests/sslSpecial/mixed_mode_sharded_transition_nossl.js b/jstests/sslSpecial/mixed_mode_sharded_transition_nossl.js index 142d48a976f..47e3c1cce4c 100644 --- a/jstests/sslSpecial/mixed_mode_sharded_transition_nossl.js +++ b/jstests/sslSpecial/mixed_mode_sharded_transition_nossl.js @@ -10,6 +10,7 @@ // directly more complicated. Skip the hooks since this test doesn't really do migrations. TestData.skipCheckingIndexesConsistentAcrossCluster = true; TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; load('jstests/ssl/libs/ssl_helpers.js'); diff --git a/jstests/ssl_x509/mixed_mode_sharded_transition.js b/jstests/ssl_x509/mixed_mode_sharded_transition.js index 39756785b38..b6fd596337d 100644 --- a/jstests/ssl_x509/mixed_mode_sharded_transition.js +++ b/jstests/ssl_x509/mixed_mode_sharded_transition.js @@ -11,8 +11,9 @@ load('jstests/ssl/libs/ssl_helpers.js'); (function() { 'use strict'; -// The check orphans hook needs to be able to connect to the individual shards. +// These hooks need to be able to connect to the individual shards. TestData.skipCheckOrphans = true; +TestData.skipCheckShardFilteringMetadata = true; var transitionToX509AllowSSL = Object.merge(allowSSL, {transitionToAuth: '', clusterAuthMode: 'x509'}); diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js index 5aa0fd6245c..9e6007ed71e 100644 --- a/src/mongo/shell/shardingtest.js +++ b/src/mongo/shell/shardingtest.js @@ -527,6 +527,7 @@ var ShardingTest = function(params) { this.checkIndexesConsistentAcrossCluster(); this.checkOrphansAreDeleted(); this.checkRoutingTableConsistency(); + this.checkShardFilteringMetadata(); if (jsTestOptions().alwaysUseLogFiles) { if (opts.noCleanData === false) { @@ -1919,3 +1920,7 @@ ShardingTest.prototype.checkOrphansAreDeleted = function() { ShardingTest.prototype.checkRoutingTableConsistency = function() { print("Unhooked checkRoutingTableConsistency function"); }; + +ShardingTest.prototype.checkShardFilteringMetadata = function() { + print("Unhooked checkShardFilteringMetadata function"); +}; diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index f728d519835..f9fb2b29ac4 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -411,6 +411,7 @@ jsTestOptions = function() { alwaysUseLogFiles: TestData.alwaysUseLogFiles || false, skipCheckOrphans: TestData.skipCheckOrphans || false, skipCheckRoutingTableConsistency: TestData.skipCheckRoutingTableConsistency || false, + skipCheckShardFilteringMetadata: TestData.skipCheckShardFilteringMetadata || false, inEvergreen: TestData.inEvergreen || false, defaultReadPreference: TestData.defaultReadPreference, |