diff options
26 files changed, 109 insertions, 7 deletions
diff --git a/jstests/gle/gle_sharded_wc.js b/jstests/gle/gle_sharded_wc.js index 8031e154f89..99f3ff9cf32 100644 --- a/jstests/gle/gle_sharded_wc.js +++ b/jstests/gle/gle_sharded_wc.js @@ -8,6 +8,7 @@ // Checking UUID consistency involves talking to the shard primaries, but by the end of this test, // one shard does not have a primary. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; +TestData.skipCheckDBHashes = true; (function() { 'use strict'; diff --git a/jstests/noPassthrough/minvalid2.js b/jstests/noPassthrough/minvalid2.js index 6b7fbeee422..d12de080248 100644 --- a/jstests/noPassthrough/minvalid2.js +++ b/jstests/noPassthrough/minvalid2.js @@ -18,6 +18,9 @@ * @tags: [requires_persistence, requires_replication] */ +// Skip db hash check because replset cannot reach consistent state. +TestData.skipCheckDBHashes = true; + print("1. make 3-member set w/arb (2)"); var name = "minvalid"; var replTest = new ReplSetTest({name: name, nodes: 3, oplogSize: 1, waitForKeys: true}); diff --git a/jstests/noPassthroughWithMongod/indexbg_restart_secondary_noretry.js b/jstests/noPassthroughWithMongod/indexbg_restart_secondary_noretry.js index a15908a744e..f59a59e0070 100644 --- a/jstests/noPassthroughWithMongod/indexbg_restart_secondary_noretry.js +++ b/jstests/noPassthroughWithMongod/indexbg_restart_secondary_noretry.js @@ -8,6 +8,10 @@ (function() { 'use strict'; + // Skip db hash check because secondary will have different number of indexes due to the + // --noIndexBuildRetry command line option. + TestData.skipCheckDBHashes = true; + // Set up replica set. var replTest = new ReplSetTest( {name: 'bgIndexNoRetry', nodes: 3, nodeOptions: {noIndexBuildRetry: "", syncdelay: 1}}); diff --git a/jstests/noPassthroughWithMongod/ttl_repl_secondary_disabled.js b/jstests/noPassthroughWithMongod/ttl_repl_secondary_disabled.js index 73ef164b2dd..135bc41a608 100644 --- a/jstests/noPassthroughWithMongod/ttl_repl_secondary_disabled.js +++ b/jstests/noPassthroughWithMongod/ttl_repl_secondary_disabled.js @@ -2,6 +2,10 @@ * @tags: [requires_replication] */ +// Skip db hash check because secondary will have extra document due to the usage of the godinsert +// command. +TestData.skipCheckDBHashes = true; + var rt = new ReplSetTest({name: "ttl_repl", nodes: 2}); // setup set diff --git a/jstests/replsets/apply_batch_only_goes_forward.js b/jstests/replsets/apply_batch_only_goes_forward.js index 4db9c825565..7dc8f1cf5da 100644 --- a/jstests/replsets/apply_batch_only_goes_forward.js +++ b/jstests/replsets/apply_batch_only_goes_forward.js @@ -14,6 +14,10 @@ * restarting, so will initial sync in this scenario, invalidating the test. * @tags: [requires_persistence] */ + +// Skip db hash check because replset cannot reach consistent state. +TestData.skipCheckDBHashes = true; + (function() { "use strict"; diff --git a/jstests/replsets/buildindexes.js b/jstests/replsets/buildindexes.js index f6a8a781014..1cd6dd1b272 100644 --- a/jstests/replsets/buildindexes.js +++ b/jstests/replsets/buildindexes.js @@ -1,7 +1,9 @@ // Check that buildIndexes config option is working (function() { - + // Skip db hash check because secondary will have different number of indexes due to + // buildIndexes=false on the secondary. + TestData.skipCheckDBHashes = true; var name = "buildIndexes"; var host = getHostName(); diff --git a/jstests/replsets/clean_shutdown_oplog_state.js b/jstests/replsets/clean_shutdown_oplog_state.js index effa8469d7b..f1e361acc5e 100644 --- a/jstests/replsets/clean_shutdown_oplog_state.js +++ b/jstests/replsets/clean_shutdown_oplog_state.js @@ -7,6 +7,9 @@ (function() { "use strict"; + // Skip db hash check because secondary restarted as standalone. + TestData.skipCheckDBHashes = true; + var rst = new ReplSetTest({ name: "name", nodes: 2, diff --git a/jstests/replsets/double_rollback_early.js b/jstests/replsets/double_rollback_early.js index 11ceb4fefa2..1e8e33cbe2b 100644 --- a/jstests/replsets/double_rollback_early.js +++ b/jstests/replsets/double_rollback_early.js @@ -7,6 +7,9 @@ (function() { 'use strict'; + // Skip db hash check because replset is partitioned. + TestData.skipCheckDBHashes = true; + load("jstests/libs/check_log.js"); load("jstests/replsets/rslib.js"); diff --git a/jstests/replsets/initial_sync_drop_collection.js b/jstests/replsets/initial_sync_drop_collection.js index fc4a953b0d5..e64fe7d38a9 100644 --- a/jstests/replsets/initial_sync_drop_collection.js +++ b/jstests/replsets/initial_sync_drop_collection.js @@ -3,6 +3,9 @@ (function() { "use strict"; + // Skip db hash check because secondary cannot complete initial sync. + TestData.skipCheckDBHashes = true; + load("jstests/libs/check_log.js"); load('jstests/replsets/libs/two_phase_drops.js'); load("jstests/libs/uuid_util.js"); diff --git a/jstests/replsets/initial_sync_invalid_index_spec.js b/jstests/replsets/initial_sync_invalid_index_spec.js index d1126477894..f8e974eaee6 100644 --- a/jstests/replsets/initial_sync_invalid_index_spec.js +++ b/jstests/replsets/initial_sync_invalid_index_spec.js @@ -6,6 +6,9 @@ (function() { "use strict"; + // Skip db hash check because of invalid index spec. + TestData.skipCheckDBHashes = true; + load("jstests/replsets/rslib.js"); const testName = "initial_sync_invalid_index_spec"; diff --git a/jstests/replsets/ismaster1.js b/jstests/replsets/ismaster1.js index 96de229832a..cfb50a12912 100644 --- a/jstests/replsets/ismaster1.js +++ b/jstests/replsets/ismaster1.js @@ -3,6 +3,9 @@ * also checks that fields that should not be in the document are absent */ +// Skip db hash check because node 2 is slave delayed and may time out on awaitReplication. +TestData.skipCheckDBHashes = true; + load("jstests/replsets/rslib.js"); // function create the error message if an assert fails diff --git a/jstests/replsets/libs/apply_ops_insert_write_conflict.js b/jstests/replsets/libs/apply_ops_insert_write_conflict.js index d0a47f5b6d8..4cbdbe18e88 100644 --- a/jstests/replsets/libs/apply_ops_insert_write_conflict.js +++ b/jstests/replsets/libs/apply_ops_insert_write_conflict.js @@ -4,6 +4,9 @@ var ApplyOpsInsertWriteConflictTest = function(options) { 'use strict'; + // Skip db hash check because this test may throw write conflicts and collmod fails. + TestData.skipCheckDBHashes = true; + if (!(this instanceof ApplyOpsInsertWriteConflictTest)) { return new ApplyOpsInsertWriteConflictTest(options); } diff --git a/jstests/replsets/libs/tags.js b/jstests/replsets/libs/tags.js index 0b61c990c49..d1aeac41b12 100644 --- a/jstests/replsets/libs/tags.js +++ b/jstests/replsets/libs/tags.js @@ -6,6 +6,9 @@ var TagsTest = function(options) { 'use strict'; + // Skip db hash check since this test leaves replset partitioned. + TestData.skipCheckDBHashes = true; + if (!(this instanceof TagsTest)) { return new TagsTest(options); } diff --git a/jstests/replsets/operation_time_read_and_write_concern.js b/jstests/replsets/operation_time_read_and_write_concern.js index 5e1591bc328..337de5ebe3a 100644 --- a/jstests/replsets/operation_time_read_and_write_concern.js +++ b/jstests/replsets/operation_time_read_and_write_concern.js @@ -5,6 +5,9 @@ (function() { "use strict"; + // Skip db hash check because replication is stopped on secondaries. + TestData.skipCheckDBHashes = true; + load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. load("jstests/libs/write_concern_util.js"); // For stopReplicationOnSecondaries, // restartReplicationOnSecondaries diff --git a/jstests/replsets/reconfig.js b/jstests/replsets/reconfig.js index 312a3cb6243..7ae2f41d3c8 100644 --- a/jstests/replsets/reconfig.js +++ b/jstests/replsets/reconfig.js @@ -4,6 +4,10 @@ */ (function() { "use strict"; + + // Skip db hash check because secondary is left with a different config. + TestData.skipCheckDBHashes = true; + var numNodes = 5; var replTest = new ReplSetTest({name: 'testSet', nodes: numNodes}); var nodes = replTest.startSet(); diff --git a/jstests/replsets/rollback_auth.js b/jstests/replsets/rollback_auth.js index eeb7a131911..25cb87418f6 100644 --- a/jstests/replsets/rollback_auth.js +++ b/jstests/replsets/rollback_auth.js @@ -209,9 +209,9 @@ // Verify data consistency between nodes. authutil.asCluster(replTest.nodes, 'jstests/libs/key1', function() { - replTest.checkReplicatedDataHashes(); replTest.checkOplogs(); }); + // DB hash check is done in stopSet. replTest.stopSet(); }()); diff --git a/jstests/replsets/slave_delay_clean_shutdown.js b/jstests/replsets/slave_delay_clean_shutdown.js index db08dfab228..fe5b9e62164 100644 --- a/jstests/replsets/slave_delay_clean_shutdown.js +++ b/jstests/replsets/slave_delay_clean_shutdown.js @@ -5,6 +5,9 @@ load('jstests/replsets/rslib.js'); (function() { "use strict"; + // Skip db hash check since secondary has slave delay. + TestData.skipCheckDBHashes = true; + var ns = "test.coll"; var rst = new ReplSetTest({ diff --git a/jstests/replsets/zero_vote_arbiter.js b/jstests/replsets/zero_vote_arbiter.js index bc7552ef47b..91ef82ab8fe 100644 --- a/jstests/replsets/zero_vote_arbiter.js +++ b/jstests/replsets/zero_vote_arbiter.js @@ -6,6 +6,9 @@ var NewReplicaSetConfigurationIncompatible = 103; var InvalidReplicaSetConfig = 93; +// Skip db hash check since replsets are assigned invalid configs. +TestData.skipCheckDBHashes = true; + /* * Create replica set with 3 nodes, add new node as 0-vote arbiter. */ diff --git a/jstests/sharding/auth_no_config_primary.js b/jstests/sharding/auth_no_config_primary.js index 8620c317272..5d79df932a5 100644 --- a/jstests/sharding/auth_no_config_primary.js +++ b/jstests/sharding/auth_no_config_primary.js @@ -9,6 +9,7 @@ // Checking UUID consistency involves talking to the config server primary, but there is no config // server primary by the end of this test. TestData.skipCheckingUUIDsConsistentAcrossCluster = true; +TestData.skipCheckDBHashes = true; (function() { 'use strict'; diff --git a/jstests/sharding/linearizable_read_concern.js b/jstests/sharding/linearizable_read_concern.js index 4088b5d11c2..7f08a07b568 100644 --- a/jstests/sharding/linearizable_read_concern.js +++ b/jstests/sharding/linearizable_read_concern.js @@ -27,6 +27,9 @@ load("jstests/libs/write_concern_util.js"); (function() { "use strict"; + // Skip db hash check since this test leaves a replica set shard partitioned. + TestData.skipCheckDBHashes = true; + var testName = "linearizable_read_concern"; var st = new ShardingTest({ diff --git a/jstests/sharding/repl_monitor_refresh.js b/jstests/sharding/repl_monitor_refresh.js index 1e437c3ff0d..3d799dfbbcc 100644 --- a/jstests/sharding/repl_monitor_refresh.js +++ b/jstests/sharding/repl_monitor_refresh.js @@ -7,6 +7,9 @@ load("jstests/replsets/rslib.js"); (function() { "use strict"; + // Skip db hash check since the removed node has wrong config and is still alive. + TestData.skipCheckDBHashes = true; + var NODE_COUNT = 3; var st = new ShardingTest({shards: {rs0: {nodes: NODE_COUNT, oplogSize: 10}}}); var replTest = st.rs0; diff --git a/jstests/ssl/libs/ssl_helpers.js b/jstests/ssl/libs/ssl_helpers.js index 1c275a9641a..6dc20a4923c 100644 --- a/jstests/ssl/libs/ssl_helpers.js +++ b/jstests/ssl/libs/ssl_helpers.js @@ -104,6 +104,7 @@ function testShardedLookup(shardingTest) { * sharding test to see if they can work together... */ function mixedShardTest(options1, options2, shouldSucceed) { + let authSucceeded = false; try { // Start ShardingTest with enableBalancer because ShardingTest attempts to turn // off the balancer otherwise, which it will not be authorized to do if auth is enabled. @@ -119,6 +120,8 @@ function mixedShardTest(options1, options2, shouldSucceed) { st.admin.createUser({user: 'admin', pwd: 'pwd', roles: ['root']}); st.admin.auth('admin', 'pwd'); + authSucceeded = true; + st.stopBalancer(); // Test that $lookup works because it causes outgoing connections to be opened @@ -158,6 +161,12 @@ function mixedShardTest(options1, options2, shouldSucceed) { // silence error if we should fail... print("IMPORTANT! => Test failed when it should have failed...continuing..."); } finally { + // Authenticate csrs so ReplSetTest.stopSet() can do db hash check. + if (authSucceeded && st.configRS) { + st.configRS.nodes.forEach((node) => { + node.getDB('admin').auth('admin', 'pwd'); + }); + } // This has to be done in order for failure // to not prevent future tests from running... if (st) { diff --git a/jstests/ssl/sharding_with_x509.js b/jstests/ssl/sharding_with_x509.js index 90101a51ca4..621bfb1468a 100644 --- a/jstests/ssl/sharding_with_x509.js +++ b/jstests/ssl/sharding_with_x509.js @@ -68,5 +68,11 @@ // Make sure the right amount of data is there assert.eq(coll.find().itcount({my: 'test'}), toInsert / 2); + // Authenticate csrs so ReplSetTest.stopSet() can do db hash check. + if (st.configRS) { + st.configRS.nodes.forEach((node) => { + node.getDB('admin').auth('admin', 'pwd'); + }); + } st.stop(); })(); diff --git a/jstests/ssl/ssl_hostname_validation.js b/jstests/ssl/ssl_hostname_validation.js index d61109293ba..93ca98c522a 100644 --- a/jstests/ssl/ssl_hostname_validation.js +++ b/jstests/ssl/ssl_hostname_validation.js @@ -75,6 +75,9 @@ testCombination(SERVER_CERT, true, true, true); // BAD_SAN_CERT has SAN=BadSAN. testCombination(BAD_SAN_CERT, false, false, false); +// Skip db hash check because replset cannot initiate. +TestData.skipCheckDBHashes = true; + // 2. Initiate ReplSetTest with invalid certs ssl_options = { sslMode: "requireSSL", @@ -90,6 +93,8 @@ assert.throws(function() { }); replTest.stopSet(); +TestData.skipCheckDBHashes = false; + // 3. Initiate ReplSetTest with invalid certs but set allowInvalidHostnames ssl_options = { sslMode: "requireSSL", diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index e331650f54c..94a4194eafd 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -1445,7 +1445,7 @@ var ReplSetTest = function(opts) { var primary = rst.liveNodes.master; var combinedDBs = new Set(primary.getDBNames()); - rst.getSecondaries().forEach(secondary => { + rst.liveNodes.slaves.forEach(secondary => { secondary.getDBNames().forEach(dbName => combinedDBs.add(dbName)); }); @@ -1456,10 +1456,15 @@ var ReplSetTest = function(opts) { var dbHashes = rst.getHashes(dbName); var primaryDBHash = dbHashes.master; + var primaryCollections = Object.keys(primaryDBHash.collections); assert.commandWorked(primaryDBHash); try { - var primaryCollInfo = primary.getDB(dbName).getCollectionInfos(); + // Filter only collections that were retrieved by the dbhash. listCollections + // may include non-replicated collections like system.profile. + var primaryCollInfo = + primary.getDB(dbName).getCollectionInfos({name: {$in: primaryCollections}}); + } catch (e) { if (jsTest.options().skipValidationOnInvalidViewDefinitions) { assert.commandFailedWithCode(e, ErrorCodes.InvalidViewDefinition); @@ -1475,8 +1480,6 @@ var ReplSetTest = function(opts) { assert.commandWorked(secondaryDBHash); var secondary = secondaryDBHash._mongo; - - var primaryCollections = Object.keys(primaryDBHash.collections); var secondaryCollections = Object.keys(secondaryDBHash.collections); if (primaryCollections.length !== secondaryCollections.length) { @@ -1511,7 +1514,9 @@ var ReplSetTest = function(opts) { // Check that collection information is consistent on the primary and // secondaries. - var secondaryCollInfo = secondary.getDB(dbName).getCollectionInfos(); + var secondaryCollInfo = secondary.getDB(dbName).getCollectionInfos( + {name: {$in: secondaryCollections}}); + secondaryCollInfo.forEach(secondaryInfo => { primaryCollInfo.forEach(primaryInfo => { if (secondaryInfo.name === primaryInfo.name && @@ -1939,6 +1944,23 @@ var ReplSetTest = function(opts) { * @param {Object} opts @see MongoRunner.stopMongod */ this.stopSet = function(signal, forRestart, opts) { + // Check to make sure data is the same on all nodes. + if (!jsTest.options().skipCheckDBHashes) { + // To skip this check add TestData.skipCheckDBHashes = true; + // Reasons to skip this test include: + // - the primary goes down and none can be elected (so fsync lock/unlock commands fail) + // - the replica set is in an unrecoverable inconsistent state. E.g. the replica set + // is partitioned. + // + if (_callIsMaster() && + this.liveNodes.slaves.length > 0) { // skip for single node replsets. + // Auth only on live nodes because authutil.assertAuthenticate + // refuses to log in live connections if some secondaries are down. + asCluster([this.liveNodes.master, ...this.liveNodes.slaves], + () => this.checkReplicatedDataHashes()); + } + } + for (var i = 0; i < this.ports.length; i++) { this.stop(i, signal, opts); } diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index 6bb552d89da..561143847a7 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -265,6 +265,7 @@ jsTestOptions = function() { overrideRetryAttempts: TestData.overrideRetryAttempts || 0, logRetryAttempts: TestData.logRetryAttempts || false, connectionString: TestData.connectionString || "", + skipCheckDBHashes: TestData.skipCheckDBHashes || false, }); } return _jsTestOptions; |