diff options
author | Kevin Albertson <kevin.albertson@10gen.com> | 2018-02-11 18:54:58 -0500 |
---|---|---|
committer | Max Hirschhorn <max.hirschhorn@mongodb.com> | 2018-02-11 18:54:58 -0500 |
commit | ba709552e7792ae53aeb7a22752ab9c65e717fc1 (patch) | |
tree | 2b0d7065d162e5a564a99ccc700f3f88578d5834 | |
parent | a1f7c461e9f635e085f93a804f3c820ccc9d614d (diff) | |
download | mongo-ba709552e7792ae53aeb7a22752ab9c65e717fc1.tar.gz |
SERVER-21630 run CheckReplDBHash on csrs and shard replset
And run ValidateCollections hook on priority=0 secondaries.
(cherry picked from commit b9decc492c0ff942d5bcd6e8c799de70fa0839af)
17 files changed, 241 insertions, 90 deletions
diff --git a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml index de8b568eee6..fa9f48091f6 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml @@ -43,6 +43,7 @@ executor: readMode: commands eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml index 48b687d8430..61ef858c0d7 100644 --- a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough.yml @@ -151,6 +151,7 @@ executor: eval: load("jstests/libs/override_methods/enable_causal_consistency.js") readMode: commands hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml index a74643c715c..7649329d3c2 100644 --- a/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml +++ b/buildscripts/resmokeconfig/suites/causally_consistent_jscore_passthrough_auth.yml @@ -202,6 +202,12 @@ executor: <<: *authOptions readMode: commands hooks: + - class: CheckReplDBHash + shell_options: + global_vars: + TestData: *TestData + eval: jsTest.authenticate(db.getMongo()) + <<: *authOptions - class: ValidateCollections shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/change_streams_mongos_passthrough.yml b/buildscripts/resmokeconfig/suites/change_streams_mongos_passthrough.yml index 41566db9a70..21fc4db91f8 100644 --- a/buildscripts/resmokeconfig/suites/change_streams_mongos_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/change_streams_mongos_passthrough.yml @@ -17,6 +17,7 @@ executor: eval: "var testingReplication = true; load('jstests/libs/override_methods/set_read_and_write_concerns.js');" readMode: commands hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/change_streams_secondary_reads.yml b/buildscripts/resmokeconfig/suites/change_streams_secondary_reads.yml index 67a5c4dba58..35c2e62224f 100644 --- a/buildscripts/resmokeconfig/suites/change_streams_secondary_reads.yml +++ b/buildscripts/resmokeconfig/suites/change_streams_secondary_reads.yml @@ -21,6 +21,7 @@ executor: load('jstests/libs/override_methods/set_read_and_write_concerns.js'); load('jstests/libs/override_methods/set_read_preference_secondary.js'); hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/change_streams_sharded_collections_passthrough.yml b/buildscripts/resmokeconfig/suites/change_streams_sharded_collections_passthrough.yml index 9d0b301e992..02c305d732d 100644 --- a/buildscripts/resmokeconfig/suites/change_streams_sharded_collections_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/change_streams_sharded_collections_passthrough.yml @@ -19,6 +19,7 @@ executor: load('jstests/libs/override_methods/implicitly_shard_accessed_collections.js'); readMode: commands hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml index 8f7d234bbd8..0172e8043c7 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml @@ -8,6 +8,7 @@ selector: executor: config: {} hooks: + - class: CheckReplDBHash - class: ValidateCollections fixture: class: ShardedClusterFixture diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml index d99ce811883..7fccdafedf9 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml @@ -9,6 +9,13 @@ executor: shell_options: readMode: commands hooks: + - class: CheckReplDBHash + shell_options: + global_vars: + TestData: + excludedDBsFromDBHash: + - config + skipValidationOnInvalidViewDefinitions: true - class: ValidateCollections shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml index 27f9365e401..87ce206929a 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_causal_consistency.yml @@ -14,6 +14,13 @@ executor: runningWithCausalConsistency: true usingReplicaSetShards: true hooks: + - class: CheckReplDBHash + shell_options: + global_vars: + TestData: + excludedDBsFromDBHash: + - config + skipValidationOnInvalidViewDefinitions: true - class: ValidateCollections shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml index 7b5d9868d08..b301c5ac10b 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded_session.yml @@ -10,6 +10,13 @@ executor: eval: load("jstests/libs/override_methods/enable_sessions.js") readMode: commands hooks: + - class: CheckReplDBHash + shell_options: + global_vars: + TestData: + excludedDBsFromDBHash: + - config + skipValidationOnInvalidViewDefinitions: true - class: ValidateCollections shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml index c6bd70af7a2..ec092595f5e 100644 --- a/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_causally_consistent_jscore_passthrough.yml @@ -202,6 +202,7 @@ executor: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js"); readMode: commands hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml index 63e7aee5b0c..9e829abc140 100644 --- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml @@ -93,6 +93,7 @@ executor: readMode: commands eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml index 5ea71956614..36d884110ac 100644 --- a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml @@ -2,6 +2,11 @@ config_variables: - &keyFile jstests/libs/authTestsKey - &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly +- &authOptions + authenticationDatabase: admin + authenticationMechanism: SCRAM-SHA-1 + password: *keyFileData + username: __system test_kind: js_test @@ -17,18 +22,21 @@ executor: config: shell_options: global_vars: - TestData: + TestData: &TestData auth: true authMechanism: SCRAM-SHA-1 keyFile: *keyFile keyFileData: *keyFileData eval: jsTest.authenticate(db.getMongo()) - authenticationDatabase: admin - authenticationMechanism: SCRAM-SHA-1 - password: *keyFileData - username: __system + <<: *authOptions readMode: commands - + hooks: + - class: CheckReplDBHash + shell_options: + global_vars: + TestData: *TestData + eval: jsTest.authenticate(db.getMongo()) + <<: *authOptions fixture: class: ShardedClusterFixture mongos_options: diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_op_query_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_op_query_passthrough.yml index 1f575054328..d4236d02f09 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_op_query_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_op_query_passthrough.yml @@ -58,6 +58,7 @@ executor: rpcProtocols: opQueryOnly readMode: commands hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml index 96a90ae1c00..9516d36fa69 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml @@ -57,6 +57,7 @@ executor: shell_options: readMode: commands hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/jstests/hooks/run_check_repl_dbhash.js b/jstests/hooks/run_check_repl_dbhash.js index 5a9606e20b1..636266e44cb 100644 --- a/jstests/hooks/run_check_repl_dbhash.js +++ b/jstests/hooks/run_check_repl_dbhash.js @@ -3,96 +3,197 @@ 'use strict'; (function() { - // A thin wrapper around master/slave nodes that provides the getHashes(), getPrimary(), - // awaitReplication(), and nodeList() methods. - // DEPRECATED: this wrapper only supports nodes started through resmoke's masterslave.py - // fixture. Please do not use it with other master/slave clusters. - var MasterSlaveDBHashTest = function(primaryHost) { - var master = new Mongo(primaryHost); - var masterPort = master.host.split(':')[1]; - var slave = new Mongo('localhost:' + String(parseInt(masterPort) + 1)); - - this.nodeList = function() { - return [master.host, slave.host]; - }; - - this.getHashes = function(db) { - var combinedRes = {}; - var res = master.getDB(db).runCommand("dbhash"); - assert.commandWorked(res); - combinedRes.master = res; - - res = slave.getDB(db).runCommand("dbhash"); - assert.commandWorked(res); - combinedRes.slaves = [res]; - - return combinedRes; - }; - - this.getPrimary = function() { - slave.setSlaveOk(); - this.liveNodes = {master: master, slaves: [slave]}; - - return master; - }; - - this.getSecondaries = function() { - return [slave]; - }; - - this.awaitReplication = function() { - assert.commandWorked(master.adminCommand({fsyncUnlock: 1}), - 'failed to unlock the primary'); - - print('Starting fsync on master to flush all pending writes'); - assert.commandWorked(master.adminCommand({fsync: 1})); - print('fsync on master completed'); - - var timeout = 60 * 1000 * 5; // 5min timeout - var dbNames = master.getDBNames(); - print('Awaiting replication of inserts into ' + dbNames); - for (var dbName of dbNames) { - if (dbName === 'local') - continue; - assert.writeOK(master.getDB(dbName).await_repl.insert( - {awaiting: 'repl'}, {writeConcern: {w: 2, wtimeout: timeout}}), - 'Awaiting replication failed'); + load('jstests/libs/parallelTester.js'); + + function isMasterSlave(uri) { + const mongo = new Mongo(uri); + jsTest.authenticate(mongo); + const cmdLineOpts = mongo.getDB('admin').adminCommand('getCmdLineOpts'); + assert.commandWorked(cmdLineOpts); + return cmdLineOpts.parsed.master === true; + } + + function isMultiNodeReplSet(uri) { + const mongo = new Mongo(uri); + let hosts = []; + const isMaster = mongo.adminCommand({isMaster: 1}); + if (isMaster.hasOwnProperty('setName')) { + let hosts = isMaster.hosts; + if (isMaster.hasOwnProperty('passives')) { + hosts = hosts.concat(isMaster.passives); } - print('Finished awaiting replication'); - assert.commandWorked(master.adminCommand({fsync: 1, lock: 1}), - 'failed to re-lock the primary'); - }; - - this.checkReplicatedDataHashes = function() { - ReplSetTest({nodes: 0}).checkReplicatedDataHashes.apply(this, arguments); - }; - - this.checkReplicaSet = function() { - ReplSetTest({nodes: 0}).checkReplicaSet.apply(this, arguments); - }; - }; - - var startTime = Date.now(); + } + return hosts.length > 1; + } + + // Adds the uri and description (replset or master-slave) if server needs dbhash check. + function checkAndAddServerDesc(uri, out) { + // No need to check the dbhash of single node replsets. + if (isMultiNodeReplSet(uri)) { + out.push({type: 'replset', uri: uri}); + } else if (isMasterSlave(uri)) { + out.push({type: 'master-slave', uri: uri}); + } + } + + function checkReplDataHashThread(serverDesc, testData, excludedDBs) { + // A thin wrapper around master/slave nodes that provides the getHashes(), getPrimary(), + // awaitReplication(), and nodeList() methods. + // DEPRECATED: this wrapper only supports nodes started through resmoke's masterslave.py + // fixture. Please do not use it with other master/slave clusters. + function MasterSlaveDBHashTest(primaryHost) { + const master = new Mongo(primaryHost); + const masterPort = master.host.split(':')[1]; + const slave = new Mongo('localhost:' + String(parseInt(masterPort) + 1)); + + this.nodeList = function() { + return [master.host, slave.host]; + }; + + this.getHashes = function(db) { + const combinedRes = {}; + let res = master.getDB(db).runCommand('dbhash'); + assert.commandWorked(res); + combinedRes.master = res; + + res = slave.getDB(db).runCommand('dbhash'); + assert.commandWorked(res); + combinedRes.slaves = [res]; + + return combinedRes; + }; + + this.getPrimary = function() { + slave.setSlaveOk(); + this.liveNodes = {master: master, slaves: [slave]}; + return master; + }; + + this.getSecondaries = function() { + return [slave]; + }; + + this.awaitReplication = function() { + assert.commandWorked(master.adminCommand({fsyncUnlock: 1}), + 'failed to unlock the primary'); + + print('Starting fsync on master to flush all pending writes'); + assert.commandWorked(master.adminCommand({fsync: 1})); + print('fsync on master completed'); + + const kTimeout = 60 * 1000 * 5; // 5min timeout + const dbNames = master.getDBNames(); + print('Awaiting replication of inserts into ' + dbNames); + for (let dbName of dbNames) { + if (dbName === 'local') + continue; + assert.writeOK( + master.getDB(dbName).await_repl.insert( + {awaiting: 'repl'}, {writeConcern: {w: 2, wtimeout: kTimeout}}), + 'Awaiting replication failed'); + } + print('Finished awaiting replication'); + assert.commandWorked(master.adminCommand({fsync: 1, lock: 1}), + 'failed to re-lock the primary'); + }; + + this.checkReplicatedDataHashes = function() { + ReplSetTest({nodes: 0}).checkReplicatedDataHashes.apply(this, ['test', [], true]); + }; + + this.checkReplicaSet = function() { + ReplSetTest({nodes: 0}).checkReplicaSet.apply(this, arguments); + }; + + this.dumpOplog = function() { + print('master-slave cannot dump oplog'); + }; + } + + TestData = testData; + + // Since UUIDs aren't explicitly replicated in master-slave deployments, we ignore the UUID + // in the output of the 'listCollections' command to avoid reporting a known data + // inconsistency issue from checkReplicatedDataHashes(). + const ignoreUUIDs = serverDesc.type === 'master-slave'; + let fixture = null; + if (serverDesc.type === 'replset') { + fixture = new ReplSetTest(serverDesc.uri); + } else if (serverDesc.type === 'master-slave') { + fixture = new MasterSlaveDBHashTest(serverDesc.uri); + } else { + throw 'unrecognized server type ' + serverDesc.type; + } + fixture.checkReplicatedDataHashes(undefined, excludedDBs, ignoreUUIDs); + } + + let startTime = Date.now(); assert.neq(typeof db, 'undefined', 'No `db` object, is the shell connected to a mongod?'); - var primaryInfo = db.isMaster(); + // stores each server type (master/slave or replset) and uri. + const serversNeedingReplDataHashCheck = []; + const primaryInfo = db.isMaster(); + const isMongos = primaryInfo.msg === 'isdbgrid'; + const isReplSet = primaryInfo.hasOwnProperty('setName'); + const uri = db.getMongo().host; assert(primaryInfo.ismaster, 'shell is not connected to the primary or master node: ' + tojson(primaryInfo)); - var cmdLineOpts = db.adminCommand('getCmdLineOpts'); - assert.commandWorked(cmdLineOpts); - var isMasterSlave = cmdLineOpts.parsed.master === true; - var testFixture = isMasterSlave ? new MasterSlaveDBHashTest(db.getMongo().host) - : new ReplSetTest(db.getMongo().host); - var excludedDBs = jsTest.options().excludedDBsFromDBHash || []; - - // Since UUIDs aren't explicitly replicated in master-slave deployments, we ignore the UUID in - // the output of the "listCollections" command to avoid reporting a known data inconsistency - // issue from checkReplicatedDataHashes(). - var ignoreUUIDs = isMasterSlave; - testFixture.checkReplicatedDataHashes(undefined, excludedDBs, ignoreUUIDs); - - var totalTime = Date.now() - startTime; + assert(isMongos || isReplSet || isMasterSlave(uri), + 'not replset, master/slave, or sharded cluster'); + + if (isMongos) { + // Add shards and config server if they are replica sets. + let res = db.adminCommand('getShardMap'); + assert.commandWorked(res); + const csURI = res.map.config; + res = db.adminCommand('listShards'); + assert.commandWorked(res); + const shardURIs = res.shards.map((shard) => shard.host); + + checkAndAddServerDesc(csURI, serversNeedingReplDataHashCheck); + shardURIs.forEach((shardURI) => { + checkAndAddServerDesc(shardURI, serversNeedingReplDataHashCheck); + }); + } else { + checkAndAddServerDesc(uri, serversNeedingReplDataHashCheck); + } + + const threads = []; + const excludedDBs = jsTest.options().excludedDBsFromDBHash || []; + serversNeedingReplDataHashCheck.forEach((serverDesc) => { + const thread = new ScopedThread(checkReplDataHashThread, serverDesc, TestData, excludedDBs); + threads.push({serverDesc: serverDesc, handle: thread}); + thread.start(); + }); + + if (serversNeedingReplDataHashCheck.length === 0) { + let skipReason = 'No multi-node replication detected in '; + if (isMongos) { + skipReason += 'sharded cluster'; + } else if (isReplSet) { + skipReason += 'replica set'; + } else { + skipReason += 'master-slave set'; + } + + print('Skipping consistency checks for cluster because ' + skipReason); + return; + } + + const failedChecks = []; + threads.forEach(thread => { + thread.handle.join(); + if (thread.handle.hasFailed()) { + failedChecks.push(thread.serverDesc.uri + ' (' + thread.serverDesc.type + ')'); + } + }); + + assert.eq(failedChecks.length, + 0, + 'dbhash check failed for the following hosts: ' + failedChecks.join(',')); + + const totalTime = Date.now() - startTime; print('Finished consistency checks of cluster in ' + totalTime + ' ms.'); })(); diff --git a/jstests/hooks/run_validate_collections.js b/jstests/hooks/run_validate_collections.js index d95c9eb5c9f..b1052413d56 100644 --- a/jstests/hooks/run_validate_collections.js +++ b/jstests/hooks/run_validate_collections.js @@ -15,6 +15,11 @@ for (let hostString of res.hosts) { connections.push(new Mongo(hostString)); } + if (res.hasOwnProperty('passives')) { + for (let hostString of res.passives) { + connections.push(new Mongo(hostString)); + } + } } else { connections.push(conn); } |