diff options
author | Kevin Albertson <kevin.albertson@10gen.com> | 2018-03-25 01:22:47 -0400 |
---|---|---|
committer | Max Hirschhorn <max.hirschhorn@mongodb.com> | 2018-03-25 01:22:47 -0400 |
commit | 92e92c023843c2f73ce752284a14fa1e3ca92933 (patch) | |
tree | 7f538b20587bf30035b90c155890a7722d929936 | |
parent | e674fb5ae0b41a0f3efad5275a62cc4514d7bd42 (diff) | |
download | mongo-92e92c023843c2f73ce752284a14fa1e3ca92933.tar.gz |
SERVER-21630 run CheckReplDBHash on csrs and shard replset
And run ValidateCollections hook on priority=0 secondaries.
(cherry picked from commit b9decc492c0ff942d5bcd6e8c799de70fa0839af)
8 files changed, 214 insertions, 91 deletions
diff --git a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml index fa367432dc5..1cb834a63eb 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml @@ -34,6 +34,7 @@ executor: readMode: commands eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml index db7bc2b0c15..a0e8c16be9b 100644 --- a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml +++ b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml @@ -8,6 +8,7 @@ executor: cpp_integration_test: config: {} hooks: + - class: CheckReplDBHash - class: ValidateCollections fixture: class: ShardedClusterFixture diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml index 8eb912aee14..9a8a02db6b9 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml @@ -9,6 +9,11 @@ executor: shell_options: readMode: commands hooks: + - class: CheckReplDBHash + shell_options: + global_vars: + TestData: + skipValidationOnInvalidViewDefinitions: true - class: ValidateCollections shell_options: global_vars: diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml index d0f205b8840..96674cfc20c 100644 --- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml @@ -74,6 +74,7 @@ executor: readMode: commands eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml index 56b4854cde9..8879e39f143 100644 --- a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml @@ -2,6 +2,11 @@ config_variables: - &keyFile jstests/libs/authTestsKey - &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly +- &authOptions + authenticationDatabase: admin + authenticationMechanism: SCRAM-SHA-1 + password: *keyFileData + username: __system selector: js_test: @@ -17,18 +22,21 @@ executor: config: shell_options: global_vars: - TestData: + TestData: &TestData auth: true authMechanism: SCRAM-SHA-1 keyFile: *keyFile keyFileData: *keyFileData eval: jsTest.authenticate(db.getMongo()) - authenticationDatabase: admin - authenticationMechanism: SCRAM-SHA-1 - password: *keyFileData - username: __system + <<: *authOptions readMode: commands - + hooks: + - class: CheckReplDBHash + shell_options: + global_vars: + TestData: *TestData + eval: jsTest.authenticate(db.getMongo()) + <<: *authOptions fixture: class: ShardedClusterFixture mongos_options: diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml index 620ca06e310..496bd265a58 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml @@ -55,6 +55,7 @@ executor: shell_options: readMode: commands hooks: + - class: CheckReplDBHash - class: ValidateCollections - class: CleanEveryN n: 20 diff --git a/jstests/hooks/run_check_repl_dbhash.js b/jstests/hooks/run_check_repl_dbhash.js index 632f4e25cfa..a17e407be0e 100644 --- a/jstests/hooks/run_check_repl_dbhash.js +++ b/jstests/hooks/run_check_repl_dbhash.js @@ -3,96 +3,197 @@ 'use strict'; (function() { - // A thin wrapper around master/slave nodes that provides the getHashes(), getPrimary(), - // awaitReplication(), and nodeList() methods. - // DEPRECATED: this wrapper only supports nodes started through resmoke's masterslave.py - // fixture. Please do not use it with other master/slave clusters. - var MasterSlaveDBHashTest = function(primaryHost) { - var master = new Mongo(primaryHost); - var resolvedHost = getHostName(); - var masterPort = master.host.split(':')[1]; - // The 'host' property is modified manually because 'localhost' is used by default in a new - // Mongo() connection. We set the value to the real hostname because that is what the server - // uses. - master.host = resolvedHost + ':' + masterPort; - - var slave = new Mongo(resolvedHost + ':' + String(parseInt(masterPort) + 1)); - - this.nodeList = function() { - return [master.host, slave.host]; - }; - - this.getHashes = function(db) { - var combinedRes = {}; - var res = master.getDB(db).runCommand("dbhash"); - assert.commandWorked(res); - combinedRes.master = res; - - res = slave.getDB(db).runCommand("dbhash"); - assert.commandWorked(res); - combinedRes.slaves = [res]; - - return combinedRes; - }; - - this.getPrimary = function() { - slave.setSlaveOk(); - this.liveNodes = {master: master, slaves: [slave]}; - - return master; - }; - - this.getSecondaries = function() { - return [slave]; - }; - - this.awaitReplication = function() { - assert.commandWorked(master.adminCommand({fsyncUnlock: 1}), - 'failed to unlock the primary'); - - print('Starting fsync on master to flush all pending writes'); - assert.commandWorked(master.adminCommand({fsync: 1})); - print('fsync on master completed'); - - var timeout = 60 * 1000 * 5; // 5min timeout - var dbNames = master.getDBNames(); - print('Awaiting replication of inserts into ' + dbNames); - for (var dbName of dbNames) { - if (dbName === 'local') - continue; - assert.writeOK(master.getDB(dbName).await_repl.insert( - {awaiting: 'repl'}, {writeConcern: {w: 2, wtimeout: timeout}}), - 'Awaiting replication failed'); + load('jstests/libs/parallelTester.js'); + + function isMasterSlave(uri) { + const mongo = new Mongo(uri); + jsTest.authenticate(mongo); + const cmdLineOpts = mongo.getDB('admin').adminCommand('getCmdLineOpts'); + assert.commandWorked(cmdLineOpts); + return cmdLineOpts.parsed.master === true; + } + + function isMultiNodeReplSet(uri) { + const mongo = new Mongo(uri); + let hosts = []; + const isMaster = mongo.adminCommand({isMaster: 1}); + if (isMaster.hasOwnProperty('setName')) { + let hosts = isMaster.hosts; + if (isMaster.hasOwnProperty('passives')) { + hosts = hosts.concat(isMaster.passives); } - print('Finished awaiting replication'); - assert.commandWorked(master.adminCommand({fsync: 1, lock: 1}), - 'failed to re-lock the primary'); - }; - - this.checkReplicatedDataHashes = function() { - ReplSetTest({nodes: 0}).checkReplicatedDataHashes.apply(this, arguments); - }; - - this.checkReplicaSet = function() { - ReplSetTest({nodes: 0}).checkReplicaSet.apply(this, arguments); - }; - }; - - var startTime = Date.now(); + } + return hosts.length > 1; + } + + // Adds the uri and description (replset or master-slave) if server needs dbhash check. + function checkAndAddServerDesc(uri, out) { + // No need to check the dbhash of single node replsets. + if (isMultiNodeReplSet(uri)) { + out.push({type: 'replset', uri: uri}); + } else if (isMasterSlave(uri)) { + out.push({type: 'master-slave', uri: uri}); + } + } + + function checkReplDataHashThread(serverDesc, testData, excludedDBs) { + // A thin wrapper around master/slave nodes that provides the getHashes(), getPrimary(), + // awaitReplication(), and nodeList() methods. + // DEPRECATED: this wrapper only supports nodes started through resmoke's masterslave.py + // fixture. Please do not use it with other master/slave clusters. + function MasterSlaveDBHashTest(primaryHost) { + const master = new Mongo(primaryHost); + const masterPort = master.host.split(':')[1]; + const slave = new Mongo('localhost:' + String(parseInt(masterPort) + 1)); + + this.nodeList = function() { + return [master.host, slave.host]; + }; + + this.getHashes = function(db) { + const combinedRes = {}; + let res = master.getDB(db).runCommand('dbhash'); + assert.commandWorked(res); + combinedRes.master = res; + + res = slave.getDB(db).runCommand('dbhash'); + assert.commandWorked(res); + combinedRes.slaves = [res]; + + return combinedRes; + }; + + this.getPrimary = function() { + slave.setSlaveOk(); + this.liveNodes = {master: master, slaves: [slave]}; + return master; + }; + + this.getSecondaries = function() { + return [slave]; + }; + + this.awaitReplication = function() { + assert.commandWorked(master.adminCommand({fsyncUnlock: 1}), + 'failed to unlock the primary'); + + print('Starting fsync on master to flush all pending writes'); + assert.commandWorked(master.adminCommand({fsync: 1})); + print('fsync on master completed'); + + const kTimeout = 60 * 1000 * 5; // 5min timeout + const dbNames = master.getDBNames(); + print('Awaiting replication of inserts into ' + dbNames); + for (let dbName of dbNames) { + if (dbName === 'local') + continue; + assert.writeOK( + master.getDB(dbName).await_repl.insert( + {awaiting: 'repl'}, {writeConcern: {w: 2, wtimeout: kTimeout}}), + 'Awaiting replication failed'); + } + print('Finished awaiting replication'); + assert.commandWorked(master.adminCommand({fsync: 1, lock: 1}), + 'failed to re-lock the primary'); + }; + + this.checkReplicatedDataHashes = function() { + ReplSetTest({nodes: 0}).checkReplicatedDataHashes.apply(this, arguments); + }; + + this.checkReplicaSet = function() { + ReplSetTest({nodes: 0}).checkReplicaSet.apply(this, arguments); + }; + + this.dumpOplog = function() { + print('master-slave cannot dump oplog'); + }; + } + + TestData = testData; + + // Since UUIDs aren't explicitly replicated in master-slave deployments, we ignore the UUID + // in the output of the 'listCollections' command to avoid reporting a known data + // inconsistency issue from checkReplicatedDataHashes(). + const ignoreUUIDs = serverDesc.type === 'master-slave'; + let fixture = null; + if (serverDesc.type === 'replset') { + fixture = new ReplSetTest(serverDesc.uri); + } else if (serverDesc.type === 'master-slave') { + fixture = new MasterSlaveDBHashTest(serverDesc.uri); + } else { + throw 'unrecognized server type ' + serverDesc.type; + } + fixture.checkReplicatedDataHashes(undefined, excludedDBs, ignoreUUIDs); + } + + let startTime = Date.now(); assert.neq(typeof db, 'undefined', 'No `db` object, is the shell connected to a mongod?'); - var primaryInfo = db.isMaster(); + // stores each server type (master/slave or replset) and uri. + const serversNeedingReplDataHashCheck = []; + const primaryInfo = db.isMaster(); + const isMongos = primaryInfo.msg === 'isdbgrid'; + const isReplSet = primaryInfo.hasOwnProperty('setName'); + const uri = db.getMongo().host; assert(primaryInfo.ismaster, 'shell is not connected to the primary or master node: ' + tojson(primaryInfo)); - var cmdLineOpts = db.adminCommand('getCmdLineOpts'); - assert.commandWorked(cmdLineOpts); - var isMasterSlave = cmdLineOpts.parsed.master === true; - var testFixture = isMasterSlave ? new MasterSlaveDBHashTest(db.getMongo().host) - : new ReplSetTest(db.getMongo().host); - testFixture.checkReplicatedDataHashes(); - - var totalTime = Date.now() - startTime; + assert(isMongos || isReplSet || isMasterSlave(uri), + 'not replset, master/slave, or sharded cluster'); + + if (isMongos) { + // Add shards and config server if they are replica sets. + let res = db.adminCommand('getShardMap'); + assert.commandWorked(res); + const csURI = res.map.config; + res = db.adminCommand('listShards'); + assert.commandWorked(res); + const shardURIs = res.shards.map((shard) => shard.host); + + checkAndAddServerDesc(csURI, serversNeedingReplDataHashCheck); + shardURIs.forEach((shardURI) => { + checkAndAddServerDesc(shardURI, serversNeedingReplDataHashCheck); + }); + } else { + checkAndAddServerDesc(uri, serversNeedingReplDataHashCheck); + } + + const threads = []; + const excludedDBs = jsTest.options().excludedDBsFromDBHash || []; + serversNeedingReplDataHashCheck.forEach((serverDesc) => { + const thread = new ScopedThread(checkReplDataHashThread, serverDesc, TestData, excludedDBs); + threads.push({serverDesc: serverDesc, handle: thread}); + thread.start(); + }); + + if (serversNeedingReplDataHashCheck.length === 0) { + let skipReason = 'No multi-node replication detected in '; + if (isMongos) { + skipReason += 'sharded cluster'; + } else if (isReplSet) { + skipReason += 'replica set'; + } else { + skipReason += 'master-slave set'; + } + + print('Skipping consistency checks for cluster because ' + skipReason); + return; + } + + const failedChecks = []; + threads.forEach(thread => { + thread.handle.join(); + if (thread.handle.hasFailed()) { + failedChecks.push(thread.serverDesc.uri + ' (' + thread.serverDesc.type + ')'); + } + }); + + assert.eq(failedChecks.length, + 0, + 'dbhash check failed for the following hosts: ' + failedChecks.join(',')); + + const totalTime = Date.now() - startTime; print('Finished consistency checks of cluster in ' + totalTime + ' ms.'); })(); diff --git a/jstests/hooks/run_validate_collections.js b/jstests/hooks/run_validate_collections.js index d95c9eb5c9f..0fd699b8e0e 100644 --- a/jstests/hooks/run_validate_collections.js +++ b/jstests/hooks/run_validate_collections.js @@ -11,10 +11,15 @@ const res = conn.adminCommand({isMaster: 1}); const connections = []; - if (res.hasOwnProperty('hosts')) { + if (res.hasOwnProperty('setName')) { for (let hostString of res.hosts) { connections.push(new Mongo(hostString)); } + if (res.hasOwnProperty('passives')) { + for (let hostString of res.passives) { + connections.push(new Mongo(hostString)); + } + } } else { connections.push(conn); } |