SERVER-33068 Fix run_check_repl_dbhash.js hook to actually run dbhash.

Consolidates the logic in the run_check_repl_dbhash.js and
run_validate_collections.js for discovering all of the mongod processes
in a MongoDB deployment into a new discover_topology.js library.

Also adds a test that relies on mongod logging to verify that the
run_check_repl_dbhash.js and run_validate_collections.js hooks execute
on all of the expected servers.

2 files changed, 203 insertions, 237 deletions

diff --git a/jstests/hooks/run_check_repl_dbhash.js b/jstests/hooks/run_check_repl_dbhash.js
index 636266e44cb..06e626ba5cd 100644
--- a/jstests/hooks/run_check_repl_dbhash.js
+++ b/jstests/hooks/run_check_repl_dbhash.js
@@ -3,197 +3,200 @@
 'use strict';
 
 (function() {
-    load('jstests/libs/parallelTester.js');
+    load('jstests/libs/discover_topology.js');  // For Topology and DiscoverTopology.
+    load('jstests/libs/parallelTester.js');     // For ScopedThread.
+
+    // A thin wrapper around master/slave nodes that provides methods necessary for checking data
+    // consistency between the master and slave nodes.
+    //
+    // DEPRECATED: This wrapper is only intended to be used for the master-slave deployment started
+    // by resmoke.py as part of the master_slave_jscore_passthrough.yml test suite and it shouldn't
+    // be used for any other master/slave tests.
+    //
+    // TODO SERVER-32143: Remove this wrapper.
+    function MasterSlaveDBHashTest(primaryHost) {
+        const master = new Mongo(primaryHost);
+        const masterPort = master.host.split(':')[1];
+        const slave = new Mongo('localhost:' + String(parseInt(masterPort) + 1));
+
+        this.nodeList = function nodeList() {
+            return [master.host, slave.host];
+        };
+
+        this.getHashes = function getHashes(db) {
+            const combinedRes = {};
+            let res = master.getDB(db).runCommand('dbhash');
+            assert.commandWorked(res);
+            combinedRes.master = res;
+
+            res = slave.getDB(db).runCommand('dbhash');
+            assert.commandWorked(res);
+            combinedRes.slaves = [res];
+
+            return combinedRes;
+        };
+
+        this.getPrimary = function getPrimary() {
+            slave.setSlaveOk();
+            this.liveNodes = {master: master, slaves: [slave]};
+            return master;
+        };
+
+        this.getSecondaries = function getSecondaries() {
+            return [slave];
+        };
+
+        this.awaitReplication = function awaitReplication() {
+            assert.commandWorked(master.adminCommand({fsyncUnlock: 1}),
+                                 'failed to unlock the primary');
+
+            print('Starting fsync on master to flush all pending writes');
+            assert.commandWorked(master.adminCommand({fsync: 1}));
+            print('fsync on master completed');
+
+            const kTimeout = 5 * 60 * 1000;  // 5 minute timeout
+            const dbNames = master.getDBNames();
+
+            for (let dbName of dbNames) {
+                if (dbName === 'local') {
+                    continue;
+                }
 
-    function isMasterSlave(uri) {
-        const mongo = new Mongo(uri);
-        jsTest.authenticate(mongo);
-        const cmdLineOpts = mongo.getDB('admin').adminCommand('getCmdLineOpts');
-        assert.commandWorked(cmdLineOpts);
-        return cmdLineOpts.parsed.master === true;
+                print('Awaiting replication of inserts into ' + dbName);
+                assert.writeOK(master.getDB(dbName).await_repl.insert(
+                                   {awaiting: 'repl'}, {writeConcern: {w: 2, wtimeout: kTimeout}}),
+                               'Awaiting replication failed');
+            }
+            print('Finished awaiting replication');
+            assert.commandWorked(master.adminCommand({fsync: 1, lock: 1}),
+                                 'failed to re-lock the primary');
+        };
+
+        this.checkReplicatedDataHashes = function checkReplicatedDataHashes() {
+            const msgPrefix = 'checkReplicatedDataHashes for master-slave deployment';
+            const excludedDBs = jsTest.options().excludedDBsFromDBHash || [];
+
+            // Since UUIDs aren't explicitly replicated in master-slave deployments, we ignore the
+            // UUID in the output of the 'listCollections' command to avoid reporting a known data
+            // inconsistency issue from checkReplicatedDataHashes().
+            const ignoreUUIDs = true;
+
+            new ReplSetTest({
+                nodes: 0
+            }).checkReplicatedDataHashes.call(this, msgPrefix, excludedDBs, ignoreUUIDs);
+        };
+
+        this.checkReplicaSet = function checkReplicaSet() {
+            new ReplSetTest({nodes: 0}).checkReplicaSet.apply(this, arguments);
+        };
+
+        this.dumpOplog = function dumpOplog() {
+            print('Not dumping oplog for master-slave deployment');
+        };
     }
 
-    function isMultiNodeReplSet(uri) {
-        const mongo = new Mongo(uri);
-        let hosts = [];
-        const isMaster = mongo.adminCommand({isMaster: 1});
-        if (isMaster.hasOwnProperty('setName')) {
-            let hosts = isMaster.hosts;
-            if (isMaster.hasOwnProperty('passives')) {
-                hosts = hosts.concat(isMaster.passives);
-            }
-        }
-        return hosts.length > 1;
+    function isMasterSlaveDeployment(conn) {
+        const cmdLineOpts = assert.commandWorked(conn.adminCommand({getCmdLineOpts: 1}));
+        return cmdLineOpts.parsed.master === true;
     }
 
-    // Adds the uri and description (replset or master-slave) if server needs dbhash check.
-    function checkAndAddServerDesc(uri, out) {
-        // No need to check the dbhash of single node replsets.
-        if (isMultiNodeReplSet(uri)) {
-            out.push({type: 'replset', uri: uri});
-        } else if (isMasterSlave(uri)) {
-            out.push({type: 'master-slave', uri: uri});
+    function checkReplicatedDataHashesThread(hosts, testData) {
+        try {
+            TestData = testData;
+            new ReplSetTest(hosts[0]).checkReplicatedDataHashes();
+            return {ok: 1};
+        } catch (e) {
+            return {ok: 0, hosts: hosts, error: e.toString(), stack: e.stack};
         }
     }
 
-    function checkReplDataHashThread(serverDesc, testData, excludedDBs) {
-        // A thin wrapper around master/slave nodes that provides the getHashes(), getPrimary(),
-        // awaitReplication(), and nodeList() methods.
-        // DEPRECATED: this wrapper only supports nodes started through resmoke's masterslave.py
-        // fixture. Please do not use it with other master/slave clusters.
-        function MasterSlaveDBHashTest(primaryHost) {
-            const master = new Mongo(primaryHost);
-            const masterPort = master.host.split(':')[1];
-            const slave = new Mongo('localhost:' + String(parseInt(masterPort) + 1));
-
-            this.nodeList = function() {
-                return [master.host, slave.host];
-            };
-
-            this.getHashes = function(db) {
-                const combinedRes = {};
-                let res = master.getDB(db).runCommand('dbhash');
-                assert.commandWorked(res);
-                combinedRes.master = res;
-
-                res = slave.getDB(db).runCommand('dbhash');
-                assert.commandWorked(res);
-                combinedRes.slaves = [res];
-
-                return combinedRes;
-            };
-
-            this.getPrimary = function() {
-                slave.setSlaveOk();
-                this.liveNodes = {master: master, slaves: [slave]};
-                return master;
-            };
-
-            this.getSecondaries = function() {
-                return [slave];
-            };
-
-            this.awaitReplication = function() {
-                assert.commandWorked(master.adminCommand({fsyncUnlock: 1}),
-                                     'failed to unlock the primary');
-
-                print('Starting fsync on master to flush all pending writes');
-                assert.commandWorked(master.adminCommand({fsync: 1}));
-                print('fsync on master completed');
-
-                const kTimeout = 60 * 1000 * 5;  // 5min timeout
-                const dbNames = master.getDBNames();
-                print('Awaiting replication of inserts into ' + dbNames);
-                for (let dbName of dbNames) {
-                    if (dbName === 'local')
-                        continue;
-                    assert.writeOK(
-                        master.getDB(dbName).await_repl.insert(
-                            {awaiting: 'repl'}, {writeConcern: {w: 2, wtimeout: kTimeout}}),
-                        'Awaiting replication failed');
-                }
-                print('Finished awaiting replication');
-                assert.commandWorked(master.adminCommand({fsync: 1, lock: 1}),
-                                     'failed to re-lock the primary');
-            };
-
-            this.checkReplicatedDataHashes = function() {
-                ReplSetTest({nodes: 0}).checkReplicatedDataHashes.apply(this, ['test', [], true]);
-            };
-
-            this.checkReplicaSet = function() {
-                ReplSetTest({nodes: 0}).checkReplicaSet.apply(this, arguments);
-            };
-
-            this.dumpOplog = function() {
-                print('master-slave cannot dump oplog');
-            };
+    const startTime = Date.now();
+    assert.neq(typeof db, 'undefined', 'No `db` object, is the shell connected to a mongod?');
+
+    let skipped = false;
+    try {
+        const conn = db.getMongo();
+
+        if (isMasterSlaveDeployment(conn)) {
+            new MasterSlaveDBHashTest(conn.host).checkReplicatedDataHashes();
+            return;
         }
 
-        TestData = testData;
-
-        // Since UUIDs aren't explicitly replicated in master-slave deployments, we ignore the UUID
-        // in the output of the 'listCollections' command to avoid reporting a known data
-        // inconsistency issue from checkReplicatedDataHashes().
-        const ignoreUUIDs = serverDesc.type === 'master-slave';
-        let fixture = null;
-        if (serverDesc.type === 'replset') {
-            fixture = new ReplSetTest(serverDesc.uri);
-        } else if (serverDesc.type === 'master-slave') {
-            fixture = new MasterSlaveDBHashTest(serverDesc.uri);
-        } else {
-            throw 'unrecognized server type ' + serverDesc.type;
+        const topology = DiscoverTopology.findConnectedNodes(conn);
+
+        if (topology.type === Topology.kStandalone) {
+            print('Skipping data consistency checks for cluster because we are connected to a' +
+                  ' stand-alone mongod: ' + tojsononeline(topology));
+            skipped = true;
+            return;
         }
-        fixture.checkReplicatedDataHashes(undefined, excludedDBs, ignoreUUIDs);
-    }
 
-    let startTime = Date.now();
-    assert.neq(typeof db, 'undefined', 'No `db` object, is the shell connected to a mongod?');
+        if (topology.type === Topology.kReplicaSet) {
+            if (topology.nodes.length === 1) {
+                print('Skipping data consistency checks for cluster because we are connected to a' +
+                      ' 1-node replica set: ' + tojsononeline(topology));
+                skipped = true;
+                return;
+            }
 
-    // stores each server type (master/slave or replset) and uri.
-    const serversNeedingReplDataHashCheck = [];
-    const primaryInfo = db.isMaster();
-    const isMongos = primaryInfo.msg === 'isdbgrid';
-    const isReplSet = primaryInfo.hasOwnProperty('setName');
-    const uri = db.getMongo().host;
-
-    assert(primaryInfo.ismaster,
-           'shell is not connected to the primary or master node: ' + tojson(primaryInfo));
-
-    assert(isMongos || isReplSet || isMasterSlave(uri),
-           'not replset, master/slave, or sharded cluster');
-
-    if (isMongos) {
-        // Add shards and config server if they are replica sets.
-        let res = db.adminCommand('getShardMap');
-        assert.commandWorked(res);
-        const csURI = res.map.config;
-        res = db.adminCommand('listShards');
-        assert.commandWorked(res);
-        const shardURIs = res.shards.map((shard) => shard.host);
-
-        checkAndAddServerDesc(csURI, serversNeedingReplDataHashCheck);
-        shardURIs.forEach((shardURI) => {
-            checkAndAddServerDesc(shardURI, serversNeedingReplDataHashCheck);
-        });
-    } else {
-        checkAndAddServerDesc(uri, serversNeedingReplDataHashCheck);
-    }
+            new ReplSetTest(topology.nodes[0]).checkReplicatedDataHashes();
+            return;
+        }
 
-    const threads = [];
-    const excludedDBs = jsTest.options().excludedDBsFromDBHash || [];
-    serversNeedingReplDataHashCheck.forEach((serverDesc) => {
-        const thread = new ScopedThread(checkReplDataHashThread, serverDesc, TestData, excludedDBs);
-        threads.push({serverDesc: serverDesc, handle: thread});
-        thread.start();
-    });
-
-    if (serversNeedingReplDataHashCheck.length === 0) {
-        let skipReason = 'No multi-node replication detected in ';
-        if (isMongos) {
-            skipReason += 'sharded cluster';
-        } else if (isReplSet) {
-            skipReason += 'replica set';
-        } else {
-            skipReason += 'master-slave set';
+        if (topology.type !== Topology.kShardedCluster) {
+            throw new Error('Unrecognized topology format: ' + tojson(topology));
         }
 
-        print('Skipping consistency checks for cluster because ' + skipReason);
-        return;
-    }
+        const threads = [];
+        try {
+            if (topology.configsvr.nodes.length > 1) {
+                const thread = new ScopedThread(
+                    checkReplicatedDataHashesThread, topology.configsvr.nodes, TestData);
+                threads.push(thread);
+                thread.start();
+            } else {
+                print('Skipping data consistency checks for 1-node CSRS: ' +
+                      tojsononeline(topology));
+            }
 
-    const failedChecks = [];
-    threads.forEach(thread => {
-        thread.handle.join();
-        if (thread.handle.hasFailed()) {
-            failedChecks.push(thread.serverDesc.uri + ' (' + thread.serverDesc.type + ')');
-        }
-    });
+            for (let shardName of Object.keys(topology.shards)) {
+                const shard = topology.shards[shardName];
 
-    assert.eq(failedChecks.length,
-              0,
-              'dbhash check failed for the following hosts: ' + failedChecks.join(','));
+                if (shard.type === Topology.kStandalone) {
+                    print('Skipping data consistency checks for stand-alone shard: ' +
+                          tojsononeline(topology));
+                    continue;
+                }
 
-    const totalTime = Date.now() - startTime;
-    print('Finished consistency checks of cluster in ' + totalTime + ' ms.');
+                if (shard.type !== Topology.kReplicaSet) {
+                    throw new Error('Unrecognized topology format: ' + tojson(topology));
+                }
+
+                if (shard.nodes.length > 1) {
+                    const thread =
+                        new ScopedThread(checkReplicatedDataHashesThread, shard.nodes, TestData);
+                    threads.push(thread);
+                    thread.start();
+                } else {
+                    print('Skipping data consistency checks for 1-node replica set shard: ' +
+                          tojsononeline(topology));
+                }
+            }
+        } finally {
+            // Wait for each thread to finish. Throw an error if any thread fails.
+            const returnData = threads.map(thread => {
+                thread.join();
+                return thread.returnData();
+            });
+
+            returnData.forEach(res => {
+                assert.commandWorked(res, 'data consistency checks failed');
+            });
+        }
+    } finally {
+        if (!skipped) {
+            const totalTime = Date.now() - startTime;
+            print('Finished data consistency checks for cluster in ' + totalTime + ' ms.');
+        }
+    }
 })();
diff --git a/jstests/hooks/run_validate_collections.js b/jstests/hooks/run_validate_collections.js
index db0ca98d3b6..6f9b9be0581 100644
--- a/jstests/hooks/run_validate_collections.js
+++ b/jstests/hooks/run_validate_collections.js
@@ -3,73 +3,36 @@
 'use strict';
 
 (function() {
-    assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?');
-
-    function getConnectionStrings(conn) {
-        // If conn does not point to a repl set, then this function returns [conn].
-        const res = conn.adminCommand({isMaster: 1});
-        let hostList = [];
-
-        if (res.hasOwnProperty('setName')) {
-            hostList = res.hosts;
-            if (res.hasOwnProperty('passives')) {
-                hostList = hostList.concat(res.passives);
-            }
-            return hostList;
-        } else {
-            return [conn.host];
-        }
-    }
-
-    function getConfigConnStr() {
-        const shardMap = db.adminCommand({getShardMap: 1});
-        if (!shardMap.hasOwnProperty('map')) {
-            throw new Error('Expected getShardMap() to return an object a "map" field: ' +
-                            tojson(shardMap));
-        }
-
-        const map = shardMap.map;
-
-        if (!map.hasOwnProperty('config')) {
-            throw new Error('Expected getShardMap().map to have a "config" field: ' + tojson(map));
-        }
-
-        return map.config;
-    }
-
-    function isMongos() {
-        return db.isMaster().msg === 'isdbgrid';
-    }
-
-    function getHostList() {
-        let hostList = [];
-
-        if (isMongos()) {
-            // We're connected to a sharded cluster through a mongos.
-
-            // 1) Add all the config servers to the server list.
-            const configConnStr = getConfigConnStr();
-            const configServerReplSetConn = new Mongo(configConnStr);
-            hostList = getConnectionStrings(configServerReplSetConn);
-
-            // 2) Add shard members to the server list.
-            const configDB = db.getSiblingDB('config');
-            const cursor = configDB.shards.find();
+    load('jstests/libs/discover_topology.js');      // For Topology and DiscoverTopology.
+    load('jstests/hooks/validate_collections.js');  // For CollectionValidator.
 
-            while (cursor.hasNext()) {
-                const shard = cursor.next();
-                const shardReplSetConn = new Mongo(shard.host);
-                hostList.push(...getConnectionStrings(shardReplSetConn));
+    assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?');
+    const topology = DiscoverTopology.findConnectedNodes(db.getMongo());
+
+    const hostList = [];
+
+    if (topology.type === Topology.kStandalone) {
+        hostList.push(topology.mongod);
+    } else if (topology.type === Topology.kReplicaSet) {
+        hostList.push(...topology.nodes);
+    } else if (topology.type === Topology.kShardedCluster) {
+        hostList.push(...topology.configsvr.nodes);
+
+        for (let shardName of Object.keys(topology.shards)) {
+            const shard = topology.shards[shardName];
+
+            if (shard.type === Topology.kStandalone) {
+                hostList.push(shard.mongod);
+            } else if (shard.type === Topology.kReplicaSet) {
+                hostList.push(...shard.nodes);
+            } else {
+                throw new Error('Unrecognized topology format: ' + tojson(topology));
             }
-        } else {
-            // We're connected to a mongod.
-            hostList = getConnectionStrings(db.getMongo());
         }
-
-        return hostList;
+    } else {
+        throw new Error('Unrecognized topology format: ' + tojson(topology));
     }
 
-    load('jstests/hooks/validate_collections.js');  // For CollectionValidator.
-    new CollectionValidator().validateNodes(getHostList());
+    new CollectionValidator().validateNodes(hostList);
 
 })();