SERVER-21630 run CheckReplDBHash on csrs and shard replset

And run ValidateCollections hook on priority=0 secondaries.

(cherry picked from commit b9decc492c0ff942d5bcd6e8c799de70fa0839af)

8 files changed, 214 insertions, 91 deletions

diff --git a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml
index fa367432dc5..1cb834a63eb 100644
--- a/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/aggregation_sharded_collections_passthrough.yml
@@ -34,6 +34,7 @@ executor:
         readMode: commands
         eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js")
     hooks:
+    - class: CheckReplDBHash
     - class: ValidateCollections
     - class: CleanEveryN
       n: 20
diff --git a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml
index db7bc2b0c15..a0e8c16be9b 100644
--- a/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml
+++ b/buildscripts/resmokeconfig/suites/integration_tests_sharded.yml
@@ -8,6 +8,7 @@ executor:
   cpp_integration_test:
     config: {}
     hooks:
+    - class: CheckReplDBHash
     - class: ValidateCollections
     fixture:
       class: ShardedClusterFixture
diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml
index 8eb912aee14..9a8a02db6b9 100644
--- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml
+++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml
@@ -9,6 +9,11 @@ executor:
       shell_options:
         readMode: commands
     hooks:
+    - class: CheckReplDBHash
+      shell_options:
+        global_vars:
+          TestData:
+            skipValidationOnInvalidViewDefinitions: true
     - class: ValidateCollections
       shell_options:
         global_vars:
diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
index d0f205b8840..96674cfc20c 100644
--- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
@@ -74,6 +74,7 @@ executor:
         readMode: commands
         eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js")
     hooks:
+    - class: CheckReplDBHash
     - class: ValidateCollections
     - class: CleanEveryN
       n: 20
diff --git a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml
index 56b4854cde9..8879e39f143 100644
--- a/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_gle_auth_basics_passthrough.yml
@@ -2,6 +2,11 @@
 config_variables:
 - &keyFile jstests/libs/authTestsKey
 - &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
+- &authOptions
+  authenticationDatabase: admin
+  authenticationMechanism: SCRAM-SHA-1
+  password: *keyFileData
+  username: __system
 
 selector:
   js_test:
@@ -17,18 +22,21 @@ executor:
     config:
       shell_options:
         global_vars:
-          TestData:
+          TestData: &TestData
             auth: true
             authMechanism: SCRAM-SHA-1
             keyFile: *keyFile
             keyFileData: *keyFileData
         eval: jsTest.authenticate(db.getMongo())
-        authenticationDatabase: admin
-        authenticationMechanism: SCRAM-SHA-1
-        password: *keyFileData
-        username: __system
+        <<: *authOptions
         readMode: commands
-
+    hooks:
+    - class: CheckReplDBHash
+      shell_options:
+        global_vars:
+          TestData: *TestData
+          eval: jsTest.authenticate(db.getMongo())
+        <<: *authOptions
     fixture:
       class: ShardedClusterFixture
       mongos_options:
diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
index 620ca06e310..496bd265a58 100644
--- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
@@ -55,6 +55,7 @@ executor:
       shell_options:
         readMode: commands
     hooks:
+    - class: CheckReplDBHash
     - class: ValidateCollections
     - class: CleanEveryN
       n: 20
diff --git a/jstests/hooks/run_check_repl_dbhash.js b/jstests/hooks/run_check_repl_dbhash.js
index 632f4e25cfa..a17e407be0e 100644
--- a/jstests/hooks/run_check_repl_dbhash.js
+++ b/jstests/hooks/run_check_repl_dbhash.js
@@ -3,96 +3,197 @@
 'use strict';
 
 (function() {
-    // A thin wrapper around master/slave nodes that provides the getHashes(), getPrimary(),
-    // awaitReplication(), and nodeList() methods.
-    // DEPRECATED: this wrapper only supports nodes started through resmoke's masterslave.py
-    // fixture. Please do not use it with other master/slave clusters.
-    var MasterSlaveDBHashTest = function(primaryHost) {
-        var master = new Mongo(primaryHost);
-        var resolvedHost = getHostName();
-        var masterPort = master.host.split(':')[1];
-        // The 'host' property is modified manually because 'localhost' is used by default in a new
-        // Mongo() connection. We set the value to the real hostname because that is what the server
-        // uses.
-        master.host = resolvedHost + ':' + masterPort;
-
-        var slave = new Mongo(resolvedHost + ':' + String(parseInt(masterPort) + 1));
-
-        this.nodeList = function() {
-            return [master.host, slave.host];
-        };
-
-        this.getHashes = function(db) {
-            var combinedRes = {};
-            var res = master.getDB(db).runCommand("dbhash");
-            assert.commandWorked(res);
-            combinedRes.master = res;
-
-            res = slave.getDB(db).runCommand("dbhash");
-            assert.commandWorked(res);
-            combinedRes.slaves = [res];
-
-            return combinedRes;
-        };
-
-        this.getPrimary = function() {
-            slave.setSlaveOk();
-            this.liveNodes = {master: master, slaves: [slave]};
-
-            return master;
-        };
-
-        this.getSecondaries = function() {
-            return [slave];
-        };
-
-        this.awaitReplication = function() {
-            assert.commandWorked(master.adminCommand({fsyncUnlock: 1}),
-                                 'failed to unlock the primary');
-
-            print('Starting fsync on master to flush all pending writes');
-            assert.commandWorked(master.adminCommand({fsync: 1}));
-            print('fsync on master completed');
-
-            var timeout = 60 * 1000 * 5;  // 5min timeout
-            var dbNames = master.getDBNames();
-            print('Awaiting replication of inserts into ' + dbNames);
-            for (var dbName of dbNames) {
-                if (dbName === 'local')
-                    continue;
-                assert.writeOK(master.getDB(dbName).await_repl.insert(
-                                   {awaiting: 'repl'}, {writeConcern: {w: 2, wtimeout: timeout}}),
-                               'Awaiting replication failed');
+    load('jstests/libs/parallelTester.js');
+
+    function isMasterSlave(uri) {
+        const mongo = new Mongo(uri);
+        jsTest.authenticate(mongo);
+        const cmdLineOpts = mongo.getDB('admin').adminCommand('getCmdLineOpts');
+        assert.commandWorked(cmdLineOpts);
+        return cmdLineOpts.parsed.master === true;
+    }
+
+    function isMultiNodeReplSet(uri) {
+        const mongo = new Mongo(uri);
+        let hosts = [];
+        const isMaster = mongo.adminCommand({isMaster: 1});
+        if (isMaster.hasOwnProperty('setName')) {
+            let hosts = isMaster.hosts;
+            if (isMaster.hasOwnProperty('passives')) {
+                hosts = hosts.concat(isMaster.passives);
             }
-            print('Finished awaiting replication');
-            assert.commandWorked(master.adminCommand({fsync: 1, lock: 1}),
-                                 'failed to re-lock the primary');
-        };
-
-        this.checkReplicatedDataHashes = function() {
-            ReplSetTest({nodes: 0}).checkReplicatedDataHashes.apply(this, arguments);
-        };
-
-        this.checkReplicaSet = function() {
-            ReplSetTest({nodes: 0}).checkReplicaSet.apply(this, arguments);
-        };
-    };
-
-    var startTime = Date.now();
+        }
+        return hosts.length > 1;
+    }
+
+    // Adds the uri and description (replset or master-slave) if server needs dbhash check.
+    function checkAndAddServerDesc(uri, out) {
+        // No need to check the dbhash of single node replsets.
+        if (isMultiNodeReplSet(uri)) {
+            out.push({type: 'replset', uri: uri});
+        } else if (isMasterSlave(uri)) {
+            out.push({type: 'master-slave', uri: uri});
+        }
+    }
+
+    function checkReplDataHashThread(serverDesc, testData, excludedDBs) {
+        // A thin wrapper around master/slave nodes that provides the getHashes(), getPrimary(),
+        // awaitReplication(), and nodeList() methods.
+        // DEPRECATED: this wrapper only supports nodes started through resmoke's masterslave.py
+        // fixture. Please do not use it with other master/slave clusters.
+        function MasterSlaveDBHashTest(primaryHost) {
+            const master = new Mongo(primaryHost);
+            const masterPort = master.host.split(':')[1];
+            const slave = new Mongo('localhost:' + String(parseInt(masterPort) + 1));
+
+            this.nodeList = function() {
+                return [master.host, slave.host];
+            };
+
+            this.getHashes = function(db) {
+                const combinedRes = {};
+                let res = master.getDB(db).runCommand('dbhash');
+                assert.commandWorked(res);
+                combinedRes.master = res;
+
+                res = slave.getDB(db).runCommand('dbhash');
+                assert.commandWorked(res);
+                combinedRes.slaves = [res];
+
+                return combinedRes;
+            };
+
+            this.getPrimary = function() {
+                slave.setSlaveOk();
+                this.liveNodes = {master: master, slaves: [slave]};
+                return master;
+            };
+
+            this.getSecondaries = function() {
+                return [slave];
+            };
+
+            this.awaitReplication = function() {
+                assert.commandWorked(master.adminCommand({fsyncUnlock: 1}),
+                                     'failed to unlock the primary');
+
+                print('Starting fsync on master to flush all pending writes');
+                assert.commandWorked(master.adminCommand({fsync: 1}));
+                print('fsync on master completed');
+
+                const kTimeout = 60 * 1000 * 5;  // 5min timeout
+                const dbNames = master.getDBNames();
+                print('Awaiting replication of inserts into ' + dbNames);
+                for (let dbName of dbNames) {
+                    if (dbName === 'local')
+                        continue;
+                    assert.writeOK(
+                        master.getDB(dbName).await_repl.insert(
+                            {awaiting: 'repl'}, {writeConcern: {w: 2, wtimeout: kTimeout}}),
+                        'Awaiting replication failed');
+                }
+                print('Finished awaiting replication');
+                assert.commandWorked(master.adminCommand({fsync: 1, lock: 1}),
+                                     'failed to re-lock the primary');
+            };
+
+            this.checkReplicatedDataHashes = function() {
+                ReplSetTest({nodes: 0}).checkReplicatedDataHashes.apply(this, arguments);
+            };
+
+            this.checkReplicaSet = function() {
+                ReplSetTest({nodes: 0}).checkReplicaSet.apply(this, arguments);
+            };
+
+            this.dumpOplog = function() {
+                print('master-slave cannot dump oplog');
+            };
+        }
+
+        TestData = testData;
+
+        // Since UUIDs aren't explicitly replicated in master-slave deployments, we ignore the UUID
+        // in the output of the 'listCollections' command to avoid reporting a known data
+        // inconsistency issue from checkReplicatedDataHashes().
+        const ignoreUUIDs = serverDesc.type === 'master-slave';
+        let fixture = null;
+        if (serverDesc.type === 'replset') {
+            fixture = new ReplSetTest(serverDesc.uri);
+        } else if (serverDesc.type === 'master-slave') {
+            fixture = new MasterSlaveDBHashTest(serverDesc.uri);
+        } else {
+            throw 'unrecognized server type ' + serverDesc.type;
+        }
+        fixture.checkReplicatedDataHashes(undefined, excludedDBs, ignoreUUIDs);
+    }
+
+    let startTime = Date.now();
     assert.neq(typeof db, 'undefined', 'No `db` object, is the shell connected to a mongod?');
 
-    var primaryInfo = db.isMaster();
+    // stores each server type (master/slave or replset) and uri.
+    const serversNeedingReplDataHashCheck = [];
+    const primaryInfo = db.isMaster();
+    const isMongos = primaryInfo.msg === 'isdbgrid';
+    const isReplSet = primaryInfo.hasOwnProperty('setName');
+    const uri = db.getMongo().host;
 
     assert(primaryInfo.ismaster,
            'shell is not connected to the primary or master node: ' + tojson(primaryInfo));
 
-    var cmdLineOpts = db.adminCommand('getCmdLineOpts');
-    assert.commandWorked(cmdLineOpts);
-    var isMasterSlave = cmdLineOpts.parsed.master === true;
-    var testFixture = isMasterSlave ? new MasterSlaveDBHashTest(db.getMongo().host)
-                                    : new ReplSetTest(db.getMongo().host);
-    testFixture.checkReplicatedDataHashes();
-
-    var totalTime = Date.now() - startTime;
+    assert(isMongos || isReplSet || isMasterSlave(uri),
+           'not replset, master/slave, or sharded cluster');
+
+    if (isMongos) {
+        // Add shards and config server if they are replica sets.
+        let res = db.adminCommand('getShardMap');
+        assert.commandWorked(res);
+        const csURI = res.map.config;
+        res = db.adminCommand('listShards');
+        assert.commandWorked(res);
+        const shardURIs = res.shards.map((shard) => shard.host);
+
+        checkAndAddServerDesc(csURI, serversNeedingReplDataHashCheck);
+        shardURIs.forEach((shardURI) => {
+            checkAndAddServerDesc(shardURI, serversNeedingReplDataHashCheck);
+        });
+    } else {
+        checkAndAddServerDesc(uri, serversNeedingReplDataHashCheck);
+    }
+
+    const threads = [];
+    const excludedDBs = jsTest.options().excludedDBsFromDBHash || [];
+    serversNeedingReplDataHashCheck.forEach((serverDesc) => {
+        const thread = new ScopedThread(checkReplDataHashThread, serverDesc, TestData, excludedDBs);
+        threads.push({serverDesc: serverDesc, handle: thread});
+        thread.start();
+    });
+
+    if (serversNeedingReplDataHashCheck.length === 0) {
+        let skipReason = 'No multi-node replication detected in ';
+        if (isMongos) {
+            skipReason += 'sharded cluster';
+        } else if (isReplSet) {
+            skipReason += 'replica set';
+        } else {
+            skipReason += 'master-slave set';
+        }
+
+        print('Skipping consistency checks for cluster because ' + skipReason);
+        return;
+    }
+
+    const failedChecks = [];
+    threads.forEach(thread => {
+        thread.handle.join();
+        if (thread.handle.hasFailed()) {
+            failedChecks.push(thread.serverDesc.uri + ' (' + thread.serverDesc.type + ')');
+        }
+    });
+
+    assert.eq(failedChecks.length,
+              0,
+              'dbhash check failed for the following hosts: ' + failedChecks.join(','));
+
+    const totalTime = Date.now() - startTime;
     print('Finished consistency checks of cluster in ' + totalTime + ' ms.');
 })();
diff --git a/jstests/hooks/run_validate_collections.js b/jstests/hooks/run_validate_collections.js
index d95c9eb5c9f..0fd699b8e0e 100644
--- a/jstests/hooks/run_validate_collections.js
+++ b/jstests/hooks/run_validate_collections.js
@@ -11,10 +11,15 @@
         const res = conn.adminCommand({isMaster: 1});
         const connections = [];
 
-        if (res.hasOwnProperty('hosts')) {
+        if (res.hasOwnProperty('setName')) {
             for (let hostString of res.hosts) {
                 connections.push(new Mongo(hostString));
             }
+            if (res.hasOwnProperty('passives')) {
+                for (let hostString of res.passives) {
+                    connections.push(new Mongo(hostString));
+                }
+            }
         } else {
             connections.push(conn);
         }