SERVER-24759 SERVER-31128 Validate collections on all nodes in cluster

When connecting to a secondary to perform validate(), we now call
slaveOk() on the connection.

This also adds support for us to call validate() on all members of
replica set shards.

(cherry picked from commit 79da41f50a567ce1b0df5a4ab7a7eb5109414762)

2 files changed, 85 insertions, 27 deletions

diff --git a/buildscripts/resmokeconfig/suites/core_auth.yml b/buildscripts/resmokeconfig/suites/core_auth.yml
index 78cbad244e5..72c6c716927 100644
--- a/buildscripts/resmokeconfig/suites/core_auth.yml
+++ b/buildscripts/resmokeconfig/suites/core_auth.yml
@@ -2,6 +2,11 @@
 config_variables:
 - &keyFile jstests/libs/authTestsKey
 - &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
+- &authOptions
+  authenticationDatabase: local
+  authenticationMechanism: SCRAM-SHA-1
+  password: *keyFileData
+  username: __system
 
 selector:
   js_test:
@@ -16,18 +21,22 @@ executor:
     config:
       shell_options:
         global_vars:
-          TestData:
+          TestData: &TestData
             auth: true
             authMechanism: SCRAM-SHA-1
             keyFile: *keyFile
             keyFileData: *keyFileData
         eval: jsTest.authenticate(db.getMongo())
-        authenticationDatabase: local
-        authenticationMechanism: SCRAM-SHA-1
-        password: *keyFileData
-        username: __system
+        <<: *authOptions
         readMode: commands
     hooks:
+    - class: ValidateCollections
+      shell_options:
+        global_vars:
+          TestData:
+            <<: *TestData
+        eval: jsTest.authenticate(db.getMongo())
+        <<: *authOptions
     - class: CleanEveryN
       n: 20
     fixture:
diff --git a/jstests/hooks/run_validate_collections.js b/jstests/hooks/run_validate_collections.js
index 28198136774..49131c9ebe9 100644
--- a/jstests/hooks/run_validate_collections.js
+++ b/jstests/hooks/run_validate_collections.js
@@ -4,32 +4,81 @@
 
 (function() {
     assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?');
-    load('jstests/hooks/validate_collections.js');  // For validateCollections
-
-    var serverList = [];
-    serverList.push(db.getMongo());
-
-    var addSecondaryNodes = function() {
-        var cmdLineOpts = db.adminCommand('getCmdLineOpts');
-        assert.commandWorked(cmdLineOpts);
-
-        if (cmdLineOpts.parsed.hasOwnProperty('replication') &&
-            cmdLineOpts.parsed.replication.hasOwnProperty('replSet')) {
-            var rst = new ReplSetTest(db.getMongo().host);
-            // Call getPrimary to populate rst with information about the nodes.
-            var primary = rst.getPrimary();
-            assert(primary, 'calling getPrimary() failed');
-            serverList.push(...rst.getSecondaries());
+    load('jstests/hooks/validate_collections.js');  // For validateCollections.
+
+    function getDirectConnections(conn) {
+        // If conn does not point to a repl set, then this function returns [conn].
+        const res = conn.adminCommand({isMaster: 1});
+        const connections = [];
+
+        if (res.hasOwnProperty('hosts')) {
+            for (let hostString of res.hosts) {
+                connections.push(new Mongo(hostString));
+            }
+        } else {
+            connections.push(conn);
+        }
+
+        return connections;
+    }
+
+    function getConfigConnStr() {
+        const shardMap = db.adminCommand({getShardMap: 1});
+        if (!shardMap.hasOwnProperty('map')) {
+            throw new Error('Expected getShardMap() to return an object a "map" field: ' +
+                            tojson(shardMap));
+        }
+
+        const map = shardMap.map;
+
+        if (!map.hasOwnProperty('config')) {
+            throw new Error('Expected getShardMap().map to have a "config" field: ' + tojson(map));
         }
-    };
 
-    addSecondaryNodes();
+        return map.config;
+    }
 
-    for (var server of serverList) {
+    function isMongos() {
+        return db.isMaster().msg === 'isdbgrid';
+    }
+
+    function getServerList() {
+        const serverList = [];
+
+        if (isMongos()) {
+            // We're connected to a sharded cluster through a mongos.
+
+            // 1) Add all the config servers to the server list.
+            const configConnStr = getConfigConnStr();
+            const configServerReplSetConn = new Mongo(configConnStr);
+            serverList.push(...getDirectConnections(configServerReplSetConn));
+
+            // 2) Add shard members to the server list.
+            const configDB = db.getSiblingDB('config');
+            const cursor = configDB.shards.find();
+
+            while (cursor.hasNext()) {
+                const shard = cursor.next();
+                const shardReplSetConn = new Mongo(shard.host);
+                serverList.push(...getDirectConnections(shardReplSetConn));
+            }
+        } else {
+            // We're connected to a mongod.
+            serverList.push(...getDirectConnections(db.getMongo()));
+        }
+
+        return serverList;
+    }
+
+    const serverList = getServerList();
+    for (let server of serverList) {
         print('Running validate() on ' + server.host);
-        var dbNames = server.getDBNames();
-        for (var dbName of dbNames) {
-            if (!validateCollections(db.getSiblingDB(dbName), {full: true})) {
+        server.setSlaveOk();
+        jsTest.authenticate(server);
+
+        const dbNames = server.getDBNames();
+        for (let dbName of dbNames) {
+            if (!validateCollections(server.getDB(dbName), {full: true})) {
                 throw new Error('Collection validation failed');
             }
         }