SERVER-31441 Run validateCollections on each server in parallel

(cherry picked from commit b7b606259d712fcb7248f1c2f45b952ecb529654)

SERVER-31441: need to copy dependencies (ScopedThread) for parallel
validation in system_perf

(cherry picked from commit 11e68f03f22a040328f34fa636bd7b69a215d16b)

2 files changed, 47 insertions, 13 deletions

diff --git a/etc/system_perf.yml b/etc/system_perf.yml
index 137bc2b4a13..4bce0e93b23 100644
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@@ -264,8 +264,8 @@ tasks:
           mv mongos${extension|} mongodb/bin
           if [ -d jstests/hooks ]
           then
-            echo "Fetching JS test DB correctness checks from directory jstests/hooks"
-            cp -a jstests/hooks/* mongodb/jstests/hooks
+            echo "Fetching JS test DB correctness checks from directory jstests"
+            cp -a jstests/* mongodb/jstests
 
             echo "Now adding our own special run_validate_collections.js wrapper"
             mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js
diff --git a/jstests/hooks/run_validate_collections.js b/jstests/hooks/run_validate_collections.js
index 49131c9ebe9..d95c9eb5c9f 100644
--- a/jstests/hooks/run_validate_collections.js
+++ b/jstests/hooks/run_validate_collections.js
@@ -4,7 +4,7 @@
 
 (function() {
     assert.eq(typeof db, 'object', 'Invalid `db` object, is the shell connected to a mongod?');
-    load('jstests/hooks/validate_collections.js');  // For validateCollections.
+    load('jstests/libs/parallelTester.js');
 
     function getDirectConnections(conn) {
         // If conn does not point to a repl set, then this function returns [conn].
@@ -70,17 +70,51 @@
         return serverList;
     }
 
-    const serverList = getServerList();
-    for (let server of serverList) {
-        print('Running validate() on ' + server.host);
-        server.setSlaveOk();
-        jsTest.authenticate(server);
-
-        const dbNames = server.getDBNames();
-        for (let dbName of dbNames) {
-            if (!validateCollections(server.getDB(dbName), {full: true})) {
-                throw new Error('Collection validation failed');
+    // Run a separate thread to validate collections on each server in parallel.
+    var validateCollectionsThread = function(host, testData) {
+        load('jstests/hooks/validate_collections.js');  // For validateCollections.
+        TestData = testData;  // Pass the TestData object from main thread.
+
+        try {
+            print('Running validate() on ' + host);
+            const conn = new Mongo(host);
+            conn.setSlaveOk();
+            jsTest.authenticate(conn);
+
+            const dbNames = conn.getDBNames();
+            for (let dbName of dbNames) {
+                if (!validateCollections(conn.getDB(dbName), {full: true})) {
+                    return {ok: 0};
+                }
             }
+            return {ok: 1};
+        } catch (e) {
+            print('Exception caught in scoped thread running validationCollections on server: ' +
+                  host);
+            return {ok: 0, error: e.toString(), stack: e.stack};
         }
+    };
+
+    // We run the scoped threads in a try/finally block in case any thread throws an exception, in
+    // which case we want to still join all the threads.
+    let threads = [];
+    const serverList = getServerList();
+
+    try {
+        serverList.forEach(server => {
+            const thread = new ScopedThread(validateCollectionsThread, server.host, TestData);
+            threads.push(thread);
+            thread.start();
+        });
+    } finally {
+        // Wait for each thread to finish. Throw an error if any thread fails.
+        const returnData = threads.map(thread => {
+            thread.join();
+            return thread.returnData();
+        });
+
+        returnData.forEach(res => {
+            assert.commandWorked(res, 'Collection validation failed');
+        });
     }
 })();