SERVER-21214 Dump config server data when the sharded concurrency suites fail

2 files changed, 85 insertions, 22 deletions

diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js
index 3cabda56ded..625e6ed9cec 100644
--- a/jstests/concurrency/fsm_libs/cluster.js
+++ b/jstests/concurrency/fsm_libs/cluster.js
@@ -102,6 +102,7 @@ var Cluster = function(options) {
     var st;
 
     var initialized = false;
+    var clusterStartTime;
 
     var _conns = {
         mongos: [],
@@ -246,6 +247,7 @@ var Cluster = function(options) {
         }
 
         initialized = true;
+        clusterStartTime = new Date();
 
         options.setupFunctions.mongod.forEach(this.executeOnMongodNodes);
         if (options.sharded) {
@@ -262,10 +264,8 @@ var Cluster = function(options) {
     };
 
     this.executeOnMongodNodes = function executeOnMongodNodes(fn) {
-        if (!initialized) {
-            throw new Error('cluster must be initialized before functions can be executed ' +
-                            'against it');
-        }
+        assert(initialized, 'cluster must be initialized first');
+
         if (!fn || typeof(fn) !== 'function' || fn.length !== 1) {
             throw new Error('mongod function must be a function that takes a db as an argument');
         }
@@ -275,10 +275,8 @@ var Cluster = function(options) {
     };
 
     this.executeOnMongosNodes = function executeOnMongosNodes(fn) {
-        if (!initialized) {
-            throw new Error('cluster must be initialized before functions can be executed ' +
-                            'against it');
-        }
+        assert(initialized, 'cluster must be initialized first');
+
         if (!fn || typeof(fn) !== 'function' || fn.length !== 1) {
             throw new Error('mongos function must be a function that takes a db as an argument');
         }
@@ -288,21 +286,17 @@ var Cluster = function(options) {
     };
 
     this.teardown = function teardown() {
+        assert(initialized, 'cluster must be initialized first');
         options.teardownFunctions.mongod.forEach(this.executeOnMongodNodes);
     };
 
     this.getDB = function getDB(dbName) {
-        if (!initialized) {
-            throw new Error('cluster has not been initialized yet');
-        }
-
+        assert(initialized, 'cluster must be initialized first');
         return conn.getDB(dbName);
     };
 
     this.getHost = function getHost() {
-        if (!initialized) {
-            throw new Error('cluster has not been initialized yet');
-        }
+        assert(initialized, 'cluster must be initialized first');
 
         // Alternate mongos connections for sharded clusters
         if (this.isSharded()) {
@@ -325,6 +319,7 @@ var Cluster = function(options) {
     };
 
     this.shardCollection = function shardCollection() {
+        assert(initialized, 'cluster must be initialized first');
         assert(this.isSharded(), 'cluster is not sharded');
         st.shardColl.apply(st, arguments);
     };
@@ -358,6 +353,8 @@ var Cluster = function(options) {
     //      }
     // }
     this.getSerializedCluster = function getSerializedCluster() {
+        assert(initialized, 'cluster must be initialized first');
+
         // TODO: Add support for non-sharded clusters.
         if (!this.isSharded()) {
             return '';
@@ -405,11 +402,13 @@ var Cluster = function(options) {
     }
 
     this.startBalancer = function startBalancer() {
+        assert(initialized, 'cluster must be initialized first');
         assert(this.isSharded(), 'cluster is not sharded');
         st.startBalancer();
     };
 
     this.stopBalancer = function stopBalancer() {
+        assert(initialized, 'cluster must be initialized first');
         assert(this.isSharded(), 'cluster is not sharded');
         st.stopBalancer();
     };
@@ -419,6 +418,7 @@ var Cluster = function(options) {
     };
 
     this.awaitReplication = function awaitReplication(message) {
+        assert(initialized, 'cluster must be initialized first');
         if (this.isReplication()) {
             var wc = {
                 writeConcern: {
@@ -500,6 +500,35 @@ var Cluster = function(options) {
             }, this);
         }, this);
     };
+
+    this.recordConfigServerData = function recordConfigServerData(configServer) {
+        assert(initialized, 'cluster must be initialized first');
+        assert(this.isSharded(), 'cluster is not sharded');
+
+        var data = {};
+        var configDB = configServer.getDB('config');
+
+        // We record the contents of the 'lockpings' and 'locks' collections to make it easier to
+        // debug issues with distributed locks in the sharded cluster.
+        data.lockpings = configDB.lockpings.find({ ping: { $gte: clusterStartTime } }).toArray();
+
+        // We suppress some fields from the result set to reduce the amount of data recorded.
+        data.locks = configDB.locks.find({ when: { $gte: clusterStartTime } },
+                                         { process: 0, ts: 0 }).toArray();
+
+        return data;
+    };
+
+    this.recordAllConfigServerData = function recordAllConfigServerData() {
+        assert(initialized, 'cluster must be initialized first');
+        assert(this.isSharded(), 'cluster is not sharded');
+
+        var data = {};
+        st._configServers.forEach(config =>
+            (data[config.host] = this.recordConfigServerData(config)));
+
+        return data;
+    };
 };
 
 /**
diff --git a/jstests/concurrency/fsm_libs/runner.js b/jstests/concurrency/fsm_libs/runner.js
index 696794f8efa..cc765223eb7 100644
--- a/jstests/concurrency/fsm_libs/runner.js
+++ b/jstests/concurrency/fsm_libs/runner.js
@@ -448,9 +448,34 @@ var runner = (function() {
         return true;
     }
 
-    function runWorkloadGroup(threadMgr, workloads, context, cluster, clusterOptions,
-                              executionMode, executionOptions, errors, maxAllowedThreads,
-                              dbHashBlacklist) {
+    function recordConfigServerData(cluster, workloads, configServerData, errors) {
+        const CONFIG_DATA_LENGTH = 3;
+
+        if (cluster.isSharded()) {
+            var newData;
+            try {
+                newData = cluster.recordAllConfigServerData();
+            } catch (e) {
+                var failureType = 'Config Server Data Collection';
+                errors.push(new WorkloadFailure(e.toString(), e.stack, failureType));
+                return;
+            }
+
+            newData.previousWorkloads = workloads;
+            newData.time = (new Date()).toISOString();
+            configServerData.push(newData);
+
+            // Limit the amount of data recorded to avoid logging too much info when a test
+            // fails.
+            while (configServerData.length > CONFIG_DATA_LENGTH) {
+                configServerData.shift();
+            }
+        }
+    }
+
+    function runWorkloadGroup(threadMgr, workloads, context, cluster, clusterOptions, executionMode,
+                              executionOptions, errors, maxAllowedThreads, dbHashBlacklist,
+                              configServerData) {
         var cleanup = [];
         var teardownFailed = false;
         var startTime = Date.now(); // Initialize in case setupWorkload fails below.
@@ -501,6 +526,8 @@ var runner = (function() {
             totalTime = Date.now() - startTime;
             jsTest.log('Workload(s) completed in ' + totalTime + ' ms: ' +
                         workloads.join(' '));
+
+            recordConfigServerData(cluster, workloads, configServerData, errors);
         }
 
         // Only drop the collections/databases if all the workloads ran successfully.
@@ -591,6 +618,7 @@ var runner = (function() {
         Random.setRandomSeed(clusterOptions.seed);
         var bgCleanup = [];
         var errors = [];
+        var configServerData = [];
 
         try {
             prepareCollections(bgWorkloads, bgContext, cluster, clusterOptions, executionOptions);
@@ -637,9 +665,9 @@ var runner = (function() {
                     });
 
                     // Run the next group of workloads in the schedule.
-                    runWorkloadGroup(threadMgr, workloads, groupContext, cluster,
-                                     clusterOptions, executionMode, executionOptions,
-                                     errors, maxAllowedThreads, dbHashBlacklist);
+                    runWorkloadGroup(threadMgr, workloads, groupContext, cluster, clusterOptions,
+                                     executionMode, executionOptions, errors, maxAllowedThreads,
+                                     dbHashBlacklist, configServerData);
                 });
             } finally {
                 // Set a flag so background threads know to terminate.
@@ -660,7 +688,13 @@ var runner = (function() {
                 // the foreground and background workloads. IterationEnd errors are ignored because
                 // they are thrown when the background workloads are instructed by the thread
                 // manager to terminate.
-                throwError(errors.filter(e => (e.err.startsWith('IterationEnd:') === false)));
+                var workloadErrors = errors.filter(e => !e.err.startsWith('IterationEnd:'));
+
+                if (cluster.isSharded() && workloadErrors.length) {
+                    jsTest.log('Config Server Data:\n' + tojsononeline(configServerData));
+                }
+
+                throwError(workloadErrors);
             } finally {
                 cluster.teardown();
             }