SERVER-32852 Capture data files on failures of the concurrency suite

(cherry picked from commit cea72553a4a27e0fcc8fc3d925129783fdecb53c)

8 files changed, 28 insertions, 10 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency.yml b/buildscripts/resmokeconfig/suites/concurrency.yml
index 50c565fc566..4af6a5e31ee 100644
--- a/buildscripts/resmokeconfig/suites/concurrency.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency.yml
@@ -15,6 +15,7 @@ executor:
     archive:
       hooks:
         - ValidateCollections
+      tests: true
     config:
       shell_options:
         readMode: commands
diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_replication.yml
index 8c9d272caea..9a575e63356 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_replication.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication.yml
@@ -6,6 +6,8 @@ selector:
 # Concurrency tests that run against a replica set start one themselves.
 executor:
   js_test:
+    archive:
+      tests: true
     config:
       shell_options:
         nodb: ''
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
index b5bd8b7169e..0e219f2f26c 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
@@ -6,6 +6,8 @@ selector:
 # Concurrency tests that run against a sharded cluster start one themselves.
 executor:
   js_test:
+    archive:
+      tests: true
     config:
       shell_options:
         nodb: ''
diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py
index 221b54ecd35..ea7f889f754 100644
--- a/buildscripts/resmokelib/utils/archival.py
+++ b/buildscripts/resmokelib/utils/archival.py
@@ -322,7 +322,8 @@ class Archival(object):
         self._archive_file_worker.join(timeout=timeout)
         self.check_thread(self._archive_file_worker, False)
 
-        self.logger.info("Total tar/gzip archive time is %0.2f seconds", self.archive_time)
+        self.logger.info("Total tar/gzip archive time is %0.2f seconds, for %d file(s) %d MB",
+                         self.archive_time, self.num_files, self.size_mb)
 
     def files_archived_num(self):
         """ Returns the number of the archived files. """
diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js
index f3e2429f898..609211d7894 100644
--- a/jstests/concurrency/fsm_libs/cluster.js
+++ b/jstests/concurrency/fsm_libs/cluster.js
@@ -223,11 +223,11 @@ var Cluster = function(options) {
 
             conn = st.s;  // mongos
 
-            this.teardown = function teardown() {
+            this.teardown = function teardown(opts) {
                 options.teardownFunctions.mongod.forEach(this.executeOnMongodNodes);
                 options.teardownFunctions.mongos.forEach(this.executeOnMongosNodes);
 
-                st.stop();
+                st.stop(opts);
             };
 
             // Save all mongos and mongod connections
@@ -273,10 +273,9 @@ var Cluster = function(options) {
             conn = rst.getPrimary();
             replSets = [rst];
 
-            this.teardown = function teardown() {
+            this.teardown = function teardown(opts) {
                 options.teardownFunctions.mongod.forEach(this.executeOnMongodNodes);
-
-                rst.stopSet();
+                rst.stopSet(undefined, undefined, opts);
             };
 
             this._addReplicaSetConns(rst);
diff --git a/jstests/concurrency/fsm_libs/runner.js b/jstests/concurrency/fsm_libs/runner.js
index ec629245dad..f5d35f1dbf6 100644
--- a/jstests/concurrency/fsm_libs/runner.js
+++ b/jstests/concurrency/fsm_libs/runner.js
@@ -647,6 +647,7 @@ var runner = (function() {
         var bgCleanup = [];
         var errors = [];
         var configServerData = [];
+        let activeException = false;
 
         try {
             prepareCollections(bgWorkloads, bgContext, cluster, clusterOptions, executionOptions);
@@ -705,6 +706,9 @@ var runner = (function() {
                                      dbHashBlacklist,
                                      configServerData);
                 });
+            } catch (err) {
+                activeException = true;
+                throw err;
             } finally {
                 // Set a flag so background threads know to terminate.
                 bgThreadMgr.markAllForTermination();
@@ -732,8 +736,14 @@ var runner = (function() {
                 }
 
                 throwError(workloadErrors);
+            } catch (err) {
+                activeException = true;
+                throw err;
             } finally {
-                cluster.teardown();
+                // We preserve the data files when an FSM workload failed so that they can later be
+                // archived to S3.
+                const opts = activeException ? {noCleanData: true} : {};
+                cluster.teardown(opts);
             }
         }
     }
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 2516b2f8fac..1ba0cedf2ce 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -1766,7 +1766,7 @@ var ReplSetTest = function(opts) {
             return;
         }
 
-        if (_alldbpaths) {
+        if ((!opts || !opts.noCleanData) && _alldbpaths) {
             print("ReplSetTest stopSet deleting all dbpaths");
             for (var i = 0; i < _alldbpaths.length; i++) {
                 resetDbpath(_alldbpaths[i]);
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index 7a0fdae70f3..daca0de8f14 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -396,8 +396,11 @@ var ShardingTest = function(params) {
             }
         }
 
-        for (var i = 0; i < _alldbpaths.length; i++) {
-            resetDbpath(MongoRunner.dataPath + _alldbpaths[i]);
+        if (!opts || !opts.noCleanData) {
+            print("ShardingTest stop deleting all dbpaths");
+            for (var i = 0; i < _alldbpaths.length; i++) {
+                resetDbpath(MongoRunner.dataPath + _alldbpaths[i]);
+            }
         }
 
         var timeMillis = new Date().getTime() - _startTime.getTime();