SERVER-35852 Convert FSM client to use a resmoke suite for easy test blackisting in backup_restore

20 files changed, 201 insertions, 142 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_for_backup_restore.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_for_backup_restore.yml
new file mode 100644
index 00000000000..535719d78bb
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication_for_backup_restore.yml
@@ -0,0 +1,46 @@
+test_kind: fsm_workload_test
+
+# This suite connects to a running mongod replica set to test backup_restore.
+# It must specify resmoke.py options '--shellConnPort' or '--shellConnString'.
+
+selector:
+  roots:
+  - jstests/concurrency/fsm_workloads/**/*.js
+  exclude_files:
+  # These workloads use >100MB of data, which can overwhelm test hosts.
+  - jstests/concurrency/fsm_workloads/agg_group_external.js
+  - jstests/concurrency/fsm_workloads/agg_sort_external.js
+
+  # The findAndModify_update_grow.js workload can cause OOM kills on test hosts.
+  - jstests/concurrency/fsm_workloads/findAndModify_update_grow.js
+
+  exclude_with_any_tags:
+  # The ability to shut down a node while its in the middle of applying ops is required for
+  # backup_restore. The workloads which perform certain operations which can cause the test to fail
+  # have been excluded.
+  # Background indexes can still be running when the mongod is shut down (SERVER-24639).
+  - creates_background_indexes
+  # Workloads that use a write concern will fail since the workload only connects to the primary.
+  - uses_write_concern
+  # Sharding is not supported for backup_restore.
+  - requires_sharding
+  # Transactions are not supported for backup_restore because the dbhash check can hang when
+  # running the collMod operation due to an idle transaction on a collection.
+  - uses_transactions
+
+executor:
+  config:
+    shell_options:
+      readMode: commands
+      global_vars:
+        TestData:
+          # The backup_restore*.js tests are expected to run the FSM workloads directly against the
+          # primary of the replica set without being aware of the other members.
+          discoverTopology: False
+  hooks:
+  - class: WaitForReplication
+  - class: CleanupConcurrencyWorkloads
+    exclude_dbs:
+      - crud
+  fixture:
+    class: ExternalFixture
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index 2b3941ba165..4781b34061c 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -174,21 +174,25 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
 
     The shell is started with the given connection string and arguments constructed from 'kwargs'.
     """
-    connection_string = utils.default_if_none(config.SHELL_CONN_STRING, connection_string)
 
-    executable = utils.default_if_none(executable, config.DEFAULT_MONGO_EXECUTABLE)
+    executable = utils.default_if_none(
+        utils.default_if_none(executable, config.MONGO_EXECUTABLE), config.DEFAULT_MONGO_EXECUTABLE)
     args = [executable]
 
     eval_sb = []  # String builder.
     global_vars = kwargs.pop("global_vars", {}).copy()
 
+    if filename is not None:
+        test_name = os.path.splitext(os.path.basename(filename))[0]
+    else:
+        test_name = None
     shortcut_opts = {
         "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True),
         "noJournal": (config.NO_JOURNAL, False),
         "serviceExecutor": (config.SERVICE_EXECUTOR, ""),
         "storageEngine": (config.STORAGE_ENGINE, ""),
         "storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""),
-        "testName": (os.path.splitext(os.path.basename(filename))[0], ""),
+        "testName": (test_name, ""),
         "transportLayer": (config.TRANSPORT_LAYER, ""),
         "wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""),
         "wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""),
@@ -286,8 +290,9 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
     if connection_string is not None:
         args.append(connection_string)
 
-    # Have the mongos shell run the specified file.
-    args.append(filename)
+    # Have the mongo shell run the specified file.
+    if filename is not None:
+        args.append(filename)
 
     _set_keyfile_permissions(test_data)
 
diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py
index 7c218800502..aeb7f8b6f3c 100644
--- a/buildscripts/resmokelib/testing/executor.py
+++ b/buildscripts/resmokelib/testing/executor.py
@@ -35,8 +35,11 @@ class TestSuiteExecutor(object):  # pylint: disable=too-many-instance-attributes
 
         if _config.SHELL_CONN_STRING is not None:
             # Specifying the shellConnString command line option should override the fixture
-            # specified in the YAML configuration to be the no-op fixture.
-            self.fixture_config = {"class": fixtures.NOOP_FIXTURE_CLASS}
+            # specified in the YAML configuration to be the external fixture.
+            self.fixture_config = {
+                "class": fixtures.EXTERNAL_FIXTURE_CLASS,
+                "shell_conn_string": _config.SHELL_CONN_STRING
+            }
         else:
             self.fixture_config = fixture
 
diff --git a/buildscripts/resmokelib/testing/fixtures/__init__.py b/buildscripts/resmokelib/testing/fixtures/__init__.py
index 87662625f7f..1c592c8f6c7 100644
--- a/buildscripts/resmokelib/testing/fixtures/__init__.py
+++ b/buildscripts/resmokelib/testing/fixtures/__init__.py
@@ -2,10 +2,12 @@
 
 from __future__ import absolute_import
 
+from .external import ExternalFixture as _ExternalFixture
 from .interface import NoOpFixture as _NoOpFixture
 from .interface import make_fixture
 from ...utils import autoloader as _autoloader
 
+EXTERNAL_FIXTURE_CLASS = _ExternalFixture.REGISTERED_NAME
 NOOP_FIXTURE_CLASS = _NoOpFixture.REGISTERED_NAME
 
 # We dynamically load all modules in the fixtures/ package so that any Fixture classes declared
diff --git a/buildscripts/resmokelib/testing/fixtures/external.py b/buildscripts/resmokelib/testing/fixtures/external.py
new file mode 100644
index 00000000000..fe95f94f844
--- /dev/null
+++ b/buildscripts/resmokelib/testing/fixtures/external.py
@@ -0,0 +1,27 @@
+"""External fixture for executing JSTests against."""
+
+from __future__ import absolute_import
+
+from . import interface
+
+
+class ExternalFixture(interface.Fixture):
+    """Fixture which provides JSTests capability to connect to external (non-resmoke) cluster."""
+
+    def __init__(self, logger, job_num, shell_conn_string=None):
+        """Initialize ExternalFixture."""
+        interface.Fixture.__init__(self, logger, job_num)
+
+        if shell_conn_string is None:
+            raise ValueError("The ExternalFixture must be specified with the resmoke option"
+                             " --shellConnString or --shellPort")
+
+        self.shell_conn_string = shell_conn_string
+
+    def get_internal_connection_string(self):
+        """Return the internal connection string."""
+        return self.shell_conn_string
+
+    def get_driver_connection_url(self):
+        """Return the driver connection URL."""
+        return "mongodb://" + self.get_internal_connection_string()
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index 9b4e69c1128..72ba03619b0 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -195,6 +195,10 @@ class NoOpFixture(Fixture):
 
     REGISTERED_NAME = "NoOpFixture"
 
+    def mongo_client(self, read_preference=None, timeout_millis=None):
+        """Return the mongo_client connection."""
+        raise NotImplementedError("NoOpFixture does not support a mongo_client")
+
     def get_internal_connection_string(self):
         """Return the internal connection string."""
         return None
diff --git a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py
new file mode 100644
index 00000000000..b4f3bc1774f
--- /dev/null
+++ b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py
@@ -0,0 +1,35 @@
+"""Test hook to wait for replication to complete on a replica set."""
+
+from __future__ import absolute_import
+
+import time
+
+from buildscripts.resmokelib import core
+from buildscripts.resmokelib import errors
+from buildscripts.resmokelib.testing.hooks import interface
+
+
+class WaitForReplication(interface.Hook):
+    """Wait for replication to complete."""
+
+    def __init__(self, hook_logger, fixture):
+        """Initialize WaitForReplication."""
+        description = "WaitForReplication waits on a replica set"
+        interface.Hook.__init__(self, hook_logger, fixture, description)
+
+        self.hook_logger = hook_logger
+        self.fixture = fixture
+
+    def after_test(self, test, test_report):
+        """Run mongo shell to call replSetTest.awaitReplication()."""
+        start_time = time.time()
+        client_conn = self.fixture.get_internal_connection_string()
+        js_cmds = "conn = '{}'; rst = new ReplSetTest(conn); rst.awaitReplication();".format(
+            client_conn)
+        shell_options = {"nodb": "", "eval": js_cmds}
+        shell_proc = core.programs.mongo_shell_program(self.hook_logger, **shell_options)
+        shell_proc.start()
+        return_code = shell_proc.wait()
+        if return_code:
+            raise errors.ServerFailure("Awaiting replication failed for {}".format(client_conn))
+        self.hook_logger.info("WaitForReplication took %0.4f seconds", time.time() - start_time)
diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py
index 3cb4ee0d505..4d36ac965b0 100644
--- a/buildscripts/resmokelib/testing/testcases/jstest.py
+++ b/buildscripts/resmokelib/testing/testcases/jstest.py
@@ -54,15 +54,17 @@ class _SingleJSTestCase(interface.ProcessTestCase):
         global_vars["MongoRunner.dataDir"] = data_dir
         global_vars["MongoRunner.dataPath"] = data_path
 
-        # Don't set the path to the executables when the user didn't specify them via the command
-        # line. The functions in the mongo shell for spawning processes have their own logic for
-        # determining the default path to use.
+        # Don't set the path to the mongod and mongos executables when the user didn't specify them
+        # via the command line. The functions in the mongo shell for spawning processes have their
+        # own logic for determining the default path to use.
         if config.MONGOD_EXECUTABLE is not None:
             global_vars["MongoRunner.mongodPath"] = config.MONGOD_EXECUTABLE
         if config.MONGOS_EXECUTABLE is not None:
             global_vars["MongoRunner.mongosPath"] = config.MONGOS_EXECUTABLE
-        if self.shell_executable is not None:
-            global_vars["MongoRunner.mongoShellPath"] = self.shell_executable
+        # We provide an absolute path for mongo shell to ensure that programs starting their own
+        # mongo shell will use the same as specified from resmoke.py.
+        global_vars["MongoRunner.mongoShellPath"] = os.path.abspath(
+            utils.default_if_none(self.shell_executable, config.DEFAULT_MONGO_EXECUTABLE))
 
         test_data = global_vars.get("TestData", {}).copy()
         test_data["minPort"] = core.network.PortAllocator.min_test_port(self.fixture.job_num)
diff --git a/jstests/concurrency/fsm_libs/resmoke_runner.js b/jstests/concurrency/fsm_libs/resmoke_runner.js
index 409b588c4c6..f915c554076 100644
--- a/jstests/concurrency/fsm_libs/resmoke_runner.js
+++ b/jstests/concurrency/fsm_libs/resmoke_runner.js
@@ -202,27 +202,33 @@
         sharded: {enabled: false},
     };
 
-    const topology = DiscoverTopology.findConnectedNodes(db.getMongo());
-
-    if (topology.type === Topology.kReplicaSet) {
-        clusterOptions.replication.enabled = true;
-        clusterOptions.replication.numNodes = topology.nodes.length;
-    } else if (topology.type === Topology.kShardedCluster) {
-        clusterOptions.replication.enabled = TestData.usingReplicaSetShards || false;
-        clusterOptions.sharded.enabled = true;
-        clusterOptions.sharded.enableAutoSplit =
-            TestData.hasOwnProperty('runningWithAutoSplit') ? TestData.runningWithAutoSplit : true;
-        clusterOptions.sharded.enableBalancer =
-            TestData.hasOwnProperty('runningWithBalancer') ? TestData.runningWithBalancer : true;
-        clusterOptions.sharded.numMongos = topology.mongos.nodes.length;
-        clusterOptions.sharded.numShards = Object.keys(topology.shards).length;
-        clusterOptions.sharded.stepdownOptions = {};
-        clusterOptions.sharded.stepdownOptions.configStepdown =
-            TestData.runningWithConfigStepdowns || false;
-        clusterOptions.sharded.stepdownOptions.shardStepdown =
-            TestData.runningWithShardStepdowns || false;
-    } else if (topology.type !== Topology.kStandalone) {
-        throw new Error('Unrecognized topology format: ' + tojson(topology));
+    // The TestData.discoverTopoloy is false when we only care about connecting to either a
+    // standalone or primary node in a replica set.
+    if (TestData.discoverTopology !== false) {
+        const topology = DiscoverTopology.findConnectedNodes(db.getMongo());
+
+        if (topology.type === Topology.kReplicaSet) {
+            clusterOptions.replication.enabled = true;
+            clusterOptions.replication.numNodes = topology.nodes.length;
+        } else if (topology.type === Topology.kShardedCluster) {
+            clusterOptions.replication.enabled = TestData.usingReplicaSetShards || false;
+            clusterOptions.sharded.enabled = true;
+            clusterOptions.sharded.enableAutoSplit = TestData.hasOwnProperty('runningWithAutoSplit')
+                ? TestData.runningWithAutoSplit
+                : true;
+            clusterOptions.sharded.enableBalancer = TestData.hasOwnProperty('runningWithBalancer')
+                ? TestData.runningWithBalancer
+                : true;
+            clusterOptions.sharded.numMongos = topology.mongos.nodes.length;
+            clusterOptions.sharded.numShards = Object.keys(topology.shards).length;
+            clusterOptions.sharded.stepdownOptions = {};
+            clusterOptions.sharded.stepdownOptions.configStepdown =
+                TestData.runningWithConfigStepdowns || false;
+            clusterOptions.sharded.stepdownOptions.shardStepdown =
+                TestData.runningWithShardStepdowns || false;
+        } else if (topology.type !== Topology.kStandalone) {
+            throw new Error('Unrecognized topology format: ' + tojson(topology));
+        }
     }
 
     clusterOptions.sameDB = TestData.sameDB;
diff --git a/jstests/concurrency/fsm_workloads/create_database.js b/jstests/concurrency/fsm_workloads/create_database.js
index 49992626c64..884c5d442df 100644
--- a/jstests/concurrency/fsm_workloads/create_database.js
+++ b/jstests/concurrency/fsm_workloads/create_database.js
@@ -8,6 +8,8 @@
  *
  * Each thread uses its own database, though sometimes threads may try to create databases with
  * names that only differ in case, expecting the appriopriate error code.
+ *
+ * @tags: [creates_background_indexes]
  */
 
 load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isEphemeralForTest
diff --git a/jstests/concurrency/fsm_workloads/create_index_background.js b/jstests/concurrency/fsm_workloads/create_index_background.js
index 93bd48dea78..a89b6944cf4 100644
--- a/jstests/concurrency/fsm_workloads/create_index_background.js
+++ b/jstests/concurrency/fsm_workloads/create_index_background.js
@@ -7,6 +7,8 @@
  * The command to create a background index completes in the shell once the
  * index has completed and the test no longer needs to execute more transitions.
  * The first thread (tid = 0) will be the one that creates the background index.
+ *
+ * @tags: [creates_background_indexes]
  */
 load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongos
 
diff --git a/jstests/concurrency/fsm_workloads/create_index_background_unique.js b/jstests/concurrency/fsm_workloads/create_index_background_unique.js
index 929ce18a4e2..7ea06d386f4 100644
--- a/jstests/concurrency/fsm_workloads/create_index_background_unique.js
+++ b/jstests/concurrency/fsm_workloads/create_index_background_unique.js
@@ -4,6 +4,8 @@
  * create_index_background_unique.js
  *
  * Creates multiple unique background indexes in parallel.
+ *
+ * @tags: [creates_background_indexes]
  */
 
 var $config = (function() {
diff --git a/jstests/concurrency/fsm_workloads/create_index_background_unique_capped.js b/jstests/concurrency/fsm_workloads/create_index_background_unique_capped.js
index 4b95e1d5b76..c6f8feb30fa 100644
--- a/jstests/concurrency/fsm_workloads/create_index_background_unique_capped.js
+++ b/jstests/concurrency/fsm_workloads/create_index_background_unique_capped.js
@@ -5,7 +5,7 @@
  *
  * Creates multiple unique background indexes in parallel, on capped collections.
  *
- * @tags: [requires_capped]
+ * @tags: [creates_background_indexes, requires_capped]
  */
 
 load('jstests/concurrency/fsm_libs/extend_workload.js');                      // for extendWorkload
diff --git a/jstests/concurrency/fsm_workloads/create_index_background_wildcard.js b/jstests/concurrency/fsm_workloads/create_index_background_wildcard.js
index 5ad4f86e2d8..46f37adb3ee 100644
--- a/jstests/concurrency/fsm_workloads/create_index_background_wildcard.js
+++ b/jstests/concurrency/fsm_workloads/create_index_background_wildcard.js
@@ -2,6 +2,8 @@
 
 /**
  * Executes the create_index_background.js workload, but with a wildcard index.
+ *
+ * @tags: [creates_background_indexes]
  */
 load('jstests/concurrency/fsm_libs/extend_workload.js');               // For extendWorkload.
 load('jstests/concurrency/fsm_workloads/create_index_background.js');  // For $config.
diff --git a/jstests/concurrency/fsm_workloads/reindex_background.js b/jstests/concurrency/fsm_workloads/reindex_background.js
index 47e2d71e192..79aad331087 100644
--- a/jstests/concurrency/fsm_workloads/reindex_background.js
+++ b/jstests/concurrency/fsm_workloads/reindex_background.js
@@ -8,7 +8,7 @@
  * that because indexes are initially built in the background, reindexing is also done in the
  * background.
  *
- * @tags: [SERVER-32709]
+ * @tags: [SERVER-32709, creates_background_indexes]
  */
 
 load('jstests/concurrency/fsm_libs/extend_workload.js');  // for extendWorkload
diff --git a/jstests/concurrency/fsm_workloads/secondary_reads.js b/jstests/concurrency/fsm_workloads/secondary_reads.js
index 4c541b60204..4970013b56d 100644
--- a/jstests/concurrency/fsm_workloads/secondary_reads.js
+++ b/jstests/concurrency/fsm_workloads/secondary_reads.js
@@ -16,7 +16,7 @@
  * For each read, we check if there is any 'hole' in the returned batch. There
  * should not be any 'hole' because oplogs are applied sequentially in batches.
  *
- * @tags: [requires_replication]
+ * @tags: [requires_replication, uses_write_concern]
  */
 
 var $config = (function() {
diff --git a/jstests/concurrency/fsm_workloads/secondary_reads_with_catalog_changes.js b/jstests/concurrency/fsm_workloads/secondary_reads_with_catalog_changes.js
index 35f0fa98f87..18218e00978 100644
--- a/jstests/concurrency/fsm_workloads/secondary_reads_with_catalog_changes.js
+++ b/jstests/concurrency/fsm_workloads/secondary_reads_with_catalog_changes.js
@@ -22,7 +22,7 @@ load('jstests/concurrency/fsm_workloads/secondary_reads.js');  // for $config
  * Note that index/collection drop could interrupt the reads, so we need to retry if the read is
  * interrupted.
  *
- * @tags: [requires_replication]
+ * @tags: [creates_background_indexes, requires_replication, uses_write_concern]
  */
 var $config = extendWorkload($config, function($config, $super) {
 
diff --git a/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js b/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js
index 7fa5da7e447..4c1b9ef5d9e 100644
--- a/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js
+++ b/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js
@@ -8,7 +8,7 @@
  * finished. Additionally, index operations running concurrently with the snapshot read may cause
  * the read to fail with a SnapshotUnavailable error.
  *
- * @tags: [uses_transactions]
+ * @tags: [creates_background_indexes, uses_transactions]
  */
 
 load('jstests/concurrency/fsm_workload_helpers/snapshot_read_utils.js');
diff --git a/jstests/noPassthrough/libs/backup_restore.js b/jstests/noPassthrough/libs/backup_restore.js
index 2de982f6883..8420526d1d7 100644
--- a/jstests/noPassthrough/libs/backup_restore.js
+++ b/jstests/noPassthrough/libs/backup_restore.js
@@ -109,7 +109,7 @@ var BackupRestoreTest = function(options) {
 
         // Returns the pid of the started mongo shell so the CRUD test client can be terminated
         // without waiting for its execution to finish.
-        return startMongoProgramNoConnect('mongo',
+        return startMongoProgramNoConnect(MongoRunner.mongoShellPath,
                                           '--eval',
                                           '(' + crudClientCmds + ')("' + dbName + '", "' +
                                               collectionName + '", ' + numNodes + ')',
@@ -119,102 +119,16 @@ var BackupRestoreTest = function(options) {
     /**
      * Starts a client that will run a FSM workload.
      */
-    function _fsmClient(host, blackListDb, numNodes) {
+    function _fsmClient(host) {
         // Launch FSM client
-        // SERVER-19488 The FSM framework assumes that there is an implicit 'db' connection when
-        // started without any cluster options. Since the shell running this test was started with
-        // --nodb, another mongo shell is used to allow implicit connections to be made to the
-        // primary of the replica set.
-        var fsmClientCmds = function(blackListDb, numNodes) {
-            'use strict';
-            load('jstests/concurrency/fsm_libs/runner.js');
-            var dir = 'jstests/concurrency/fsm_workloads';
-            var blacklist = [
-                // Disabled due to MongoDB restrictions and/or workload restrictions
-                'agg_group_external.js',  // uses >100MB of data, which can overwhelm test hosts
-                'agg_out_mode_insert_documents.js',
-                'agg_out_mode_replace_documents.js',
-                'agg_sort_external.js',  // uses >100MB of data, which can overwhelm test hosts
-                'agg_with_chunk_migrations.js',
-                'auth_create_role.js',
-                'auth_create_user.js',
-                'auth_drop_role.js',
-                'auth_drop_user.js',
-                'create_index_background.js',
-                'create_index_background_unique_capped.js',
-                'create_index_background_unique.js',
-                'database_versioning.js',
-                'findAndModify_update_grow.js',  // can cause OOM kills on test hosts
-                'multi_statement_transaction_atomicity_isolation.js',
-                'multi_statement_transaction_atomicity_isolation_multi_db.js',
-                'multi_statement_transaction_atomicity_isolation_repeated_reads.js',
-                'multi_statement_transaction_kill_sessions_atomicity_isolation.js',
-                'multi_statement_transaction_atomicity_isolation_metrics_test.js',
-                'multi_statement_transaction_simple.js',
-                'multi_statement_transaction_simple_repeated_reads.js',
-                'reindex_background.js',
-                'remove_multiple_documents.js',
-                'remove_where.js',
-                'rename_capped_collection_chain.js',
-                'rename_capped_collection_dbname_chain.js',
-                'rename_capped_collection_dbname_droptarget.js',
-                'rename_capped_collection_droptarget.js',
-                'rename_collection_chain.js',
-                'rename_collection_dbname_chain.js',
-                'rename_collection_dbname_droptarget.js',
-                'rename_collection_droptarget.js',
-                'secondary_reads.js',
-                'secondary_reads_with_catalog_changes.js',
-                'sharded_base_partitioned.js',
-                'sharded_mergeChunks_partitioned.js',
-                'sharded_moveChunk_drop_shard_key_index.js',
-                'sharded_moveChunk_partitioned.js',
-                'sharded_splitChunk_partitioned.js',
-                'snapshot_read_catalog_operations.js',
-                'snapshot_read_kill_operations.js',
-                'snapshot_read_kill_op_only.js',
-                'update_rename.js',
-                'update_rename_noindex.js',
-                'yield_sort.js',
-            ].map(function(file) {
-                return dir + '/' + file;
-            });
-            Random.setRandomSeed();
-            // Run indefinitely.
-            while (true) {
-                try {
-                    var workloads = Array.shuffle(ls(dir).filter(function(file) {
-                        return !Array.contains(blacklist, file);
-                    }));
-                    // Run workloads one at a time, so we ensure replication completes.
-                    workloads.forEach(function(workload) {
-                        runWorkloadsSerially(
-                            [workload], {}, {}, {dropDatabaseBlacklist: [blackListDb]});
-                        // Wait for replication to complete between workloads.
-                        var wc = {
-                            writeConcern: {w: numNodes, wtimeout: ReplSetTest.kDefaultTimeoutMS}
-                        };
-                        var result = db.getSiblingDB('test').fsm_teardown.insert({a: 1}, wc);
-                        assert.writeOK(result, 'teardown insert failed: ' + tojson(result));
-                        result = db.getSiblingDB('test').fsm_teardown.drop(
-                            {writeConcern: {w: "majority"}});
-                        assert(result, 'teardown drop failed');
-                    });
-                } catch (e) {
-                    if (e instanceof ReferenceError || e instanceof TypeError) {
-                        throw e;
-                    }
-                }
-            }
-        };
-
-        // Returns the pid of the started mongo shell so the FSM test client can be terminated
-        // without waiting for its execution to finish.
-        return startMongoProgramNoConnect(
-            'mongo',
-            '--eval',
-            '(' + fsmClientCmds + ')("' + blackListDb + '", ' + numNodes + ');',
-            host);
+        const suite = 'concurrency_replication_for_backup_restore';
+        const resmokeCmd = 'python buildscripts/resmoke.py --shuffle --continueOnFailure' +
+            ' --repeat=99999 --mongo=' + MongoRunner.mongoShellPath + ' --shellConnString=' + host +
+            ' --suites=' + suite;
+
+        // Returns the pid of the FSM test client so it can be terminated without waiting for its
+        // execution to finish.
+        return _startMongoProgram({args: resmokeCmd.split(' ')});
     }
 
     /**
@@ -278,7 +192,7 @@ var BackupRestoreTest = function(options) {
         var crudPid = _crudClient(primary.host, crudDb, crudColl, numNodes);
 
         // Launch FSM client
-        var fsmPid = _fsmClient(primary.host, crudDb, numNodes);
+        var fsmPid = _fsmClient(primary.host);
 
         // Let clients run for specified time before backing up secondary
         sleep(clientTime);
@@ -429,13 +343,20 @@ var BackupRestoreTest = function(options) {
                    fsmStatus.exitCode);
         stopMongoProgramByPid(fsmPid);
 
-        // Make sure the test database is not in a drop-pending state. This can happen if we
-        // killed the FSM client while it was in the middle of dropping it.
-        assert.soonNoExcept(function() {
-            let result = primary.getDB("test").afterClientKills.insert(
-                {"a": 1}, {writeConcern: {w: "majority"}});
+        // Make sure the databases are not in a drop-pending state. This can happen if we
+        // killed the FSM client while it was in the middle of dropping them.
+        let result = primary.adminCommand({
+            listDatabases: 1,
+            nameOnly: true,
+            filter: {'name': {$nin: ['admin', 'config', 'local', '$external']}}
+        });
+        assert.commandWorked(result);
+        const databases = result.databases.map(dbs => dbs.name);
+        databases.forEach(dbName => assert.soonNoExcept(function() {
+            let result = primary.getDB(dbName).afterClientKills.insert(
+                {'a': 1}, {writeConcern: {w: 'majority'}});
             return (result.nInserted === 1);
-        }, "failed to insert to test collection", 10 * 60 * 1000);
+        }, 'failed to insert to test collection', 10 * 60 * 1000));
 
         // Wait up to 5 minutes until the new hidden node is in state SECONDARY.
         jsTestLog('CRUD and FSM clients stopped. Waiting for hidden node ' + hiddenHost +
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index aa572cc5f42..1d56ff18481 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -2618,7 +2618,7 @@ var ReplSetTest = function(opts) {
             // and too slowly processing heartbeats. When it steps down, it closes all of
             // its connections.
             _constructFromExistingSeedNode(opts);
-        }, 10);
+        }, 60);
     } else {
         _constructStartNewInstances(opts);
     }