From b04cc1e53b5d4b13d0b1f25303dd6f33b2e730f8 Mon Sep 17 00:00:00 2001 From: Jonathan Abrahams Date: Mon, 8 Oct 2018 17:32:36 -0400 Subject: SERVER-35852 Convert FSM client to use a resmoke suite for easy test blackisting in backup_restore --- .../concurrency_replication_for_backup_restore.yml | 46 ++++++++ buildscripts/resmokelib/core/programs.py | 15 ++- buildscripts/resmokelib/testing/executor.py | 7 +- .../resmokelib/testing/fixtures/__init__.py | 2 + .../resmokelib/testing/fixtures/external.py | 27 +++++ .../resmokelib/testing/fixtures/interface.py | 4 + .../testing/hooks/wait_for_replication.py | 35 ++++++ .../resmokelib/testing/testcases/jstest.py | 12 +- jstests/concurrency/fsm_libs/resmoke_runner.js | 48 ++++---- .../concurrency/fsm_workloads/create_database.js | 2 + .../fsm_workloads/create_index_background.js | 2 + .../create_index_background_unique.js | 2 + .../create_index_background_unique_capped.js | 2 +- .../create_index_background_wildcard.js | 2 + .../fsm_workloads/reindex_background.js | 2 +- .../concurrency/fsm_workloads/secondary_reads.js | 2 +- .../secondary_reads_with_catalog_changes.js | 2 +- .../snapshot_read_catalog_operations.js | 2 +- jstests/noPassthrough/libs/backup_restore.js | 127 ++++----------------- src/mongo/shell/replsettest.js | 2 +- 20 files changed, 201 insertions(+), 142 deletions(-) create mode 100644 buildscripts/resmokeconfig/suites/concurrency_replication_for_backup_restore.yml create mode 100644 buildscripts/resmokelib/testing/fixtures/external.py create mode 100644 buildscripts/resmokelib/testing/hooks/wait_for_replication.py diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_for_backup_restore.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_for_backup_restore.yml new file mode 100644 index 00000000000..535719d78bb --- /dev/null +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_for_backup_restore.yml @@ -0,0 +1,46 @@ +test_kind: fsm_workload_test + +# This suite connects to a running mongod replica set to test backup_restore. +# It must specify resmoke.py options '--shellConnPort' or '--shellConnString'. + +selector: + roots: + - jstests/concurrency/fsm_workloads/**/*.js + exclude_files: + # These workloads use >100MB of data, which can overwhelm test hosts. + - jstests/concurrency/fsm_workloads/agg_group_external.js + - jstests/concurrency/fsm_workloads/agg_sort_external.js + + # The findAndModify_update_grow.js workload can cause OOM kills on test hosts. + - jstests/concurrency/fsm_workloads/findAndModify_update_grow.js + + exclude_with_any_tags: + # The ability to shut down a node while its in the middle of applying ops is required for + # backup_restore. The workloads which perform certain operations which can cause the test to fail + # have been excluded. + # Background indexes can still be running when the mongod is shut down (SERVER-24639). + - creates_background_indexes + # Workloads that use a write concern will fail since the workload only connects to the primary. + - uses_write_concern + # Sharding is not supported for backup_restore. + - requires_sharding + # Transactions are not supported for backup_restore because the dbhash check can hang when + # running the collMod operation due to an idle transaction on a collection. + - uses_transactions + +executor: + config: + shell_options: + readMode: commands + global_vars: + TestData: + # The backup_restore*.js tests are expected to run the FSM workloads directly against the + # primary of the replica set without being aware of the other members. + discoverTopology: False + hooks: + - class: WaitForReplication + - class: CleanupConcurrencyWorkloads + exclude_dbs: + - crud + fixture: + class: ExternalFixture diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 2b3941ba165..4781b34061c 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -174,21 +174,25 @@ def mongo_shell_program( # pylint: disable=too-many-branches,too-many-locals,to The shell is started with the given connection string and arguments constructed from 'kwargs'. """ - connection_string = utils.default_if_none(config.SHELL_CONN_STRING, connection_string) - executable = utils.default_if_none(executable, config.DEFAULT_MONGO_EXECUTABLE) + executable = utils.default_if_none( + utils.default_if_none(executable, config.MONGO_EXECUTABLE), config.DEFAULT_MONGO_EXECUTABLE) args = [executable] eval_sb = [] # String builder. global_vars = kwargs.pop("global_vars", {}).copy() + if filename is not None: + test_name = os.path.splitext(os.path.basename(filename))[0] + else: + test_name = None shortcut_opts = { "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True), "noJournal": (config.NO_JOURNAL, False), "serviceExecutor": (config.SERVICE_EXECUTOR, ""), "storageEngine": (config.STORAGE_ENGINE, ""), "storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""), - "testName": (os.path.splitext(os.path.basename(filename))[0], ""), + "testName": (test_name, ""), "transportLayer": (config.TRANSPORT_LAYER, ""), "wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""), "wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""), @@ -286,8 +290,9 @@ def mongo_shell_program( # pylint: disable=too-many-branches,too-many-locals,to if connection_string is not None: args.append(connection_string) - # Have the mongos shell run the specified file. - args.append(filename) + # Have the mongo shell run the specified file. + if filename is not None: + args.append(filename) _set_keyfile_permissions(test_data) diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 7c218800502..aeb7f8b6f3c 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -35,8 +35,11 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes if _config.SHELL_CONN_STRING is not None: # Specifying the shellConnString command line option should override the fixture - # specified in the YAML configuration to be the no-op fixture. - self.fixture_config = {"class": fixtures.NOOP_FIXTURE_CLASS} + # specified in the YAML configuration to be the external fixture. + self.fixture_config = { + "class": fixtures.EXTERNAL_FIXTURE_CLASS, + "shell_conn_string": _config.SHELL_CONN_STRING + } else: self.fixture_config = fixture diff --git a/buildscripts/resmokelib/testing/fixtures/__init__.py b/buildscripts/resmokelib/testing/fixtures/__init__.py index 87662625f7f..1c592c8f6c7 100644 --- a/buildscripts/resmokelib/testing/fixtures/__init__.py +++ b/buildscripts/resmokelib/testing/fixtures/__init__.py @@ -2,10 +2,12 @@ from __future__ import absolute_import +from .external import ExternalFixture as _ExternalFixture from .interface import NoOpFixture as _NoOpFixture from .interface import make_fixture from ...utils import autoloader as _autoloader +EXTERNAL_FIXTURE_CLASS = _ExternalFixture.REGISTERED_NAME NOOP_FIXTURE_CLASS = _NoOpFixture.REGISTERED_NAME # We dynamically load all modules in the fixtures/ package so that any Fixture classes declared diff --git a/buildscripts/resmokelib/testing/fixtures/external.py b/buildscripts/resmokelib/testing/fixtures/external.py new file mode 100644 index 00000000000..fe95f94f844 --- /dev/null +++ b/buildscripts/resmokelib/testing/fixtures/external.py @@ -0,0 +1,27 @@ +"""External fixture for executing JSTests against.""" + +from __future__ import absolute_import + +from . import interface + + +class ExternalFixture(interface.Fixture): + """Fixture which provides JSTests capability to connect to external (non-resmoke) cluster.""" + + def __init__(self, logger, job_num, shell_conn_string=None): + """Initialize ExternalFixture.""" + interface.Fixture.__init__(self, logger, job_num) + + if shell_conn_string is None: + raise ValueError("The ExternalFixture must be specified with the resmoke option" + " --shellConnString or --shellPort") + + self.shell_conn_string = shell_conn_string + + def get_internal_connection_string(self): + """Return the internal connection string.""" + return self.shell_conn_string + + def get_driver_connection_url(self): + """Return the driver connection URL.""" + return "mongodb://" + self.get_internal_connection_string() diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index 9b4e69c1128..72ba03619b0 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -195,6 +195,10 @@ class NoOpFixture(Fixture): REGISTERED_NAME = "NoOpFixture" + def mongo_client(self, read_preference=None, timeout_millis=None): + """Return the mongo_client connection.""" + raise NotImplementedError("NoOpFixture does not support a mongo_client") + def get_internal_connection_string(self): """Return the internal connection string.""" return None diff --git a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py new file mode 100644 index 00000000000..b4f3bc1774f --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py @@ -0,0 +1,35 @@ +"""Test hook to wait for replication to complete on a replica set.""" + +from __future__ import absolute_import + +import time + +from buildscripts.resmokelib import core +from buildscripts.resmokelib import errors +from buildscripts.resmokelib.testing.hooks import interface + + +class WaitForReplication(interface.Hook): + """Wait for replication to complete.""" + + def __init__(self, hook_logger, fixture): + """Initialize WaitForReplication.""" + description = "WaitForReplication waits on a replica set" + interface.Hook.__init__(self, hook_logger, fixture, description) + + self.hook_logger = hook_logger + self.fixture = fixture + + def after_test(self, test, test_report): + """Run mongo shell to call replSetTest.awaitReplication().""" + start_time = time.time() + client_conn = self.fixture.get_internal_connection_string() + js_cmds = "conn = '{}'; rst = new ReplSetTest(conn); rst.awaitReplication();".format( + client_conn) + shell_options = {"nodb": "", "eval": js_cmds} + shell_proc = core.programs.mongo_shell_program(self.hook_logger, **shell_options) + shell_proc.start() + return_code = shell_proc.wait() + if return_code: + raise errors.ServerFailure("Awaiting replication failed for {}".format(client_conn)) + self.hook_logger.info("WaitForReplication took %0.4f seconds", time.time() - start_time) diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py index 3cb4ee0d505..4d36ac965b0 100644 --- a/buildscripts/resmokelib/testing/testcases/jstest.py +++ b/buildscripts/resmokelib/testing/testcases/jstest.py @@ -54,15 +54,17 @@ class _SingleJSTestCase(interface.ProcessTestCase): global_vars["MongoRunner.dataDir"] = data_dir global_vars["MongoRunner.dataPath"] = data_path - # Don't set the path to the executables when the user didn't specify them via the command - # line. The functions in the mongo shell for spawning processes have their own logic for - # determining the default path to use. + # Don't set the path to the mongod and mongos executables when the user didn't specify them + # via the command line. The functions in the mongo shell for spawning processes have their + # own logic for determining the default path to use. if config.MONGOD_EXECUTABLE is not None: global_vars["MongoRunner.mongodPath"] = config.MONGOD_EXECUTABLE if config.MONGOS_EXECUTABLE is not None: global_vars["MongoRunner.mongosPath"] = config.MONGOS_EXECUTABLE - if self.shell_executable is not None: - global_vars["MongoRunner.mongoShellPath"] = self.shell_executable + # We provide an absolute path for mongo shell to ensure that programs starting their own + # mongo shell will use the same as specified from resmoke.py. + global_vars["MongoRunner.mongoShellPath"] = os.path.abspath( + utils.default_if_none(self.shell_executable, config.DEFAULT_MONGO_EXECUTABLE)) test_data = global_vars.get("TestData", {}).copy() test_data["minPort"] = core.network.PortAllocator.min_test_port(self.fixture.job_num) diff --git a/jstests/concurrency/fsm_libs/resmoke_runner.js b/jstests/concurrency/fsm_libs/resmoke_runner.js index 409b588c4c6..f915c554076 100644 --- a/jstests/concurrency/fsm_libs/resmoke_runner.js +++ b/jstests/concurrency/fsm_libs/resmoke_runner.js @@ -202,27 +202,33 @@ sharded: {enabled: false}, }; - const topology = DiscoverTopology.findConnectedNodes(db.getMongo()); - - if (topology.type === Topology.kReplicaSet) { - clusterOptions.replication.enabled = true; - clusterOptions.replication.numNodes = topology.nodes.length; - } else if (topology.type === Topology.kShardedCluster) { - clusterOptions.replication.enabled = TestData.usingReplicaSetShards || false; - clusterOptions.sharded.enabled = true; - clusterOptions.sharded.enableAutoSplit = - TestData.hasOwnProperty('runningWithAutoSplit') ? TestData.runningWithAutoSplit : true; - clusterOptions.sharded.enableBalancer = - TestData.hasOwnProperty('runningWithBalancer') ? TestData.runningWithBalancer : true; - clusterOptions.sharded.numMongos = topology.mongos.nodes.length; - clusterOptions.sharded.numShards = Object.keys(topology.shards).length; - clusterOptions.sharded.stepdownOptions = {}; - clusterOptions.sharded.stepdownOptions.configStepdown = - TestData.runningWithConfigStepdowns || false; - clusterOptions.sharded.stepdownOptions.shardStepdown = - TestData.runningWithShardStepdowns || false; - } else if (topology.type !== Topology.kStandalone) { - throw new Error('Unrecognized topology format: ' + tojson(topology)); + // The TestData.discoverTopoloy is false when we only care about connecting to either a + // standalone or primary node in a replica set. + if (TestData.discoverTopology !== false) { + const topology = DiscoverTopology.findConnectedNodes(db.getMongo()); + + if (topology.type === Topology.kReplicaSet) { + clusterOptions.replication.enabled = true; + clusterOptions.replication.numNodes = topology.nodes.length; + } else if (topology.type === Topology.kShardedCluster) { + clusterOptions.replication.enabled = TestData.usingReplicaSetShards || false; + clusterOptions.sharded.enabled = true; + clusterOptions.sharded.enableAutoSplit = TestData.hasOwnProperty('runningWithAutoSplit') + ? TestData.runningWithAutoSplit + : true; + clusterOptions.sharded.enableBalancer = TestData.hasOwnProperty('runningWithBalancer') + ? TestData.runningWithBalancer + : true; + clusterOptions.sharded.numMongos = topology.mongos.nodes.length; + clusterOptions.sharded.numShards = Object.keys(topology.shards).length; + clusterOptions.sharded.stepdownOptions = {}; + clusterOptions.sharded.stepdownOptions.configStepdown = + TestData.runningWithConfigStepdowns || false; + clusterOptions.sharded.stepdownOptions.shardStepdown = + TestData.runningWithShardStepdowns || false; + } else if (topology.type !== Topology.kStandalone) { + throw new Error('Unrecognized topology format: ' + tojson(topology)); + } } clusterOptions.sameDB = TestData.sameDB; diff --git a/jstests/concurrency/fsm_workloads/create_database.js b/jstests/concurrency/fsm_workloads/create_database.js index 49992626c64..884c5d442df 100644 --- a/jstests/concurrency/fsm_workloads/create_database.js +++ b/jstests/concurrency/fsm_workloads/create_database.js @@ -8,6 +8,8 @@ * * Each thread uses its own database, though sometimes threads may try to create databases with * names that only differ in case, expecting the appriopriate error code. + * + * @tags: [creates_background_indexes] */ load('jstests/concurrency/fsm_workload_helpers/server_types.js'); // for isEphemeralForTest diff --git a/jstests/concurrency/fsm_workloads/create_index_background.js b/jstests/concurrency/fsm_workloads/create_index_background.js index 93bd48dea78..a89b6944cf4 100644 --- a/jstests/concurrency/fsm_workloads/create_index_background.js +++ b/jstests/concurrency/fsm_workloads/create_index_background.js @@ -7,6 +7,8 @@ * The command to create a background index completes in the shell once the * index has completed and the test no longer needs to execute more transitions. * The first thread (tid = 0) will be the one that creates the background index. + * + * @tags: [creates_background_indexes] */ load('jstests/concurrency/fsm_workload_helpers/server_types.js'); // for isMongos diff --git a/jstests/concurrency/fsm_workloads/create_index_background_unique.js b/jstests/concurrency/fsm_workloads/create_index_background_unique.js index 929ce18a4e2..7ea06d386f4 100644 --- a/jstests/concurrency/fsm_workloads/create_index_background_unique.js +++ b/jstests/concurrency/fsm_workloads/create_index_background_unique.js @@ -4,6 +4,8 @@ * create_index_background_unique.js * * Creates multiple unique background indexes in parallel. + * + * @tags: [creates_background_indexes] */ var $config = (function() { diff --git a/jstests/concurrency/fsm_workloads/create_index_background_unique_capped.js b/jstests/concurrency/fsm_workloads/create_index_background_unique_capped.js index 4b95e1d5b76..c6f8feb30fa 100644 --- a/jstests/concurrency/fsm_workloads/create_index_background_unique_capped.js +++ b/jstests/concurrency/fsm_workloads/create_index_background_unique_capped.js @@ -5,7 +5,7 @@ * * Creates multiple unique background indexes in parallel, on capped collections. * - * @tags: [requires_capped] + * @tags: [creates_background_indexes, requires_capped] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload diff --git a/jstests/concurrency/fsm_workloads/create_index_background_wildcard.js b/jstests/concurrency/fsm_workloads/create_index_background_wildcard.js index 5ad4f86e2d8..46f37adb3ee 100644 --- a/jstests/concurrency/fsm_workloads/create_index_background_wildcard.js +++ b/jstests/concurrency/fsm_workloads/create_index_background_wildcard.js @@ -2,6 +2,8 @@ /** * Executes the create_index_background.js workload, but with a wildcard index. + * + * @tags: [creates_background_indexes] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // For extendWorkload. load('jstests/concurrency/fsm_workloads/create_index_background.js'); // For $config. diff --git a/jstests/concurrency/fsm_workloads/reindex_background.js b/jstests/concurrency/fsm_workloads/reindex_background.js index 47e2d71e192..79aad331087 100644 --- a/jstests/concurrency/fsm_workloads/reindex_background.js +++ b/jstests/concurrency/fsm_workloads/reindex_background.js @@ -8,7 +8,7 @@ * that because indexes are initially built in the background, reindexing is also done in the * background. * - * @tags: [SERVER-32709] + * @tags: [SERVER-32709, creates_background_indexes] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload diff --git a/jstests/concurrency/fsm_workloads/secondary_reads.js b/jstests/concurrency/fsm_workloads/secondary_reads.js index 4c541b60204..4970013b56d 100644 --- a/jstests/concurrency/fsm_workloads/secondary_reads.js +++ b/jstests/concurrency/fsm_workloads/secondary_reads.js @@ -16,7 +16,7 @@ * For each read, we check if there is any 'hole' in the returned batch. There * should not be any 'hole' because oplogs are applied sequentially in batches. * - * @tags: [requires_replication] + * @tags: [requires_replication, uses_write_concern] */ var $config = (function() { diff --git a/jstests/concurrency/fsm_workloads/secondary_reads_with_catalog_changes.js b/jstests/concurrency/fsm_workloads/secondary_reads_with_catalog_changes.js index 35f0fa98f87..18218e00978 100644 --- a/jstests/concurrency/fsm_workloads/secondary_reads_with_catalog_changes.js +++ b/jstests/concurrency/fsm_workloads/secondary_reads_with_catalog_changes.js @@ -22,7 +22,7 @@ load('jstests/concurrency/fsm_workloads/secondary_reads.js'); // for $config * Note that index/collection drop could interrupt the reads, so we need to retry if the read is * interrupted. * - * @tags: [requires_replication] + * @tags: [creates_background_indexes, requires_replication, uses_write_concern] */ var $config = extendWorkload($config, function($config, $super) { diff --git a/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js b/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js index 7fa5da7e447..4c1b9ef5d9e 100644 --- a/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js +++ b/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js @@ -8,7 +8,7 @@ * finished. Additionally, index operations running concurrently with the snapshot read may cause * the read to fail with a SnapshotUnavailable error. * - * @tags: [uses_transactions] + * @tags: [creates_background_indexes, uses_transactions] */ load('jstests/concurrency/fsm_workload_helpers/snapshot_read_utils.js'); diff --git a/jstests/noPassthrough/libs/backup_restore.js b/jstests/noPassthrough/libs/backup_restore.js index 2de982f6883..8420526d1d7 100644 --- a/jstests/noPassthrough/libs/backup_restore.js +++ b/jstests/noPassthrough/libs/backup_restore.js @@ -109,7 +109,7 @@ var BackupRestoreTest = function(options) { // Returns the pid of the started mongo shell so the CRUD test client can be terminated // without waiting for its execution to finish. - return startMongoProgramNoConnect('mongo', + return startMongoProgramNoConnect(MongoRunner.mongoShellPath, '--eval', '(' + crudClientCmds + ')("' + dbName + '", "' + collectionName + '", ' + numNodes + ')', @@ -119,102 +119,16 @@ var BackupRestoreTest = function(options) { /** * Starts a client that will run a FSM workload. */ - function _fsmClient(host, blackListDb, numNodes) { + function _fsmClient(host) { // Launch FSM client - // SERVER-19488 The FSM framework assumes that there is an implicit 'db' connection when - // started without any cluster options. Since the shell running this test was started with - // --nodb, another mongo shell is used to allow implicit connections to be made to the - // primary of the replica set. - var fsmClientCmds = function(blackListDb, numNodes) { - 'use strict'; - load('jstests/concurrency/fsm_libs/runner.js'); - var dir = 'jstests/concurrency/fsm_workloads'; - var blacklist = [ - // Disabled due to MongoDB restrictions and/or workload restrictions - 'agg_group_external.js', // uses >100MB of data, which can overwhelm test hosts - 'agg_out_mode_insert_documents.js', - 'agg_out_mode_replace_documents.js', - 'agg_sort_external.js', // uses >100MB of data, which can overwhelm test hosts - 'agg_with_chunk_migrations.js', - 'auth_create_role.js', - 'auth_create_user.js', - 'auth_drop_role.js', - 'auth_drop_user.js', - 'create_index_background.js', - 'create_index_background_unique_capped.js', - 'create_index_background_unique.js', - 'database_versioning.js', - 'findAndModify_update_grow.js', // can cause OOM kills on test hosts - 'multi_statement_transaction_atomicity_isolation.js', - 'multi_statement_transaction_atomicity_isolation_multi_db.js', - 'multi_statement_transaction_atomicity_isolation_repeated_reads.js', - 'multi_statement_transaction_kill_sessions_atomicity_isolation.js', - 'multi_statement_transaction_atomicity_isolation_metrics_test.js', - 'multi_statement_transaction_simple.js', - 'multi_statement_transaction_simple_repeated_reads.js', - 'reindex_background.js', - 'remove_multiple_documents.js', - 'remove_where.js', - 'rename_capped_collection_chain.js', - 'rename_capped_collection_dbname_chain.js', - 'rename_capped_collection_dbname_droptarget.js', - 'rename_capped_collection_droptarget.js', - 'rename_collection_chain.js', - 'rename_collection_dbname_chain.js', - 'rename_collection_dbname_droptarget.js', - 'rename_collection_droptarget.js', - 'secondary_reads.js', - 'secondary_reads_with_catalog_changes.js', - 'sharded_base_partitioned.js', - 'sharded_mergeChunks_partitioned.js', - 'sharded_moveChunk_drop_shard_key_index.js', - 'sharded_moveChunk_partitioned.js', - 'sharded_splitChunk_partitioned.js', - 'snapshot_read_catalog_operations.js', - 'snapshot_read_kill_operations.js', - 'snapshot_read_kill_op_only.js', - 'update_rename.js', - 'update_rename_noindex.js', - 'yield_sort.js', - ].map(function(file) { - return dir + '/' + file; - }); - Random.setRandomSeed(); - // Run indefinitely. - while (true) { - try { - var workloads = Array.shuffle(ls(dir).filter(function(file) { - return !Array.contains(blacklist, file); - })); - // Run workloads one at a time, so we ensure replication completes. - workloads.forEach(function(workload) { - runWorkloadsSerially( - [workload], {}, {}, {dropDatabaseBlacklist: [blackListDb]}); - // Wait for replication to complete between workloads. - var wc = { - writeConcern: {w: numNodes, wtimeout: ReplSetTest.kDefaultTimeoutMS} - }; - var result = db.getSiblingDB('test').fsm_teardown.insert({a: 1}, wc); - assert.writeOK(result, 'teardown insert failed: ' + tojson(result)); - result = db.getSiblingDB('test').fsm_teardown.drop( - {writeConcern: {w: "majority"}}); - assert(result, 'teardown drop failed'); - }); - } catch (e) { - if (e instanceof ReferenceError || e instanceof TypeError) { - throw e; - } - } - } - }; - - // Returns the pid of the started mongo shell so the FSM test client can be terminated - // without waiting for its execution to finish. - return startMongoProgramNoConnect( - 'mongo', - '--eval', - '(' + fsmClientCmds + ')("' + blackListDb + '", ' + numNodes + ');', - host); + const suite = 'concurrency_replication_for_backup_restore'; + const resmokeCmd = 'python buildscripts/resmoke.py --shuffle --continueOnFailure' + + ' --repeat=99999 --mongo=' + MongoRunner.mongoShellPath + ' --shellConnString=' + host + + ' --suites=' + suite; + + // Returns the pid of the FSM test client so it can be terminated without waiting for its + // execution to finish. + return _startMongoProgram({args: resmokeCmd.split(' ')}); } /** @@ -278,7 +192,7 @@ var BackupRestoreTest = function(options) { var crudPid = _crudClient(primary.host, crudDb, crudColl, numNodes); // Launch FSM client - var fsmPid = _fsmClient(primary.host, crudDb, numNodes); + var fsmPid = _fsmClient(primary.host); // Let clients run for specified time before backing up secondary sleep(clientTime); @@ -429,13 +343,20 @@ var BackupRestoreTest = function(options) { fsmStatus.exitCode); stopMongoProgramByPid(fsmPid); - // Make sure the test database is not in a drop-pending state. This can happen if we - // killed the FSM client while it was in the middle of dropping it. - assert.soonNoExcept(function() { - let result = primary.getDB("test").afterClientKills.insert( - {"a": 1}, {writeConcern: {w: "majority"}}); + // Make sure the databases are not in a drop-pending state. This can happen if we + // killed the FSM client while it was in the middle of dropping them. + let result = primary.adminCommand({ + listDatabases: 1, + nameOnly: true, + filter: {'name': {$nin: ['admin', 'config', 'local', '$external']}} + }); + assert.commandWorked(result); + const databases = result.databases.map(dbs => dbs.name); + databases.forEach(dbName => assert.soonNoExcept(function() { + let result = primary.getDB(dbName).afterClientKills.insert( + {'a': 1}, {writeConcern: {w: 'majority'}}); return (result.nInserted === 1); - }, "failed to insert to test collection", 10 * 60 * 1000); + }, 'failed to insert to test collection', 10 * 60 * 1000)); // Wait up to 5 minutes until the new hidden node is in state SECONDARY. jsTestLog('CRUD and FSM clients stopped. Waiting for hidden node ' + hiddenHost + diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index aa572cc5f42..1d56ff18481 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -2618,7 +2618,7 @@ var ReplSetTest = function(opts) { // and too slowly processing heartbeats. When it steps down, it closes all of // its connections. _constructFromExistingSeedNode(opts); - }, 10); + }, 60); } else { _constructStartNewInstances(opts); } -- cgit v1.2.1