diff options
author | Louis Williams <louis.williams@mongodb.com> | 2023-04-26 09:40:43 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-04-26 10:33:28 +0000 |
commit | 75a095075298ed6344b9f288c2063d19e42630fb (patch) | |
tree | f2620930cc1902ed2bbb1c1d560c5ee2271e5423 /buildscripts | |
parent | bc26bc0fcb01ffb24bba056c5625d09a47985fb3 (diff) | |
download | mongo-75a095075298ed6344b9f288c2063d19e42630fb.tar.gz |
SERVER-70759 Fuzz sharding parameters in config_fuzzer
Diffstat (limited to 'buildscripts')
17 files changed, 94 insertions, 937 deletions
diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_100ms_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_100ms_refresh_jscore_passthrough.yml deleted file mode 100644 index 02b9116d197..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_100ms_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,58 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - exclude_files: - # The set_param1.js test attempts to compare the response from running the {getParameter: "*"} - # command multiple times, which may observe the change to the "transactionLifetimeLimitSeconds" - # server parameter. - - jstests/core/**/set_param1.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal - # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same - # time, the logical session cache refresh thread will flush these sessions to disk, creating more - # opLog entries. To avoid this infinite loop, we will denylist the test from this suite. - - jstests/core/**/awaitdata_getmore_cmd.js - # These tests verify that an expected number of update operations were tracked in the server - # status metrics, but the logical session cache refresh causes additional updates to be recorded. - - jstests/core/**/find_and_modify_metrics.js - - jstests/core/**/update_metrics.js - - exclude_with_any_tags: - - assumes_standalone_mongod - -executor: - archive: - hooks: - - CheckReplDBHashInBackground - - CheckReplDBHash - - CheckReplOplogs - - ValidateCollections - config: - shell_options: - eval: "testingReplication = true;" - hooks: - # The CheckReplDBHash hook waits until all operations have replicated to and have been applied - # on the secondaries, so we run the ValidateCollections hook after it to ensure we're - # validating the entire contents of the collection. - - class: CheckReplDBHashInBackground - - class: CheckReplOplogs - - class: CheckReplDBHash - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ReplicaSetFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 100 - num_nodes: 3 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_10sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_10sec_refresh_jscore_passthrough.yml deleted file mode 100644 index 1cb8d42ce2d..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_10sec_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,58 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - exclude_files: - # The set_param1.js test attempts to compare the response from running the {getParameter: "*"} - # command multiple times, which may observe the change to the "transactionLifetimeLimitSeconds" - # server parameter. - - jstests/core/**/set_param1.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal - # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same - # time, the logical session cache refresh thread will flush these sessions to disk, creating more - # opLog entries. To avoid this infinite loop, we will denylist the test from this suite. - - jstests/core/**/awaitdata_getmore_cmd.js - # These tests verify that an expected number of update operations were tracked in the server - # status metrics, but the logical session cache refresh causes additional updates to be recorded. - - jstests/core/**/find_and_modify_metrics.js - - jstests/core/**/update_metrics.js - - exclude_with_any_tags: - - assumes_standalone_mongod - -executor: - archive: - hooks: - - CheckReplDBHashInBackground - - CheckReplDBHash - - CheckReplOplogs - - ValidateCollections - config: - shell_options: - eval: "testingReplication = true;" - hooks: - # The CheckReplDBHash hook waits until all operations have replicated to and have been applied - # on the secondaries, so we run the ValidateCollections hook after it to ensure we're - # validating the entire contents of the collection. - - class: CheckReplDBHashInBackground - - class: CheckReplOplogs - - class: CheckReplDBHash - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ReplicaSetFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 10000 - num_nodes: 3 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_1sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_1sec_refresh_jscore_passthrough.yml deleted file mode 100644 index 95e276ed112..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_1sec_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,58 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - exclude_files: - # The set_param1.js test attempts to compare the response from running the {getParameter: "*"} - # command multiple times, which may observe the change to the "transactionLifetimeLimitSeconds" - # server parameter. - - jstests/core/**/set_param1.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal - # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same - # time, the logical session cache refresh thread will flush these sessions to disk, creating more - # opLog entries. To avoid this infinite loop, we will denylist the test from this suite. - - jstests/core/**/awaitdata_getmore_cmd.js - # These tests verify that an expected number of update operations were tracked in the server - # status metrics, but the logical session cache refresh causes additional updates to be recorded. - - jstests/core/**/find_and_modify_metrics.js - - jstests/core/**/update_metrics.js - - exclude_with_any_tags: - - assumes_standalone_mongod - -executor: - archive: - hooks: - - CheckReplDBHashInBackground - - CheckReplDBHash - - CheckReplOplogs - - ValidateCollections - config: - shell_options: - eval: "testingReplication = true;" - hooks: - # The CheckReplDBHash hook waits until all operations have replicated to and have been applied - # on the secondaries, so we run the ValidateCollections hook after it to ensure we're - # validating the entire contents of the collection. - - class: CheckReplDBHashInBackground - - class: CheckReplOplogs - - class: CheckReplDBHash - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ReplicaSetFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 1000 - num_nodes: 3 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml deleted file mode 100644 index a8434414a22..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_replication_default_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,59 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - exclude_files: - # The set_param1.js test attempts to compare the response from running the {getParameter: "*"} - # command multiple times, which may observe the change to the "transactionLifetimeLimitSeconds" - # server parameter. - - jstests/core/**/set_param1.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # The awaitdata_getmore_cmd.js test tails the oplog and waits for the getMore batch size to equal - # zero. The CheckReplDBHashInBackground hook consistently runs and creates sessions. At the same - # time, the logical session cache refresh thread will flush these sessions to disk, creating more - # opLog entries. To prevent this infinite loop, we will denylist the test from this suite. - - jstests/core/**/awaitdata_getmore_cmd.js - # These tests verify that an expected number of update operations were tracked in the server - # status metrics, but the logical session cache refresh causes additional updates to be recorded. - - jstests/core/**/find_and_modify_metrics.js - - jstests/core/**/update_metrics.js - - exclude_with_any_tags: - - assumes_standalone_mongod - -executor: - archive: - hooks: - - CheckReplDBHashInBackground - - ValidateCollectionsInBackground - - CheckReplDBHash - - CheckReplOplogs - - ValidateCollections - config: - shell_options: - eval: "testingReplication = true;" - hooks: - # The CheckReplDBHash hook waits until all operations have replicated to and have been applied - # on the secondaries, so we run the ValidateCollections hook after it to ensure we're - # validating the entire contents of the collection. - - class: CheckReplDBHashInBackground - - class: ValidateCollectionsInBackground - - class: CheckReplOplogs - - class: CheckReplDBHash - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ReplicaSetFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - num_nodes: 3 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_passthrough.yml deleted file mode 100644 index 0e9f1c4d1a0..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,116 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - exclude_files: - # These tests are run in logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml. - - jstests/core/txns/**/*.js - # The following tests fail because a certain command or functionality is not supported by - # mongos. This command or functionality is placed in a comment next to the failing test. - - jstests/core/**/apitest_db.js # serverStatus output doesn't have storageEngine. - - jstests/core/**/awaitdata_getmore_cmd.js # capped collections. - - jstests/core/**/bypass_doc_validation.js # sharded $out output not permitted - - jstests/core/**/check_shard_index.js # checkShardingIndex. - - jstests/core/**/collection_truncate.js # emptycapped. - - jstests/core/**/compact_keeps_indexes.js # compact. - - jstests/core/**/currentop.js # uses fsync. - - jstests/core/**/dbhash.js # dbhash. - - jstests/core/**/dbhash2.js # dbhash. - - jstests/core/**/fsync.js # uses fsync. - - jstests/core/**/geo_s2cursorlimitskip.js # profiling. - - jstests/core/**/geo_update_btree2.js # notablescan. - - jstests/core/**/index9.js # "local" database. - - jstests/core/**/queryoptimizera.js # "local" database. - - jstests/core/**/stages*.js # stageDebug. - - jstests/core/**/startup_log.js # "local" database. - - jstests/core/**/tailable_cursor_invalidation.js # capped collections. - - jstests/core/**/tailable_getmore_batch_size.js # capped collections. - - jstests/core/**/tailable_skip_limit.js # capped collections. - - jstests/core/**/top.js # top. - # The following tests fail because mongos behaves differently from mongod when testing certain - # functionality. The differences are in a comment next to the failing test. - - jstests/core/**/explain_missing_database.js # Behavior with no db different on mongos, SERVER-18047. - - jstests/core/**/geo_2d_explain.js # executionSuccess in different spot in explain(). - - jstests/core/**/geo_s2explain.js # inputStage in different spot in explain(). - - jstests/core/**/geo_s2sparse.js # keysPerIndex in different spot in validate(). - - jstests/core/**/operation_latency_histogram.js # Stats are counted differently on mongos, SERVER-24880. - # The following tests fail because they count indexes. These counts do not take into account the - # additional hashed shard key indexes that are automatically added by this passthrough. - - jstests/core/**/apitest_dbcollection.js - - jstests/core/**/bad_index_plugin.js - - jstests/core/**/create_indexes.js - - jstests/core/**/list_indexes_non_existent_ns.js - - jstests/core/**/mr_preserve_indexes.js - # The following tests fail because they expect no databases to be created. However a DB is created - # automatically when we shard a collection. - - jstests/core/**/dbcase.js - - jstests/core/**/dbcase2.js - - jstests/core/**/no_db_created.js - - jstests/core/**/killop_drop_collection.js # Uses fsyncLock. - # These tests fail because sharded clusters do not clean up correctly after failed index builds. - # See SERVER-33207 as an example. - - jstests/core/**/geo_borders.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded - # queries with a limit or for distinct commands. - - jstests/core/**/distinct_index1.js - - jstests/core/**/explain1.js - - jstests/core/**/explain4.js - - jstests/core/**/sortk.js - # TODO: Remove after fixing SERVER-32563. The implementation of explain for the count command is - # incorrect on sharded collections. - - jstests/core/**/explain_count.js - - jstests/core/**/explain_server_params.js - # TODO SERVER-32311: These tests use plan stage helpers which can't handle sharded explain output. - - jstests/core/**/expr_index_use.js - - jstests/core/**/index_multikey.js - - jstests/core/**/optimized_match_explain.js - - jstests/core/**/sort_array.js - - exclude_with_any_tags: - - assumes_standalone_mongod - - assumes_against_mongod_not_mongos - # Tests tagged with the following will fail because they assume collections are not sharded. - - assumes_no_implicit_collection_creation_after_drop - - assumes_no_implicit_index_creation - - assumes_unsharded_collection - - cannot_create_unique_index_when_using_hashed_shard_key - # system.profile collection doesn't exist on mongos. - - requires_profiling - -executor: - archive: - hooks: - - CheckReplDBHash - - CheckMetadataConsistencyInBackground - - ValidateCollections - config: - shell_options: - eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") - hooks: - - class: CheckReplDBHash - - class: CheckMetadataConsistencyInBackground - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - num_shards: 2 - enable_balancer: false - mongos_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 100 - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 100 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml deleted file mode 100644 index 3464f572f79..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml +++ /dev/null @@ -1,72 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/txns/**/*.js - exclude_files: - # Profile can only be run against the admin database on mongos. - - jstests/core/txns/transactions_profiling.js - - jstests/core/txns/transactions_profiling_with_drops.js - - # Implicitly creates a database through a collection rename, which does not work in a sharded - # cluster. - - jstests/core/txns/transactions_block_ddl.js - - # transactionLifetimeLimitSeconds parameter is not available in mongos. - - jstests/core/txns/abort_expired_transaction.js - - jstests/core/txns/abort_transaction_thread_does_not_block_on_locks.js - - jstests/core/txns/kill_op_on_txn_expiry.js - - # Uses hangAfterCollectionInserts failpoint not available on mongos. - - jstests/core/txns/speculative_snapshot_includes_all_writes.js - - # View tests aren't expected to work when collections are implicitly sharded. - - jstests/core/txns/view_reads_in_transaction.js - - # These workloads explicitly create collections inside multi-document transactions. These are - # non-idempotent operations, and the implicit collection sharding logic upon collection access - # results in premature collection creation, causing the workloads to fail. - - jstests/core/txns/create_collection.js - - jstests/core/txns/create_collection_parallel.js - - jstests/core/txns/create_indexes.js - - jstests/core/txns/create_indexes_parallel.js - - exclude_with_any_tags: - - assumes_against_mongod_not_mongos - # Tests tagged with the following will fail because they assume collections are not sharded. - - assumes_no_implicit_collection_creation_after_drop - - assumes_no_implicit_index_creation - - assumes_unsharded_collection - - cannot_create_unique_index_when_using_hashed_shard_key - # Transactions are not allowed to operate on capped collections. - - requires_capped - # Prepare is not a command on mongos. - - uses_prepare_transaction - -executor: - archive: - hooks: - - CheckReplDBHash - - ValidateCollections - config: - shell_options: - eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") - hooks: - - class: CheckReplDBHash - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - num_shards: 2 - enable_balancer: false - mongos_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 100 - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 100 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_10sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_10sec_refresh_jscore_passthrough.yml deleted file mode 100644 index 076717acd11..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_10sec_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,116 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - exclude_files: - # These tests are run in logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml. - - jstests/core/txns/**/*.js - # The following tests fail because a certain command or functionality is not supported by - # mongos. This command or functionality is placed in a comment next to the failing test. - - jstests/core/**/apitest_db.js # serverStatus output doesn't have storageEngine. - - jstests/core/**/awaitdata_getmore_cmd.js # capped collections. - - jstests/core/**/bypass_doc_validation.js # sharded $out output not permitted - - jstests/core/**/check_shard_index.js # checkShardingIndex. - - jstests/core/**/collection_truncate.js # emptycapped. - - jstests/core/**/compact_keeps_indexes.js # compact. - - jstests/core/**/currentop.js # uses fsync. - - jstests/core/**/dbhash.js # dbhash. - - jstests/core/**/dbhash2.js # dbhash. - - jstests/core/**/fsync.js # uses fsync. - - jstests/core/**/geo_s2cursorlimitskip.js # profiling. - - jstests/core/**/geo_update_btree2.js # notablescan. - - jstests/core/**/index9.js # "local" database. - - jstests/core/**/queryoptimizera.js # "local" database. - - jstests/core/**/stages*.js # stageDebug. - - jstests/core/**/startup_log.js # "local" database. - - jstests/core/**/tailable_cursor_invalidation.js # capped collections. - - jstests/core/**/tailable_getmore_batch_size.js # capped collections. - - jstests/core/**/tailable_skip_limit.js # capped collections. - - jstests/core/**/top.js # top. - # The following tests fail because mongos behaves differently from mongod when testing certain - # functionality. The differences are in a comment next to the failing test. - - jstests/core/**/explain_missing_database.js # Behavior with no db different on mongos, SERVER-18047. - - jstests/core/**/geo_2d_explain.js # executionSuccess in different spot in explain(). - - jstests/core/**/geo_s2explain.js # inputStage in different spot in explain(). - - jstests/core/**/geo_s2sparse.js # keysPerIndex in different spot in validate(). - - jstests/core/**/operation_latency_histogram.js # Stats are counted differently on mongos, SERVER-24880. - # The following tests fail because they count indexes. These counts do not take into account the - # additional hashed shard key indexes that are automatically added by this passthrough. - - jstests/core/**/apitest_dbcollection.js - - jstests/core/**/bad_index_plugin.js - - jstests/core/**/create_indexes.js - - jstests/core/**/list_indexes_non_existent_ns.js - - jstests/core/**/mr_preserve_indexes.js - # The following tests fail because they expect no databases to be created. However a DB is created - # automatically when we shard a collection. - - jstests/core/**/dbcase.js - - jstests/core/**/dbcase2.js - - jstests/core/**/no_db_created.js - - jstests/core/**/killop_drop_collection.js # Uses fsyncLock. - # These tests fail because sharded clusters do not clean up correctly after failed index builds. - # See SERVER-33207 as an example. - - jstests/core/**/geo_borders.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded - # queries with a limit or for distinct commands. - - jstests/core/**/distinct_index1.js - - jstests/core/**/explain1.js - - jstests/core/**/explain4.js - - jstests/core/**/sortk.js - # TODO: Remove after fixing SERVER-32563. The implementation of explain for the count command is - # incorrect on sharded collections. - - jstests/core/**/explain_count.js - - jstests/core/**/explain_server_params.js - # TODO SERVER-32311: These tests use plan stage helpers which can't handle sharded explain output. - - jstests/core/**/expr_index_use.js - - jstests/core/**/index_multikey.js - - jstests/core/**/optimized_match_explain.js - - jstests/core/**/sort_array.js - - exclude_with_any_tags: - - assumes_standalone_mongod - - assumes_against_mongod_not_mongos - # Tests tagged with the following will fail because they assume collections are not sharded. - - assumes_no_implicit_collection_creation_after_drop - - assumes_no_implicit_index_creation - - assumes_unsharded_collection - - cannot_create_unique_index_when_using_hashed_shard_key - # system.profile collection doesn't exist on mongos. - - requires_profiling - -executor: - archive: - hooks: - - CheckReplDBHash - - CheckMetadataConsistencyInBackground - - ValidateCollections - config: - shell_options: - eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") - hooks: - - class: CheckReplDBHash - - class: CheckMetadataConsistencyInBackground - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - num_shards: 2 - enable_balancer: false - mongos_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 10000 - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 10000 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_1sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_1sec_refresh_jscore_passthrough.yml deleted file mode 100644 index 26838b46c04..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_1sec_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,116 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - exclude_files: - # These tests are run in logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml. - - jstests/core/txns/**/*.js - # The following tests fail because a certain command or functionality is not supported by - # mongos. This command or functionality is placed in a comment next to the failing test. - - jstests/core/**/apitest_db.js # serverStatus output doesn't have storageEngine. - - jstests/core/**/awaitdata_getmore_cmd.js # capped collections. - - jstests/core/**/bypass_doc_validation.js # sharded $out output not permitted - - jstests/core/**/check_shard_index.js # checkShardingIndex. - - jstests/core/**/collection_truncate.js # emptycapped. - - jstests/core/**/compact_keeps_indexes.js # compact. - - jstests/core/**/currentop.js # uses fsync. - - jstests/core/**/dbhash.js # dbhash. - - jstests/core/**/dbhash2.js # dbhash. - - jstests/core/**/fsync.js # uses fsync. - - jstests/core/**/geo_s2cursorlimitskip.js # profiling. - - jstests/core/**/geo_update_btree2.js # notablescan. - - jstests/core/**/index9.js # "local" database. - - jstests/core/**/queryoptimizera.js # "local" database. - - jstests/core/**/stages*.js # stageDebug. - - jstests/core/**/startup_log.js # "local" database. - - jstests/core/**/tailable_cursor_invalidation.js # capped collections. - - jstests/core/**/tailable_getmore_batch_size.js # capped collections. - - jstests/core/**/tailable_skip_limit.js # capped collections. - - jstests/core/**/top.js # top. - # The following tests fail because mongos behaves differently from mongod when testing certain - # functionality. The differences are in a comment next to the failing test. - - jstests/core/**/explain_missing_database.js # Behavior with no db different on mongos, SERVER-18047. - - jstests/core/**/geo_2d_explain.js # executionSuccess in different spot in explain(). - - jstests/core/**/geo_s2explain.js # inputStage in different spot in explain(). - - jstests/core/**/geo_s2sparse.js # keysPerIndex in different spot in validate(). - - jstests/core/**/operation_latency_histogram.js # Stats are counted differently on mongos, SERVER-24880. - # The following tests fail because they count indexes. These counts do not take into account the - # additional hashed shard key indexes that are automatically added by this passthrough. - - jstests/core/**/apitest_dbcollection.js - - jstests/core/**/bad_index_plugin.js - - jstests/core/**/create_indexes.js - - jstests/core/**/list_indexes_non_existent_ns.js - - jstests/core/**/mr_preserve_indexes.js - # The following tests fail because they expect no databases to be created. However a DB is created - # automatically when we shard a collection. - - jstests/core/**/dbcase.js - - jstests/core/**/dbcase2.js - - jstests/core/**/no_db_created.js - - jstests/core/**/killop_drop_collection.js # Uses fsyncLock. - # These tests fail because sharded clusters do not clean up correctly after failed index builds. - # See SERVER-33207 as an example. - - jstests/core/**/geo_borders.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded - # queries with a limit or for distinct commands. - - jstests/core/**/distinct_index1.js - - jstests/core/**/explain1.js - - jstests/core/**/explain4.js - - jstests/core/**/sortk.js - # TODO: Remove after fixing SERVER-32563. The implementation of explain for the count command is - # incorrect on sharded collections. - - jstests/core/**/explain_count.js - - jstests/core/**/explain_server_params.js - # TODO SERVER-32311: These tests use plan stage helpers which can't handle sharded explain output. - - jstests/core/**/expr_index_use.js - - jstests/core/**/index_multikey.js - - jstests/core/**/optimized_match_explain.js - - jstests/core/**/sort_array.js - - exclude_with_any_tags: - - assumes_standalone_mongod - - assumes_against_mongod_not_mongos - # Tests tagged with the following will fail because they assume collections are not sharded. - - assumes_no_implicit_collection_creation_after_drop - - assumes_no_implicit_index_creation - - assumes_unsharded_collection - - cannot_create_unique_index_when_using_hashed_shard_key - # system.profile collection doesn't exist on mongos. - - requires_profiling - -executor: - archive: - hooks: - - CheckReplDBHash - - CheckMetadataConsistencyInBackground - - ValidateCollections - config: - shell_options: - eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") - hooks: - - class: CheckReplDBHash - - class: CheckMetadataConsistencyInBackground - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - num_shards: 2 - enable_balancer: false - mongos_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 1000 - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 1000 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_default_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_default_refresh_jscore_passthrough.yml deleted file mode 100644 index fbc8e2736e3..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_sharding_default_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,114 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - exclude_files: - # These tests are run in logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough.yml. - - jstests/core/txns/**/*.js - # The following tests fail because a certain command or functionality is not supported by - # mongos. This command or functionality is placed in a comment next to the failing test. - - jstests/core/**/apitest_db.js # serverStatus output doesn't have storageEngine. - - jstests/core/**/awaitdata_getmore_cmd.js # capped collections. - - jstests/core/**/bypass_doc_validation.js # sharded $out output not permitted - - jstests/core/**/check_shard_index.js # checkShardingIndex. - - jstests/core/**/collection_truncate.js # emptycapped. - - jstests/core/**/compact_keeps_indexes.js # compact. - - jstests/core/**/currentop.js # uses fsync. - - jstests/core/**/dbhash.js # dbhash. - - jstests/core/**/dbhash2.js # dbhash. - - jstests/core/**/fsync.js # uses fsync. - - jstests/core/**/geo_s2cursorlimitskip.js # profiling. - - jstests/core/**/geo_update_btree2.js # notablescan. - - jstests/core/**/index9.js # "local" database. - - jstests/core/**/queryoptimizera.js # "local" database. - - jstests/core/**/stages*.js # stageDebug. - - jstests/core/**/startup_log.js # "local" database. - - jstests/core/**/tailable_cursor_invalidation.js # capped collections. - - jstests/core/**/tailable_getmore_batch_size.js # capped collections. - - jstests/core/**/tailable_skip_limit.js # capped collections. - - jstests/core/**/top.js # top. - # The following tests fail because mongos behaves differently from mongod when testing certain - # functionality. The differences are in a comment next to the failing test. - - jstests/core/**/explain_missing_database.js # Behavior with no db different on mongos, SERVER-18047. - - jstests/core/**/geo_2d_explain.js # executionSuccess in different spot in explain(). - - jstests/core/**/geo_s2explain.js # inputStage in different spot in explain(). - - jstests/core/**/geo_s2sparse.js # keysPerIndex in different spot in validate(). - - jstests/core/**/operation_latency_histogram.js # Stats are counted differently on mongos, SERVER-24880. - # The following tests fail because they count indexes. These counts do not take into account the - # additional hashed shard key indexes that are automatically added by this passthrough. - - jstests/core/**/apitest_dbcollection.js - - jstests/core/**/bad_index_plugin.js - - jstests/core/**/create_indexes.js - - jstests/core/**/list_indexes_non_existent_ns.js - - jstests/core/**/mr_preserve_indexes.js - # The following tests fail because they expect no databases to be created. However a DB is created - # automatically when we shard a collection. - - jstests/core/**/dbcase.js - - jstests/core/**/dbcase2.js - - jstests/core/**/no_db_created.js - - jstests/core/**/killop_drop_collection.js # Uses fsyncLock. - # These tests fail because sharded clusters do not clean up correctly after failed index builds. - # See SERVER-33207 as an example. - - jstests/core/**/geo_borders.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # TODO: Remove after fixing SERVER-29449. executionStats.nReturned is incorrect for sharded - # queries with a limit or for distinct commands. - - jstests/core/**/distinct_index1.js - - jstests/core/**/explain1.js - - jstests/core/**/explain4.js - - jstests/core/**/sortk.js - # TODO: Remove after fixing SERVER-32563. The implementation of explain for the count command is - # incorrect on sharded collections. - - jstests/core/**/explain_count.js - - jstests/core/**/explain_server_params.js - # TODO SERVER-32311: These tests use plan stage helpers which can't handle sharded explain output. - - jstests/core/**/expr_index_use.js - - jstests/core/**/index_multikey.js - - jstests/core/**/optimized_match_explain.js - - jstests/core/**/sort_array.js - - exclude_with_any_tags: - - assumes_standalone_mongod - - assumes_against_mongod_not_mongos - # Tests tagged with the following will fail because they assume collections are not sharded. - - assumes_no_implicit_collection_creation_after_drop - - assumes_no_implicit_index_creation - - assumes_unsharded_collection - - cannot_create_unique_index_when_using_hashed_shard_key - # system.profile collection doesn't exist on mongos. - - requires_profiling - -executor: - archive: - hooks: - - CheckReplDBHash - - CheckMetadataConsistencyInBackground - - ValidateCollections - config: - shell_options: - eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js") - hooks: - - class: CheckReplDBHash - - class: CheckMetadataConsistencyInBackground - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - num_shards: 2 - enable_balancer: false - mongos_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_100ms_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_100ms_refresh_jscore_passthrough.yml deleted file mode 100644 index 2b61effb3a1..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_100ms_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,37 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - exclude_files: - # Transactions are not supported on MongoDB standalone nodes. - - jstests/core/txns/**/*.js - # This test expects a fixed number of operations. The logical session cache will perform its own - # operations, inflating the number of operations and causing the test to fail. - - jstests/core/**/opcounters_write_cmd.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # These tests verify that an expected number of update operations were tracked in the server - # status metrics, but the logical session cache refresh causes additional updates to be recorded. - - jstests/core/**/find_and_modify_metrics.js - - jstests/core/**/update_metrics.js - -executor: - archive: - hooks: - - ValidateCollections - config: {} - hooks: - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: MongoDFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 100 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_10sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_10sec_refresh_jscore_passthrough.yml deleted file mode 100644 index 4d3ef6b2701..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_10sec_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,37 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - exclude_files: - # Transactions are not supported on MongoDB standalone nodes. - - jstests/core/txns/**/*.js - # This test expects a fixed number of operations. The logical session cache will perform its own - # operations, inflating the number of operations and causing the test to fail. - - jstests/core/**/opcounters_write_cmd.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # These tests verify that an expected number of update operations were tracked in the server - # status metrics, but the logical session cache refresh causes additional updates to be recorded. - - jstests/core/**/find_and_modify_metrics.js - - jstests/core/**/update_metrics.js - -executor: - archive: - hooks: - - ValidateCollections - config: {} - hooks: - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: MongoDFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 10000 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_1sec_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_1sec_refresh_jscore_passthrough.yml deleted file mode 100644 index fa7b9c13daa..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_1sec_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,37 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - exclude_files: - # Transactions are not supported on MongoDB standalone nodes. - - jstests/core/txns/**/*.js - # This test expects a fixed number of operations. The logical session cache will perform its own - # operations, inflating the number of operations and causing the test to fail. - - jstests/core/**/opcounters_write_cmd.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # These tests verify that an expected number of update operations were tracked in the server - # status metrics, but the logical session cache refresh causes additional updates to be recorded. - - jstests/core/**/find_and_modify_metrics.js - - jstests/core/**/update_metrics.js - -executor: - archive: - hooks: - - ValidateCollections - config: {} - hooks: - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: MongoDFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false - logicalSessionRefreshMillis: 1000 diff --git a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_default_refresh_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_default_refresh_jscore_passthrough.yml deleted file mode 100644 index 5f91f3b17f3..00000000000 --- a/buildscripts/resmokeconfig/suites/logical_session_cache_standalone_default_refresh_jscore_passthrough.yml +++ /dev/null @@ -1,36 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/core/**/*.js - exclude_files: - # Transactions are not supported on MongoDB standalone nodes. - - jstests/core/txns/**/*.js - # This test expects a fixed number of operations. The logical session cache will perform its own - # operations, inflating the number of operations and causing the test to fail. - - jstests/core/**/opcounters_write_cmd.js - # These tests expect the logical session cache refresh thread to be turned off, so that refreshes - # can be triggered deterministically. - - jstests/core/**/list_all_local_sessions.js - - jstests/core/**/list_all_sessions.js - - jstests/core/**/list_sessions.js - # These tests verify that an expected number of update operations were tracked in the server - # status metrics, but the logical session cache refresh causes additional updates to be recorded. - - jstests/core/**/find_and_modify_metrics.js - - jstests/core/**/update_metrics.js - -executor: - archive: - hooks: - - ValidateCollections - config: {} - hooks: - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: MongoDFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - disableLogicalSessionCacheRefresh: false diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 36dee63d936..bf744d4ff75 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -67,6 +67,7 @@ DEFAULTS = { "flow_control_tickets": None, "force_excluded_tests": False, "fuzz_mongod_configs": None, + "fuzz_mongos_configs": None, "config_fuzz_seed": None, "genny_executable": None, "include_with_any_tags": None, @@ -357,7 +358,13 @@ EXCLUDED_TAG = "__TEMPORARILY_DISABLED__" # If true, then a test failure or error will cause resmoke.py to exit and not run any more tests. FAIL_FAST = None +# Defines how to fuzz mongod parameters FUZZ_MONGOD_CONFIGS = None + +# Defines how to fuzz mongos parameters +FUZZ_MONGOS_CONFIGS = None + +# This seeds the random number generator used to fuzz mongod and mongos parameters CONFIG_FUZZ_SEED = None # Executable file for genny, passed in as a command line arg. diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 2656b97d754..84914e292be 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -20,7 +20,7 @@ from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import utils -from buildscripts.resmokelib import mongod_fuzzer_configs +from buildscripts.resmokelib import mongo_fuzzer_configs from buildscripts.resmokelib.suitesconfig import SuiteFinder @@ -301,6 +301,7 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.MONGOD_SET_PARAMETERS = _merge_set_params(mongod_set_parameters) _config.FUZZ_MONGOD_CONFIGS = config.pop("fuzz_mongod_configs") + _config.FUZZ_MONGOS_CONFIGS = config.pop("fuzz_mongos_configs") _config.CONFIG_FUZZ_SEED = config.pop("config_fuzz_seed") if _config.FUZZ_MONGOD_CONFIGS: @@ -309,7 +310,7 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py else: _config.CONFIG_FUZZ_SEED = int(_config.CONFIG_FUZZ_SEED) _config.MONGOD_SET_PARAMETERS, _config.WT_ENGINE_CONFIG, _config.WT_COLL_CONFIG, \ - _config.WT_INDEX_CONFIG = mongod_fuzzer_configs.fuzz_set_parameters( + _config.WT_INDEX_CONFIG = mongo_fuzzer_configs.fuzz_mongod_set_parameters( _config.FUZZ_MONGOD_CONFIGS, _config.CONFIG_FUZZ_SEED, _config.MONGOD_SET_PARAMETERS) _config.EXCLUDE_WITH_ANY_TAGS.extend(["uses_compact"]) _config.EXCLUDE_WITH_ANY_TAGS.extend(["requires_emptycapped"]) @@ -318,6 +319,15 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py mongos_set_parameters = config.pop("mongos_set_parameters") _config.MONGOS_SET_PARAMETERS = _merge_set_params(mongos_set_parameters) + if _config.FUZZ_MONGOS_CONFIGS: + if not _config.CONFIG_FUZZ_SEED: + _config.CONFIG_FUZZ_SEED = random.randrange(sys.maxsize) + else: + _config.CONFIG_FUZZ_SEED = int(_config.CONFIG_FUZZ_SEED) + + _config.MONGOS_SET_PARAMETERS = mongo_fuzzer_configs.fuzz_mongos_set_parameters( + _config.FUZZ_MONGOS_CONFIGS, _config.CONFIG_FUZZ_SEED, _config.MONGOS_SET_PARAMETERS) + _config.MONGOCRYPTD_SET_PARAMETERS = _merge_set_params(config.pop("mongocryptd_set_parameters")) _config.MRLOG = config.pop("mrlog") diff --git a/buildscripts/resmokelib/mongod_fuzzer_configs.py b/buildscripts/resmokelib/mongo_fuzzer_configs.py index 42a92a0bfed..79978af8eec 100644 --- a/buildscripts/resmokelib/mongod_fuzzer_configs.py +++ b/buildscripts/resmokelib/mongo_fuzzer_configs.py @@ -112,33 +112,57 @@ def generate_flow_control_parameters(rng): return configs -def generate_independent_parameters(rng, mode): - """Return a dictionary with values for each independent parameter.""" +def generate_mongod_parameters(rng, mode): + """Return a dictionary with values for each mongod parameter.""" ret = {} + ret["analyzeShardKeySplitPointExpirationSecs"] = rng.randint(1, 300) + ret["chunkMigrationConcurrency"] = rng.choice([1, 4, 16]) + ret["disableLogicalSessionCacheRefresh"] = rng.choice([True, False]) + ret["initialServiceExecutorUseDedicatedThread"] = rng.choice([True, False]) + # TODO (SERVER-75632): Uncomment this to enable passthrough testing. + # ret["lockCodeSegmentsInMemory"] = rng.choice([True, False]) + if not ret["disableLogicalSessionCacheRefresh"]: + ret["logicalSessionRefreshMillis"] = rng.choice([100, 1000, 10000, 100000]) + ret["maxNumberOfTransactionOperationsInSingleOplogEntry"] = rng.randint(1, 10) * rng.choice( + [1, 10, 100]) + ret["minSnapshotHistoryWindowInSeconds"] = rng.choice([300, rng.randint(30, 600)]) + ret["mirrorReads"] = {"samplingRate": rng.random()} + ret["queryAnalysisSampleExpirationSecs"] = rng.choice([1, 10, 100, 1000]) + ret["queryAnalysisSamplerConfigurationRefreshSecs"] = rng.choice([1, 10, 100]) + ret["queryAnalysisWriterIntervalSecs"] = rng.choice([1, 10, 100]) + ret["queryAnalysisWriterMaxMemoryUsageBytes"] = rng.randint(1, 100) * 1024 * 1024 + ret["syncdelay"] = rng.choice([60, rng.randint(15, 180)]) ret["wiredTigerCursorCacheSize"] = rng.randint(-100, 100) ret["wiredTigerSessionCloseIdleTimeSecs"] = rng.randint(0, 300) ret["storageEngineConcurrencyAdjustmentAlgorithm"] = "fixedConcurrentTransactions" - ret["wiredTigerConcurrentWriteTransactions"] = rng.randint(5, 32) - ret["wiredTigerConcurrentReadTransactions"] = rng.randint(5, 32) - ret["wiredTigerStressConfig"] = False if mode != 'stress' else rng.choice([True, False]) if rng.choice(3 * [True] + [False]): # The old retryable writes format is used by other variants. Weight towards turning on the # new retryable writes format on in this one. ret["storeFindAndModifyImagesInSideCollection"] = True - ret["syncdelay"] = rng.choice([60, rng.randint(15, 180)]) - ret["minSnapshotHistoryWindowInSeconds"] = rng.choice([300, rng.randint(5, 600)]) - # TODO (SERVER-75632): Uncomment this to enable passthrough testing. - # ret["lockCodeSegmentsInMemory"] = rng.choice([True, False]) + ret["wiredTigerConcurrentWriteTransactions"] = rng.randint(5, 32) + ret["wiredTigerConcurrentReadTransactions"] = rng.randint(5, 32) + ret["wiredTigerStressConfig"] = False if mode != 'stress' else rng.choice([True, False]) + + # We need a higher timeout to account for test slowness + ret["receiveChunkWaitForRangeDeleterTimeoutMS"] = 300000 + return ret + +def generate_mongos_parameters(rng, mode): + """Return a dictionary with values for each mongos parameter.""" + ret = {} + ret["initialServiceExecutorUseDedicatedThread"] = rng.choice([True, False]) + ret["opportunisticSecondaryTargeting"] = rng.choice([True, False]) + ret["queryAnalysisSamplerConfigurationRefreshSecs"] = rng.choice([1, 10, 100]) return ret -def fuzz_set_parameters(mode, seed, user_provided_params): +def fuzz_mongod_set_parameters(mode, seed, user_provided_params): """Randomly generate mongod configurations and wiredTigerConnectionString.""" rng = random.Random(seed) ret = {} - params = [generate_flow_control_parameters(rng), generate_independent_parameters(rng, mode)] + params = [generate_flow_control_parameters(rng), generate_mongod_parameters(rng, mode)] for dct in params: for key, value in dct.items(): ret[key] = value @@ -148,3 +172,18 @@ def fuzz_set_parameters(mode, seed, user_provided_params): return utils.dump_yaml(ret), generate_eviction_configs(rng, mode), generate_table_configs(rng), \ generate_table_configs(rng) + + +def fuzz_mongos_set_parameters(mode, seed, user_provided_params): + """Randomly generate mongos configurations.""" + rng = random.Random(seed) + + ret = {} + params = generate_mongos_parameters(rng, mode) + for key, value in params.items(): + ret[key] = value + + for key, value in utils.load_yaml(user_provided_params).items(): + ret[key] = value + + return utils.dump_yaml(ret) diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index deab6b701c9..5b9fa04f129 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -260,20 +260,30 @@ class TestRunner(Subcommand): def _log_local_resmoke_invocation(self): """Log local resmoke invocation example.""" local_args = to_local_args() + local_args = strip_fuzz_config_params(local_args) local_resmoke_invocation = ( f"{os.path.join('buildscripts', 'resmoke.py')} {' '.join(local_args)}") + using_config_fuzzer = False if config.FUZZ_MONGOD_CONFIGS: - local_args = strip_fuzz_config_params(local_args) - local_resmoke_invocation = ( - f"{os.path.join('buildscripts', 'resmoke.py')} {' '.join(local_args)}" - f" --fuzzMongodConfigs={config.FUZZ_MONGOD_CONFIGS} --configFuzzSeed={str(config.CONFIG_FUZZ_SEED)}" - ) + using_config_fuzzer = True + local_resmoke_invocation += f" --fuzzMongodConfigs={config.FUZZ_MONGOD_CONFIGS}" self._resmoke_logger.info("Fuzzed mongodSetParameters:\n%s", config.MONGOD_SET_PARAMETERS) self._resmoke_logger.info("Fuzzed wiredTigerConnectionString: %s", config.WT_ENGINE_CONFIG) + + if config.FUZZ_MONGOS_CONFIGS: + using_config_fuzzer = True + local_resmoke_invocation += f" --fuzzMongosConfigs={config.FUZZ_MONGOS_CONFIGS}" + + self._resmoke_logger.info("Fuzzed mongosSetParameters:\n%s", + config.MONGOS_SET_PARAMETERS) + + if using_config_fuzzer: + local_resmoke_invocation += f" --configFuzzSeed={str(config.CONFIG_FUZZ_SEED)}" + resmoke_env_options = '' if os.path.exists('resmoke_env_options.txt'): with open('resmoke_env_options.txt') as fin: @@ -936,14 +946,19 @@ class RunPlugin(PluginInterface): mongodb_server_options.add_argument( "--fuzzMongodConfigs", dest="fuzz_mongod_configs", - help="Randomly chooses server parameters that were not specified. Use 'stress' to fuzz " + help="Randomly chooses mongod parameters that were not specified. Use 'stress' to fuzz " "all configs including stressful storage configurations that may significantly " "slow down the server. Use 'normal' to only fuzz non-stressful configurations. ", metavar="MODE", choices=('normal', 'stress')) - mongodb_server_options.add_argument("--configFuzzSeed", dest="config_fuzz_seed", - metavar="PATH", - help="Sets the seed used by storage config fuzzer") + mongodb_server_options.add_argument( + "--fuzzMongosConfigs", dest="fuzz_mongos_configs", + help="Randomly chooses mongos parameters that were not specified", metavar="MODE", + choices=('normal', )) + + mongodb_server_options.add_argument( + "--configFuzzSeed", dest="config_fuzz_seed", metavar="PATH", + help="Sets the seed used by mongod and mongos config fuzzers") mongodb_server_options.add_argument( "--configShard", dest="config_shard", metavar="CONFIG", @@ -1289,7 +1304,7 @@ def strip_fuzz_config_params(input_args): ret = [] for arg in input_args: - if "--fuzzMongodConfigs" not in arg and "--fuzzConfigSeed" not in arg: + if not arg.startswith(("--fuzzMongodConfigs", "--fuzzMongosConfigs", "--configFuzzSeed")): ret.append(arg) return ret |