diff options
34 files changed, 623 insertions, 486 deletions
diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 2a05a307362..75e6ef0378a 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -212,6 +212,10 @@ class Task(object): """Check if the task requires running the multiversion setup.""" return "multiversion" in self.tags + def require_multiversion_version_combo(self): + """Check if the task requires generating combinations of multiversion versions.""" + return "multiversion" in self.tags and "no_version_combination" not in self.tags + def requires_npm(self): """Check if the task needs to run npm setup.""" return "require_npm" in self.tags diff --git a/buildscripts/evergreen_burn_in_tests.py b/buildscripts/evergreen_burn_in_tests.py index 60464592615..7ffcbf4153b 100644 --- a/buildscripts/evergreen_burn_in_tests.py +++ b/buildscripts/evergreen_burn_in_tests.py @@ -214,13 +214,23 @@ class BurnInGenTaskService: return TimeoutInfo.default_timeout() - def _generate_resmoke_args(self, task_name: str, params: BurnInGenTaskParams, + def _generate_resmoke_args(self, task_name: str, suite_name: str, params: BurnInGenTaskParams, test_arg: str) -> str: resmoke_args = f"{params.resmoke_args} {self.repeat_config.generate_resmoke_options()} {test_arg}" if params.require_multiversion_setup: - tag_file = EXCLUDES_TAGS_FILE_PATH - resmoke_args += f" --tagFile={tag_file}" + # TODO: inspect the suite for version instead of doing string parsing on the name. + tag_file_base_path = EXCLUDES_TAGS_FILE_PATH[:-4] + if "last_continuous" in suite_name: + tag_file = f"{tag_file_base_path}_last_continuous.yml" + elif "last_lts" in suite_name: + tag_file = f"{tag_file_base_path}_last_lts.yml" + else: + tag_file = None + + if tag_file is not None: + resmoke_args += f" --tagFile={tag_file}" + resmoke_args += f" --excludeWithAnyTags={EXCLUDE_TAGS},{task_name}_{BACKPORT_REQUIRED_TAG} " return resmoke_args @@ -252,7 +262,8 @@ class BurnInGenTaskService: run_tests_vars = { "suite": gen_suite.suite_name, "resmoke_args": - self._generate_resmoke_args(gen_suite.task_name, params, test_unix_style) + self._generate_resmoke_args(gen_suite.task_name, gen_suite.suite_name, + params, test_unix_style) } timeout_cmd = self.generate_timeouts(test_name) diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py index e7f7c840506..b59195099da 100644 --- a/buildscripts/evergreen_gen_build_variant.py +++ b/buildscripts/evergreen_gen_build_variant.py @@ -217,6 +217,7 @@ class GenerateBuildVariantOrchestrator: return ResmokeGenTaskParams( use_large_distro=run_vars.get("use_large_distro"), require_multiversion_setup=task_def.require_multiversion_setup(), + require_multiversion_version_combo=task_def.require_multiversion_version_combo(), repeat_suites=repeat_suites, resmoke_args=run_vars.get("resmoke_args"), resmoke_jobs_max=run_vars.get("resmoke_jobs_max"), diff --git a/buildscripts/patch_builds/task_generation.py b/buildscripts/patch_builds/task_generation.py index 887e7e2a008..d8d9ff267d4 100644 --- a/buildscripts/patch_builds/task_generation.py +++ b/buildscripts/patch_builds/task_generation.py @@ -7,6 +7,8 @@ from shrub.v2 import FunctionCall, ShrubProject from shrub.v2.command import timeout_update, ShrubCommand from structlog import get_logger +from buildscripts.task_generation.constants import CONFIGURE_EVG_CREDENTIALS + LOGGER = get_logger(__name__) MAX_SHRUB_TASKS_FOR_SINGLE_TASK = 1000 @@ -30,18 +32,21 @@ def resmoke_commands(run_tests_fn_name: str, run_tests_vars: Dict[str, Any], """ Create a list of commands to run a resmoke task. + Used by burn_in* only. Other tasks use a standalone multiversion decorator. + :param run_tests_fn_name: Name of function to run resmoke tests. :param run_tests_vars: Dictionary of variables to pass to run_tests function. :param timeout_info: Timeout info for task. :param require_multiversion_setup: Requires downloading Multiversion binaries. :return: List of commands to run a resmoke task. """ + commands = [ timeout_info.cmd, FunctionCall("git get project no modules") if require_multiversion_setup else None, FunctionCall("add git tag") if require_multiversion_setup else None, FunctionCall("do setup"), - FunctionCall("configure evergreen api credentials"), + FunctionCall(CONFIGURE_EVG_CREDENTIALS), FunctionCall("do multiversion setup") if require_multiversion_setup else None, FunctionCall(run_tests_fn_name, run_tests_vars), ] diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml index d154c896665..c1cce697c95 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml @@ -1,64 +1,207 @@ -- suite_name: change_streams_multiversion +- suite_name: change_streams_last_continuous_new_new_old base_suite: change_streams overrides: - - "multiversion.replica_fixture" - - "multiversion.test_kind" + - "multiversion.replica_fixture_last_continuous_new_new_old" + +- suite_name: change_streams_last_continuous_new_old_new + base_suite: change_streams + overrides: + - "multiversion.replica_fixture_last_continuous_new_old_new" + +- suite_name: change_streams_last_continuous_old_new_new + base_suite: change_streams + overrides: + - "multiversion.replica_fixture_last_continuous_old_new_new" + +- suite_name: change_streams_last_lts_new_new_old + base_suite: change_streams + overrides: + - "multiversion.replica_fixture_last_lts_new_new_old" + +- suite_name: change_streams_last_lts_new_old_new + base_suite: change_streams + overrides: + - "multiversion.replica_fixture_last_lts_new_old_new" -- suite_name: change_streams_sharded_collections_multiversion +- suite_name: change_streams_last_lts_old_new_new + base_suite: change_streams + overrides: + - "multiversion.replica_fixture_last_lts_old_new_new" + + +- suite_name: change_streams_sharded_collections_passthrough_last_continuous_new_old_old_new + base_suite: change_streams_sharded_collections_passthrough + overrides: + - "multiversion.sharded_fixture_last_continuous_new_old_old_new" + +- suite_name: change_streams_sharded_collections_passthrough_last_lts_new_old_old_new base_suite: change_streams_sharded_collections_passthrough overrides: - - "multiversion.sharded_fixture" - - "multiversion.test_kind" + - "multiversion.sharded_fixture_last_lts_new_old_old_new" + -- suite_name: concurrency_replication_multiversion +- suite_name: concurrency_replication_last_continuous_new_new_old base_suite: concurrency_replication overrides: - - "multiversion.replica_fixture" + - "multiversion.replica_fixture_last_continuous_new_new_old" + +- suite_name: concurrency_replication_last_continuous_new_old_new + base_suite: concurrency_replication + overrides: + - "multiversion.replica_fixture_last_continuous_new_old_new" + +- suite_name: concurrency_replication_last_continuous_old_new_new + base_suite: concurrency_replication + overrides: + - "multiversion.replica_fixture_last_continuous_old_new_new" + +- suite_name: concurrency_replication_last_lts_new_new_old + base_suite: concurrency_replication + overrides: + - "multiversion.replica_fixture_last_lts_new_new_old" + +- suite_name: concurrency_replication_last_lts_new_old_new + base_suite: concurrency_replication + overrides: + - "multiversion.replica_fixture_last_lts_new_old_new" + +- suite_name: concurrency_replication_last_lts_old_new_new + base_suite: concurrency_replication + overrides: + - "multiversion.replica_fixture_last_lts_old_new_new" + + +- suite_name: multiversion_sanity_check # Dummy "base suite" to placate the task generator. + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.multiversion_sanity_check_selector" + +- suite_name: multiversion_sanity_check_last_continuous_new_new_old + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_continuous_new_new_old" + - "multiversion.multiversion_sanity_check_selector" + +- suite_name: multiversion_sanity_check_last_continuous_new_old_new + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_continuous_new_old_new" + - "multiversion.multiversion_sanity_check_selector" + +- suite_name: multiversion_sanity_check_last_continuous_old_new_new + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_continuous_old_new_new" + - "multiversion.multiversion_sanity_check_selector" + +- suite_name: multiversion_sanity_check_last_lts_new_new_old + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_lts_new_new_old" + - "multiversion.multiversion_sanity_check_selector" + +- suite_name: multiversion_sanity_check_last_lts_new_old_new + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_lts_new_old_new" + - "multiversion.multiversion_sanity_check_selector" -- suite_name: multiversion_sanity_check +- suite_name: multiversion_sanity_check_last_lts_old_new_new base_suite: replica_sets_jscore_passthrough overrides: - - "multiversion.replica_fixture" + - "multiversion.replica_fixture_last_lts_old_new_new" - "multiversion.multiversion_sanity_check_selector" - - "multiversion.test_kind" -- suite_name: replica_sets_jscore_multiversion + +- suite_name: replica_sets_jscore_passthrough_last_continuous_new_new_old + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_continuous_new_new_old" + +- suite_name: replica_sets_jscore_passthrough_last_continuous_new_old_new + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_continuous_new_old_new" + +- suite_name: replica_sets_jscore_passthrough_last_continuous_old_new_new + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_continuous_old_new_new" + +- suite_name: replica_sets_jscore_passthrough_last_lts_new_new_old base_suite: replica_sets_jscore_passthrough overrides: - - "multiversion.replica_fixture" - - "multiversion.test_kind" + - "multiversion.replica_fixture_last_lts_new_new_old" -- suite_name: concurrency_sharded_replication_multiversion +- suite_name: replica_sets_jscore_passthrough_last_lts_new_old_new + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_lts_new_old_new" + +- suite_name: replica_sets_jscore_passthrough_last_lts_old_new_new + base_suite: replica_sets_jscore_passthrough + overrides: + - "multiversion.replica_fixture_last_lts_old_new_new" + + +- suite_name: concurrency_sharded_replication_last_continuous_new_old_old_new base_suite: concurrency_sharded_replication overrides: - - "multiversion.sharded_fixture" + - "multiversion.sharded_fixture_last_continuous_new_old_old_new" - "multiversion.concurrency_sharded_replication_multiversion_mongod_options" -- suite_name: sharding_jscore_multiversion - base_suite: sharding_jscore_passthrough +- suite_name: concurrency_sharded_replication_last_lts_new_old_old_new + base_suite: concurrency_sharded_replication overrides: - - "multiversion.sharded_fixture" - - "multiversion.test_kind" + - "multiversion.sharded_fixture_last_lts_new_old_old_new" + - "multiversion.concurrency_sharded_replication_multiversion_mongod_options" -- suite_name: sharded_collections_jscore_multiversion +- suite_name: sharded_collections_jscore_passthrough_last_continuous_new_old_old_new base_suite: sharded_collections_jscore_passthrough overrides: - - "multiversion.sharded_fixture" - - "multiversion.test_kind" + - "multiversion.sharded_fixture_last_continuous_new_old_old_new" -- suite_name: replica_sets_multiversion +- suite_name: sharded_collections_jscore_passthrough_last_lts_new_old_old_new + base_suite: sharded_collections_jscore_passthrough + overrides: + - "multiversion.sharded_fixture_last_lts_new_old_old_new" + + +- suite_name: sharding_jscore_passthrough_last_continuous_new_old_old_new + base_suite: sharding_jscore_passthrough + overrides: + - "multiversion.sharded_fixture_last_continuous_new_old_old_new" + +- suite_name: sharding_jscore_passthrough_last_lts_new_old_old_new + base_suite: sharding_jscore_passthrough + overrides: + - "multiversion.sharded_fixture_last_lts_new_old_old_new" + + +- suite_name: replica_sets_last_lts base_suite: replica_sets overrides: - "multiversion.replica_sets_multiversion_selector" - "multiversion.replica_sets_multiversion_testdata_last_lts" - - "multiversion.test_kind" -- suite_name: sharding_multiversion +- suite_name: replica_sets_last_continuous + base_suite: replica_sets + overrides: + - "multiversion.replica_sets_multiversion_selector" + - "multiversion.replica_sets_multiversion_testdata_last_continuous" + + +- suite_name: sharding_last_lts base_suite: sharding overrides: - "multiversion.sharding_multiversion_selector" - - "multiversion.sharding_multiversion_testdata" - - "multiversion.test_kind" + - "multiversion.sharding_multiversion_testdata_last_lts" + +- suite_name: sharding_last_continuous + base_suite: sharding + overrides: + - "multiversion.sharding_multiversion_selector" + - "multiversion.sharding_multiversion_testdata_last_continuous" - suite_name: jstestfuzz_replication_last_continuous_new_new_old @@ -66,42 +209,36 @@ overrides: - "multiversion.replica_fixture_last_continuous_new_new_old" - "multiversion.jstestfuzz_replication_multiversion_hooks" - - "multiversion.test_kind" - suite_name: jstestfuzz_replication_last_continuous_new_old_new base_suite: jstestfuzz_replication overrides: - "multiversion.replica_fixture_last_continuous_new_old_new" - "multiversion.jstestfuzz_replication_multiversion_hooks" - - "multiversion.test_kind" - suite_name: jstestfuzz_replication_last_continuous_old_new_new base_suite: jstestfuzz_replication overrides: - "multiversion.replica_fixture_last_continuous_old_new_new" - "multiversion.jstestfuzz_replication_multiversion_hooks" - - "multiversion.test_kind" - suite_name: jstestfuzz_replication_last_lts_new_new_old base_suite: jstestfuzz_replication overrides: - "multiversion.replica_fixture_last_lts_new_new_old" - "multiversion.jstestfuzz_replication_multiversion_hooks" - - "multiversion.test_kind" - suite_name: jstestfuzz_replication_last_lts_new_old_new base_suite: jstestfuzz_replication overrides: - "multiversion.replica_fixture_last_lts_new_old_new" - "multiversion.jstestfuzz_replication_multiversion_hooks" - - "multiversion.test_kind" - suite_name: jstestfuzz_replication_last_lts_old_new_new base_suite: jstestfuzz_replication overrides: - "multiversion.replica_fixture_last_lts_old_new_new" - "multiversion.jstestfuzz_replication_multiversion_hooks" - - "multiversion.test_kind" - suite_name: jstestfuzz_sharded_last_continuous_new_old_old_new @@ -109,63 +246,87 @@ overrides: - "multiversion.sharded_fixture_last_continuous_new_old_old_new" - "multiversion.jstestfuzz_sharded_multiversion_hooks" - - "multiversion.test_kind" - suite_name: jstestfuzz_sharded_last_lts_new_old_old_new base_suite: jstestfuzz_sharded overrides: - "multiversion.sharded_fixture_last_lts_new_old_old_new" - "multiversion.jstestfuzz_sharded_multiversion_hooks" - - "multiversion.test_kind" - suite_name: initial_sync_fuzzer_last_lts base_suite: initial_sync_fuzzer overrides: - "multiversion.replica_sets_multiversion_testdata_last_lts" - - "multiversion.test_kind" - suite_name: initial_sync_fuzzer_last_continuous base_suite: initial_sync_fuzzer overrides: - "multiversion.replica_sets_multiversion_testdata_last_continuous" - - "multiversion.test_kind" - suite_name: generational_fuzzer_last_lts base_suite: generational_fuzzer overrides: - "multiversion.replica_sets_multiversion_testdata_last_lts" - - "multiversion.test_kind" - suite_name: generational_fuzzer_last_continuous base_suite: generational_fuzzer overrides: - "multiversion.replica_sets_multiversion_testdata_last_continuous" - - "multiversion.test_kind" - suite_name: generational_fuzzer_replication_last_lts base_suite: generational_fuzzer_replication overrides: - "multiversion.replica_sets_multiversion_testdata_last_lts" - - "multiversion.test_kind" - suite_name: generational_fuzzer_replication_last_continuous base_suite: generational_fuzzer_replication overrides: - "multiversion.replica_sets_multiversion_testdata_last_continuous" - - "multiversion.test_kind" - suite_name: rollback_fuzzer_last_lts base_suite: rollback_fuzzer overrides: - "multiversion.rollback_multiversion_fuzzer_testdata_last_lts" - - "multiversion.test_kind" - suite_name: rollback_fuzzer_last_continuous base_suite: rollback_fuzzer overrides: - "multiversion.rollback_multiversion_fuzzer_testdata_last_continuous" - - "multiversion.test_kind" + + +- suite_name: sharded_retryable_writes_downgrade_last_continuous_new_old_old_new + base_suite: sharded_retryable_writes_downgrade + overrides: + - "multiversion.sharded_fixture_last_continuous_new_old_old_new" + +- suite_name: sharded_retryable_writes_downgrade_last_lts_new_old_old_new + base_suite: sharded_retryable_writes_downgrade + overrides: + - "multiversion.sharded_fixture_last_lts_new_old_old_new" + + +- suite_name: change_streams_downgrade_last_continuous_new_old_old_new + base_suite: change_streams_downgrade + overrides: + - "multiversion.sharded_fixture_last_continuous_new_old_old_new" + +- suite_name: change_streams_downgrade_last_lts_new_old_old_new + base_suite: change_streams_downgrade + overrides: + - "multiversion.sharded_fixture_last_lts_new_old_old_new" + + +# The downgrade suites only need one starting version. No need for all version combinations. +- suite_name: retryable_writes_downgrade_last_continuous + base_suite: retryable_writes_downgrade + overrides: + - "multiversion.replica_fixture_last_continuous_new_old_new" + +- suite_name: retryable_writes_downgrade_last_lts + base_suite: retryable_writes_downgrade + overrides: + - "multiversion.replica_fixture_last_lts_old_new_new" diff --git a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml index 6273eb23d98..6abcfb72910 100644 --- a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml +++ b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml @@ -1,17 +1,11 @@ ### Overrides for more than 1 suite ### -- name: replica_fixture - value: - executor: - fixture: - num_nodes: 3 - - name: replica_fixture_last_lts_new_new_old value: executor: fixture: num_nodes: 3 old_bin_version: last_lts - mixed_bin_versions: new-new-old + mixed_bin_versions: new_new_old - name: replica_fixture_last_lts_new_old_new value: @@ -19,7 +13,7 @@ fixture: num_nodes: 3 old_bin_version: last_lts - mixed_bin_versions: new-old-new + mixed_bin_versions: new_old_new - name: replica_fixture_last_lts_old_new_new value: @@ -27,7 +21,7 @@ fixture: num_nodes: 3 old_bin_version: last_lts - mixed_bin_versions: old-new-new + mixed_bin_versions: old_new_new - name: replica_fixture_last_continuous_new_new_old value: @@ -35,7 +29,7 @@ fixture: num_nodes: 3 old_bin_version: last_continuous - mixed_bin_versions: new-new-old + mixed_bin_versions: new_new_old - name: replica_fixture_last_continuous_new_old_new value: @@ -43,7 +37,7 @@ fixture: num_nodes: 3 old_bin_version: last_continuous - mixed_bin_versions: new-old-new + mixed_bin_versions: new_old_new - name: replica_fixture_last_continuous_old_new_new value: @@ -51,14 +45,7 @@ fixture: num_nodes: 3 old_bin_version: last_continuous - mixed_bin_versions: old-new-new - -- name: sharded_fixture - value: - executor: - fixture: - num_shards: 2 - num_rs_nodes_per_shard: 2 + mixed_bin_versions: old_new_new - name: sharded_fixture_last_lts_new_old_old_new value: @@ -67,7 +54,7 @@ num_shards: 2 num_rs_nodes_per_shard: 2 old_bin_version: last_lts - mixed_bin_versions: new-old-old-new + mixed_bin_versions: new_old_old_new - name: sharded_fixture_last_continuous_new_old_old_new value: @@ -76,11 +63,7 @@ num_shards: 2 num_rs_nodes_per_shard: 2 old_bin_version: last_continuous - mixed_bin_versions: new-old-old-new - -- name: test_kind - value: - test_kind: all_versions_js_test + mixed_bin_versions: new_old_old_new ### Suite-specific overrides ### @@ -185,7 +168,7 @@ - jstests/sharding/drop_indexes_with_stale_config_error.js - jstests/sharding/query/merge_write_concern.js -- name: sharding_multiversion_testdata +- name: sharding_multiversion_testdata_last_lts value: executor: config: @@ -195,6 +178,16 @@ useRandomBinVersionsWithinReplicaSet: 'last-lts' mongosBinVersion: 'last-lts' +- name: sharding_multiversion_testdata_last_continuous + value: + executor: + config: + shell_options: + global_vars: + TestData: + useRandomBinVersionsWithinReplicaSet: 'last-continuous' + mongosBinVersion: 'last-continuous' + - name: jstestfuzz_replication_multiversion_hooks value: executor: diff --git a/buildscripts/resmokeconfig/matrix_suites/tags.py b/buildscripts/resmokeconfig/matrix_suites/tags.py deleted file mode 100644 index b862e0c531a..00000000000 --- a/buildscripts/resmokeconfig/matrix_suites/tags.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Dynamically Generated tags.""" - - -# TODO SERVER-55857: Let this file be the single source of truth for tags. Remove dupe definitions. -# Base tag for all temporary multiversion exlcusions. Also can be used directly for -# exclusion from all suites. -class Tags(object): - """Wrapper class for tags.""" - - BACKPORT_REQUIRED_TAG = "backport_required_multiversion" - - from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG - # Base exclusion tag list. - EXCLUDE_TAGS_TEMPLATE = f"{REQUIRES_FCV_TAG},multiversion_incompatible,{BACKPORT_REQUIRED_TAG}" - - # TODO SERVER-55857: move this value to remsoke's internal config and move generate_exclude_yaml to resmoke. - # Call generate_exclude_yaml() when fetching fixture files so we only reach out to github once and - # call `mongo --version` once. - EXCLUDE_TAGS_FILE = "multiversion_exclude_tags.yml" diff --git a/buildscripts/resmokeconfig/suites/change_streams_multiversion_downgrade_passthrough.yml b/buildscripts/resmokeconfig/suites/change_streams_downgrade.yml index 3806c0577af..3806c0577af 100644 --- a/buildscripts/resmokeconfig/suites/change_streams_multiversion_downgrade_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/change_streams_downgrade.yml diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_downgrade_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml index c19b4a2fdbd..c19b4a2fdbd 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_downgrade_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml diff --git a/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml index 453214ba77a..453214ba77a 100644 --- a/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml diff --git a/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml index 5a7401296c4..3665df98932 100644 --- a/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml +++ b/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml @@ -7,12 +7,6 @@ selector: - jstests/sharding/query/*.js - jstests/sharding/change_streams/*.js exclude_with_any_tags: - - multiversion_incompatible - # TODO (SERVER-55857): Remove this hardcoded tag once tests are filtered out using the - # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'. - - requires_fcv_51 - - backport_required_multiversion - - sharding_last_lts_mongos_and_mixed_shards_backport_required_multiversion - disabled_due_to_server_60490 exclude_files: # SERVER-33683: We added a restriction on using an aggregation within a transaction against diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index de93f1840f1..21b37a91f35 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -108,7 +108,7 @@ DEFAULTS = { "transport_layer": None, "user_friendly_output": None, "mixed_bin_versions": None, - "old_bin_version": "last_continuous", + "old_bin_version": None, "linear_chain": None, "num_replset_nodes": None, "num_shards": None, @@ -275,6 +275,9 @@ ARCHIVE_LIMIT_MB = None # The limit number of tests to archive for an Evergreen task. ARCHIVE_LIMIT_TESTS = None +# Whether to back up data when restarting a process. +BACKUP_ON_RESTART_DIR = None + # The starting port number to use for mongod and mongos processes spawned by resmoke.py and the # mongo shell. BASE_PORT = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index d32a1e1e758..50da8c1b0f0 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -162,10 +162,11 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many if _config.RUN_ALL_FEATURE_FLAG_TESTS: _config.RUN_ALL_FEATURE_FLAGS = True - all_feature_flags = [] + all_ff = [] enabled_feature_flags = [] try: - all_feature_flags = open(ALL_FEATURE_FLAG_FILE).read().split() + with open(ALL_FEATURE_FLAG_FILE) as fd: + all_ff = fd.read().split() except FileNotFoundError: # If we ask resmoke to run with all feature flags, the feature flags file # needs to exist. @@ -173,7 +174,7 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many raise if _config.RUN_ALL_FEATURE_FLAGS: - enabled_feature_flags = all_feature_flags[:] + enabled_feature_flags = all_ff[:] # Specify additional feature flags from the command line. # Set running all feature flag tests to True if this options is specified. @@ -181,7 +182,7 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many if additional_feature_flags is not None: enabled_feature_flags.extend(additional_feature_flags) - return enabled_feature_flags, all_feature_flags + return enabled_feature_flags, all_ff _config.ENABLED_FEATURE_FLAGS, all_feature_flags = setup_feature_flags() not_enabled_feature_flags = list(set(all_feature_flags) - set(_config.ENABLED_FEATURE_FLAGS)) @@ -246,10 +247,6 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable")) mongod_set_parameters = config.pop("mongod_set_parameters") - # TODO: This should eventually be migrated entirely to _builder.py - if _config.ENABLED_FEATURE_FLAGS and not _config.MIXED_BIN_VERSIONS: - feature_flag_dict = {ff: "true" for ff in _config.ENABLED_FEATURE_FLAGS} - mongod_set_parameters.append(str(feature_flag_dict)) _config.MONGOD_SET_PARAMETERS = _merge_set_params(mongod_set_parameters) _config.FUZZ_MONGOD_CONFIGS = config.pop("fuzz_mongod_configs") diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 8a421e6342e..f9154ff202c 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -136,10 +136,15 @@ def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches else: test_name = None + # the Shell fixtures uses hyphen-delimited versions (e.g. last-lts) while resmoke.py + # uses underscore (e.g. last_lts). resmoke's version is needed as it's part of the task name. + shell_mixed_version = (config.MULTIVERSION_BIN_VERSION or "").replace("_", "-") + shortcut_opts = { "backupOnRestartDir": (config.BACKUP_ON_RESTART_DIR, None), "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True), "mixedBinVersions": (config.MIXED_BIN_VERSIONS, ""), + "multiversionBinVersion": (shell_mixed_version, ""), "noJournal": (config.NO_JOURNAL, False), "storageEngine": (config.STORAGE_ENGINE, ""), "storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""), @@ -172,15 +177,21 @@ def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches mongos_set_parameters = test_data.get("setParametersMongos", {}).copy() mongocryptd_set_parameters = test_data.get("setParametersMongocryptd", {}).copy() + feature_flag_dict = {} + if config.ENABLED_FEATURE_FLAGS is not None: + feature_flag_dict = {ff: "true" for ff in config.ENABLED_FEATURE_FLAGS} + # Propagate additional setParameters to mongod processes spawned by the mongo shell. Command # line options to resmoke.py override the YAML configuration. if config.MONGOD_SET_PARAMETERS is not None: mongod_set_parameters.update(utils.load_yaml(config.MONGOD_SET_PARAMETERS)) + mongod_set_parameters.update(feature_flag_dict) # Propagate additional setParameters to mongos processes spawned by the mongo shell. Command # line options to resmoke.py override the YAML configuration. if config.MONGOS_SET_PARAMETERS is not None: mongos_set_parameters.update(utils.load_yaml(config.MONGOS_SET_PARAMETERS)) + mongos_set_parameters.update(feature_flag_dict) # Propagate additional setParameters to mongocryptd processes spawned by the mongo shell. # Command line options to resmoke.py override the YAML configuration. diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 0d9a3432ea4..303b4a267d6 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -801,19 +801,6 @@ class RunPlugin(PluginInterface): " to run a particular test under a particular suite configuration.") parser.add_argument( - "--mixedBinVersions", type=str, dest="mixed_bin_versions", - metavar="version1-version2-..-versionN", - help="Runs the test with the provided replica set" - " binary version configuration. Specify 'old-new' to configure a replica set with a" - " 'last-lts' version primary and 'latest' version secondary. For a sharded cluster" - " with two shards and two replica set nodes each, specify 'old-new-old-new'.") - - parser.add_argument( - "--oldBinVersion", type=str, dest="old_bin_version", - choices=config.MultiversionOptions.all_options(), - help="Choose the multiversion binary version as last-lts or last-continuous.") - - parser.add_argument( "--linearChain", action="store", dest="linear_chain", choices=("on", "off"), metavar="ON|OFF", help="Enable or disable linear chaining for tests using " "ReplicaSetFixture.") diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py index 709a1373d65..63353d56b3c 100644 --- a/buildscripts/resmokelib/suitesconfig.py +++ b/buildscripts/resmokelib/suitesconfig.py @@ -140,36 +140,40 @@ class SuiteConfigInterface: class ExplicitSuiteConfig(SuiteConfigInterface): """Class for storing the resmoke.py suite YAML configuration.""" + _named_suites = {} + @classmethod def get_config_obj(cls, suite_name): """Get the suite config object in the given file.""" - # Named executors or suites are specified as the basename of the file, without the .yml - # extension. - if not fs.is_yaml_file(suite_name) and not os.path.dirname(suite_name): - named_suites = cls.get_named_suites() - if suite_name not in named_suites: # pylint: disable=unsupported-membership-test - return None - suite_name = named_suites[suite_name] # pylint: disable=unsubscriptable-object - - if not fs.is_yaml_file(suite_name) or not os.path.isfile(suite_name): - raise ValueError("Expected a suite YAML config, but got '%s'" % suite_name) - return utils.load_yaml_file(suite_name) + if suite_name in cls.get_named_suites(): + # Check if is a named suite first for efficiency. + suite_path = cls.get_named_suites()[suite_name] + elif fs.is_yaml_file(suite_name): + # Check if is a path to a YAML file. + if os.path.isfile(suite_name): + suite_path = suite_name + else: + raise ValueError("Expected a suite YAML config, but got '%s'" % suite_name) + else: + # Not an explicit suite, return None. + return None + + return utils.load_yaml_file(suite_path) @classmethod def get_named_suites(cls) -> Dict[str, str]: """Populate the named suites by scanning config_dir/suites.""" - named_suites = {} - - suites_dir = os.path.join(_config.CONFIG_DIR, "suites") - root = os.path.abspath(suites_dir) - files = os.listdir(root) - for filename in files: - (short_name, ext) = os.path.splitext(filename) - if ext in (".yml", ".yaml"): - pathname = os.path.join(root, filename) - named_suites[short_name] = pathname - - return named_suites + if not cls._named_suites: + suites_dir = os.path.join(_config.CONFIG_DIR, "suites") + root = os.path.abspath(suites_dir) + files = os.listdir(root) + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml"): + pathname = os.path.join(root, filename) + cls._named_suites[short_name] = pathname + + return cls._named_suites @classmethod def get_suite_files(cls): @@ -180,7 +184,8 @@ class ExplicitSuiteConfig(SuiteConfigInterface): class MatrixSuiteConfig(SuiteConfigInterface): """Class for storing the resmoke.py suite YAML configuration.""" - _all_mappings = None + _all_mappings = {} + _all_overrides = {} @staticmethod def get_all_yamls(target_dir): @@ -240,20 +245,19 @@ class MatrixSuiteConfig(SuiteConfigInterface): @classmethod def parse_override_file(cls, suites_dir): """Get a dictionary of all overrides in a given directory keyed by the suite name.""" - overrides_dir = os.path.join(suites_dir, "overrides") - overrides_files = cls.get_all_yamls(overrides_dir) - - all_overrides = {} - for filename, override_config_file in overrides_files.items(): - for override_config in override_config_file: - if "name" in override_config and "value" in override_config: - all_overrides[f"{filename}.{override_config['name']}"] = override_config[ - "value"] - else: - raise ValueError("Invalid override configuration, missing required keys. ", - override_config) - - return all_overrides + if not cls._all_overrides: + overrides_dir = os.path.join(suites_dir, "overrides") + overrides_files = cls.get_all_yamls(overrides_dir) + + for filename, override_config_file in overrides_files.items(): + for override_config in override_config_file: + if "name" in override_config and "value" in override_config: + cls._all_overrides[ + f"{filename}.{override_config['name']}"] = override_config["value"] + else: + raise ValueError("Invalid override configuration, missing required keys. ", + override_config) + return cls._all_overrides @classmethod def parse_mappings_file(cls, suites_dir, suite_name): @@ -280,19 +284,17 @@ class MatrixSuiteConfig(SuiteConfigInterface): @classmethod def get_all_mappings(cls, suites_dir) -> Dict[str, str]: """Get a dictionary of all suite mapping files keyed by the suite name.""" - if cls._all_mappings is None: + if not cls._all_mappings: mappings_dir = os.path.join(suites_dir, "mappings") mappings_files = cls.get_all_yamls(mappings_dir) - all_mappings = {} for _, suite_config_file in mappings_files.items(): for suite_config in suite_config_file: if "suite_name" in suite_config and "base_suite" in suite_config: - all_mappings[suite_config["suite_name"]] = suite_config + cls._all_mappings[suite_config["suite_name"]] = suite_config else: raise ValueError("Invalid suite configuration, missing required keys. ", suite_config) - cls._all_mappings = all_mappings return cls._all_mappings @classmethod diff --git a/buildscripts/resmokelib/testing/fixtures/_builder.py b/buildscripts/resmokelib/testing/fixtures/_builder.py index 19bc0c103c0..fc4215b807e 100644 --- a/buildscripts/resmokelib/testing/fixtures/_builder.py +++ b/buildscripts/resmokelib/testing/fixtures/_builder.py @@ -73,10 +73,13 @@ class ReplSetBuilder(FixtureBuilder): # We have the same code in configure_resmoke.py to split config.MIXED_BIN_VERSIONS, # but here it is for the case, when it comes from resmoke suite definition if isinstance(mixed_bin_versions, str): - mixed_bin_versions = mixed_bin_versions.split("-") + mixed_bin_versions = mixed_bin_versions.split("_") if config.MIXED_BIN_VERSIONS is None: config.MIXED_BIN_VERSIONS = mixed_bin_versions + old_bin_version = kwargs.pop("old_bin_version", config.MULTIVERSION_BIN_VERSION) + if config.MULTIVERSION_BIN_VERSION is None: + config.MULTIVERSION_BIN_VERSION = old_bin_version # We also hijack the num_nodes because we need it here. num_nodes = kwargs.pop("num_nodes", 2) @@ -283,10 +286,14 @@ class ShardedClusterBuilder(FixtureBuilder): # We have the same code in configure_resmoke.py to split config.MIXED_BIN_VERSIONS, # but here it is for the case, when it comes from resmoke suite definition if isinstance(mixed_bin_versions, str): - mixed_bin_versions = mixed_bin_versions.split("-") + mixed_bin_versions = mixed_bin_versions.split("_") if config.MIXED_BIN_VERSIONS is None: config.MIXED_BIN_VERSIONS = mixed_bin_versions + old_bin_version = kwargs.pop("old_bin_version", config.MULTIVERSION_BIN_VERSION) + if config.MULTIVERSION_BIN_VERSION is None: + config.MULTIVERSION_BIN_VERSION = old_bin_version + is_multiversion = mixed_bin_versions is not None num_shards = kwargs.pop("num_shards", 1) diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py index 068f9e1b939..9bbcbba442e 100644 --- a/buildscripts/selected_tests.py +++ b/buildscripts/selected_tests.py @@ -406,6 +406,7 @@ class SelectedTestsOrchestrator: use_large_distro=task_config.get("use_large_distro", False), large_distro_name=task_config.get("large_distro_name"), require_multiversion_setup=task_def.require_multiversion_setup(), + require_multiversion_version_combo=False, repeat_suites=task_config.get("repeat_suites", 1), resmoke_args=task_config["resmoke_args"], resmoke_jobs_max=task_config.get("resmoke_jobs_max"), diff --git a/buildscripts/task_generation/evg_config_builder.py b/buildscripts/task_generation/evg_config_builder.py index c3fc3cb403a..db28a45f570 100644 --- a/buildscripts/task_generation/evg_config_builder.py +++ b/buildscripts/task_generation/evg_config_builder.py @@ -15,8 +15,9 @@ from buildscripts.task_generation.suite_split import SuiteSplitService, Generate SuiteSplitParameters from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerTask - # pylint: disable=too-many-instance-attributes + + class EvgConfigBuilder: """A builder class for building evergreen configuration.""" @@ -59,16 +60,6 @@ class EvgConfigBuilder: self.build_variants[build_variant] = BuildVariant(build_variant, activate=False) return self.build_variants[build_variant] - def _generate_suites_config(self, generated_suite: GeneratedSuite) -> List[GeneratedFile]: - """ - Generate the suites files and evergreen configuration for the generated task. - - :param generated_suite: Generated suite to create config files for. - :return: The suites files and evergreen configuration for the generated task. - """ - return self.resmoke_proxy.render_suite_files(generated_suite, - self.gen_options.create_misc_suite) - def generate_suite(self, split_params: SuiteSplitParameters, gen_params: ResmokeGenTaskParams) -> None: """ @@ -80,8 +71,9 @@ class EvgConfigBuilder: generated_suite = self.suite_split_service.split_suite(split_params) with self.lock: build_variant = self.get_build_variant(generated_suite.build_variant) - self.evg_config_gen_service.generate_task(generated_suite, build_variant, gen_params) - self.generated_files.extend(self._generate_suites_config(generated_suite)) + resmoke_tasks = self.evg_config_gen_service.generate_task(generated_suite, + build_variant, gen_params) + self.generated_files.extend(self.resmoke_proxy.render_suite_files(resmoke_tasks)) def generate_fuzzer(self, fuzzer_params: FuzzerGenTaskParams) -> FuzzerTask: """ diff --git a/buildscripts/task_generation/gen_task_service.py b/buildscripts/task_generation/gen_task_service.py index 0f6d9c240ee..50b0cec2b48 100644 --- a/buildscripts/task_generation/gen_task_service.py +++ b/buildscripts/task_generation/gen_task_service.py @@ -18,6 +18,7 @@ from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskPa from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions from buildscripts.task_generation.task_types.resmoke_tasks import ResmokeGenTaskParams, \ ResmokeGenTaskService +from buildscripts.task_generation.task_types.models.resmoke_task_model import ResmokeTask from buildscripts.task_generation.gen_config import GenerationConfiguration from buildscripts.task_generation.suite_split import GeneratedSuite # pylint: enable=wrong-import-position @@ -80,7 +81,7 @@ class GenTaskService: return fuzzer_task def generate_task(self, generated_suite: GeneratedSuite, build_variant: BuildVariant, - gen_params: ResmokeGenTaskParams) -> Set[Task]: + gen_params: ResmokeGenTaskParams) -> List[ResmokeTask]: """ Generate evergreen configuration for the given suite and add it to the build_variant. @@ -91,7 +92,8 @@ class GenTaskService: execution_tasks = self.resmoke_gen_task_service.generate_tasks(generated_suite, gen_params) distros = self._get_distro(build_variant.name, gen_params.use_large_distro, gen_params.large_distro_name) - build_variant.display_task(generated_suite.task_name, execution_tasks=execution_tasks, + build_variant.display_task(generated_suite.task_name, + execution_tasks=[task.shrub_task for task in execution_tasks], distros=distros, activate=False) return execution_tasks diff --git a/buildscripts/task_generation/resmoke_proxy.py b/buildscripts/task_generation/resmoke_proxy.py index d9e521a328b..81a3f56f962 100644 --- a/buildscripts/task_generation/resmoke_proxy.py +++ b/buildscripts/task_generation/resmoke_proxy.py @@ -1,7 +1,6 @@ """A service to proxy requests to resmoke.""" -import os from copy import deepcopy -from typing import List, Dict, Any, NamedTuple, TYPE_CHECKING +from typing import List, Dict, Any, NamedTuple, TYPE_CHECKING, Set import inject import structlog @@ -10,6 +9,7 @@ import yaml import buildscripts.resmokelib.parser as _parser import buildscripts.resmokelib.suitesconfig as _suiteconfig from buildscripts.task_generation.generated_config import GeneratedFile +from buildscripts.task_generation.task_types.models.resmoke_task_model import ResmokeTask if TYPE_CHECKING: from buildscripts.task_generation.suite_split import GeneratedSuite, SubSuite @@ -58,38 +58,29 @@ class ResmokeProxyService: """ return self._suite_config.SuiteFinder.get_config_obj(suite_name) - def render_suite_files(self, suite: "GeneratedSuite", - create_misc_suite: bool) -> List[GeneratedFile]: + def render_suite_files(self, tasks: List[ResmokeTask]) -> List[GeneratedFile]: """ Render the given list of suites. This will create a dictionary of all the resmoke config files to create with the filename of each file as the key and the contents as the value. - :param suite: Generated suite files belong to. - :param create_misc_suite: Whether to create the file for the _misc task. + :param tasks: resmoke tasks to generate config files for. :return: Dictionary of rendered resmoke config files. """ + suite_configs = [] # pylint: disable=too-many-arguments - original_suite_name = suite.suite_name - test_list = suite.get_test_list() - - source_config = self._suite_config.SuiteFinder.get_config_obj(original_suite_name) - suite_configs = [ - GeneratedFile( - file_name=suite.sub_suite_config_file(i), - content=sub_suite.generate_resmoke_config(source_config)) - for i, sub_suite in enumerate(suite.sub_suites) - ] - if create_misc_suite: + for resmoke_task in tasks: + source_config = self._suite_config.SuiteFinder.get_config_obj( + resmoke_task.resmoke_suite_name) suite_configs.append( GeneratedFile( - file_name=suite.sub_suite_config_file(None), - content=generate_resmoke_suite_config(source_config, original_suite_name, - excludes=test_list))) + file_name=resmoke_task.execution_task_suite_yaml_name, + content=generate_resmoke_suite_config( + source_config, resmoke_task.resmoke_suite_name, + roots=resmoke_task.test_list, excludes=resmoke_task.excludes))) - LOGGER.debug("Generated files for suite", suite=original_suite_name, - files=[f.file_name for f in suite_configs]) + LOGGER.debug("Generated files", files=[f.file_name for f in suite_configs]) return suite_configs diff --git a/buildscripts/task_generation/suite_split.py b/buildscripts/task_generation/suite_split.py index 555c449f655..e0b3cfbb449 100644 --- a/buildscripts/task_generation/suite_split.py +++ b/buildscripts/task_generation/suite_split.py @@ -2,7 +2,6 @@ from __future__ import annotations import os -from copy import deepcopy from datetime import datetime from itertools import chain from typing import NamedTuple, Callable, Optional, List, Dict, Any @@ -10,14 +9,12 @@ from typing import NamedTuple, Callable, Optional, List, Dict, Any import inject import requests import structlog -import yaml from evergreen import EvergreenApi from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy from buildscripts.task_generation.timeout import TimeoutEstimate from buildscripts.util import taskname -from buildscripts.util.taskname import remove_gen_suffix from buildscripts.util.teststats import HistoricTaskData, TestRuntime, normalize_test_name LOGGER = structlog.getLogger(__name__) @@ -27,37 +24,8 @@ HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost. # This file was generated by {file}. """ -# pylint: disable=too-many-arguments - - -def update_suite_config(suite_config, roots=None, excludes=None): - """ - Update suite config based on the roots and excludes passed in. - - :param suite_config: suite_config to update. - :param roots: new roots to run, or None if roots should not be updated. - :param excludes: excludes to add, or None if excludes should not be include. - :return: updated suite_config - """ - if roots: - suite_config["selector"]["roots"] = roots - - if excludes: - # This must be a misc file, if the exclude_files section exists, extend it, otherwise, - # create it. - if "exclude_files" in suite_config["selector"] and \ - suite_config["selector"]["exclude_files"]: - suite_config["selector"]["exclude_files"] += excludes - else: - suite_config["selector"]["exclude_files"] = excludes - else: - # if excludes was not specified this must not a misc file, so don"t exclude anything. - if "exclude_files" in suite_config["selector"]: - del suite_config["selector"]["exclude_files"] - - return suite_config - +# pylint: disable=too-many-arguments class SubSuite(object): """A suite of tests that can be run by evergreen.""" @@ -104,18 +72,6 @@ class SubSuite(object): def __len__(self) -> int: return len(self.test_list) - def generate_resmoke_config(self, source_config: Dict) -> str: - """ - Generate the contents of resmoke config for this suite. - - :param source_config: Resmoke config to base generate config on. - :return: Resmoke config to run this suite. - """ - suite_config = update_suite_config(deepcopy(source_config), roots=self.test_list) - contents = HEADER_TEMPLATE.format(file=__file__) - contents += yaml.safe_dump(suite_config, default_flow_style=False) - return contents - class GeneratedSuite(NamedTuple): """ @@ -161,6 +117,10 @@ class GeneratedSuite(NamedTuple): return taskname.name_generated_task(self.task_name, index, len(self.sub_suites), self.build_variant) + def sub_suite_test_list(self, index: int) -> List[str]: + """Get the list of tests from a sub-suite.""" + return self.sub_suites[index].test_list + class SuiteSplitParameters(NamedTuple): """ diff --git a/buildscripts/task_generation/task_types/fuzzer_tasks.py b/buildscripts/task_generation/task_types/fuzzer_tasks.py index 869835315b3..61ce29527f3 100644 --- a/buildscripts/task_generation/task_types/fuzzer_tasks.py +++ b/buildscripts/task_generation/task_types/fuzzer_tasks.py @@ -6,7 +6,8 @@ from shrub.v2 import Task, FunctionCall, TaskDependency from buildscripts.patch_builds.task_generation import TimeoutInfo from buildscripts.task_generation.constants import ARCHIVE_DIST_TEST_DEBUG_TASK, CONFIGURE_EVG_CREDENTIALS, \ RUN_GENERATED_TESTS -from buildscripts.task_generation.task_types.multiversion_decorator import MultiversionGenTaskDecorator +from buildscripts.task_generation.task_types.multiversion_decorator import MultiversionGenTaskDecorator, \ + MultiversionDecoratorParams from buildscripts.util import taskname _MINIMIZABLE_FUZZERS = ["agg-fuzzer", "query-fuzzer"] @@ -98,7 +99,14 @@ class FuzzerGenTaskService: for index in range(params.num_tasks)}) if params.require_multiversion_setup: - sub_tasks = self.multiversion_decorator.decorate_tasks(sub_tasks, params) + mv_params = MultiversionDecoratorParams( + base_suite=params.suite, + task=params.task_name, + variant=params.variant, + num_tasks=params.num_tasks, + ) + sub_tasks = self.multiversion_decorator.decorate_tasks_with_dynamically_generated_files( + sub_tasks, mv_params) return FuzzerTask(task_name=params.task_name, sub_tasks=sub_tasks) diff --git a/buildscripts/task_generation/task_types/models/__init__.py b/buildscripts/task_generation/task_types/models/__init__.py new file mode 100644 index 00000000000..59fbf55ea38 --- /dev/null +++ b/buildscripts/task_generation/task_types/models/__init__.py @@ -0,0 +1 @@ +"""Empty file.""" diff --git a/buildscripts/task_generation/task_types/models/resmoke_task_model.py b/buildscripts/task_generation/task_types/models/resmoke_task_model.py new file mode 100644 index 00000000000..c1b49fb4345 --- /dev/null +++ b/buildscripts/task_generation/task_types/models/resmoke_task_model.py @@ -0,0 +1,16 @@ +"""Model mapping resmoke suites to tasks.""" +from typing import NamedTuple, Optional, List + +from shrub.v2 import Task + + +class ResmokeTask(NamedTuple): + """Wrapper around shrub.py Task objects with tighter integration with resmoke.py.""" + + shrub_task: Task + resmoke_suite_name: str + # Path to the generated file does not include the generated config directory. + execution_task_suite_yaml_path: str + execution_task_suite_yaml_name: str + test_list: Optional[List[str]] + excludes: Optional[List[str]] diff --git a/buildscripts/task_generation/task_types/multiversion_decorator.py b/buildscripts/task_generation/task_types/multiversion_decorator.py index 36d8bcfd395..f7d6cd8976a 100644 --- a/buildscripts/task_generation/task_types/multiversion_decorator.py +++ b/buildscripts/task_generation/task_types/multiversion_decorator.py @@ -1,6 +1,7 @@ """Multiversion decorator for basic generated tasks.""" import copy -from typing import List, Set, Union, Optional +import os.path +from typing import List, Set, Union, Optional, NamedTuple import inject import structlog @@ -9,6 +10,7 @@ from shrub.v2.command import ShrubCommand from buildscripts.task_generation.constants import DO_MULTIVERSION_SETUP, CONFIGURE_EVG_CREDENTIALS, RUN_GENERATED_TESTS from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService +from buildscripts.task_generation.task_types.models.resmoke_task_model import ResmokeTask from buildscripts.util import taskname LOGGER = structlog.get_logger(__name__) @@ -23,6 +25,15 @@ class _SuiteFixtureType: OTHER = "other" +class MultiversionDecoratorParams(NamedTuple): + """Parameters for converting tasks into multiversion tasks.""" + + base_suite: str + task: str + variant: str + num_tasks: int + + class MultiversionGenTaskDecorator: """Multiversion decorator for basic generated tasks.""" @@ -33,9 +44,16 @@ class MultiversionGenTaskDecorator: self.resmoke_proxy = resmoke_proxy self.old_versions = self._init_old_versions() - def decorate_tasks(self, sub_tasks: Set[Task], params) -> Set[Task]: - """Make multiversion subtasks based on generated subtasks.""" - fixture_type = self._get_suite_fixture_type(params.suite) + def decorate_tasks_with_dynamically_generated_files( + self, sub_tasks: Set[Task], params: MultiversionDecoratorParams) -> Set[Task]: + """ + Make multiversion subtasks based on generated subtasks, for tasks with generated files. E.g. fuzzers. + + @param sub_tasks: set of existing sub-tasks to be converted to multiversion. + @param params: decoration parameters. + @return: Set of multiversion tasks. + """ + fixture_type = self._get_suite_fixture_type(params.base_suite) versions_combinations = self._get_versions_combinations(fixture_type) decorated_tasks = set() @@ -43,20 +61,86 @@ class MultiversionGenTaskDecorator: for mixed_bin_versions in versions_combinations: for index, sub_task in enumerate(sub_tasks): commands = list(sub_task.commands) - base_task_name = self._build_name(params.task_name, old_version, - mixed_bin_versions.replace("-", "_")) + base_task_name = self._build_name(params.task, old_version, mixed_bin_versions) sub_task_name = taskname.name_generated_task(base_task_name, index, params.num_tasks, params.variant) - suite_name = self._build_name(params.suite, old_version, - mixed_bin_versions.replace("-", "_")) - self._update_suite_name(commands, suite_name) + suite_name = self._build_name(params.base_suite, old_version, + mixed_bin_versions) + self._update_execution_task_suite_info(commands, suite_name, old_version) commands = self._add_multiversion_commands(commands) decorated_tasks.add( Task(name=sub_task_name, commands=commands, dependencies=sub_task.dependencies)) return decorated_tasks - def _init_old_versions(self) -> List[str]: + def decorate_tasks_with_explicit_files( + self, sub_tasks: List[ResmokeTask], + params: MultiversionDecoratorParams) -> List[ResmokeTask]: + """ + Make multiversion subtasks based on generated subtasks for explicit tasks. + + Explicit tasks need to have new resmoke.py suite files created for each one with a + unique list of test roots. + """ + fixture_type = self._get_suite_fixture_type(params.base_suite) + versions_combinations = self._get_versions_combinations(fixture_type) + + decorated_tasks = [] + for old_version in self.old_versions: + for mixed_bin_versions in versions_combinations: + for index, sub_task in enumerate(sub_tasks): + shrub_task = sub_task.shrub_task + commands = list(shrub_task.commands) + + # Decorate the task name. + base_task_name = self._build_name(params.task, old_version, mixed_bin_versions) + sub_task_name = taskname.name_generated_task(base_task_name, index, + params.num_tasks, params.variant) + + # Decorate the suite name + resmoke_suite_name = self._build_name(sub_task.resmoke_suite_name, old_version, + mixed_bin_versions) + execution_task_suite_name = taskname.name_generated_task( + resmoke_suite_name, index, params.num_tasks) + execution_task_suite_yaml_dir = os.path.dirname( + sub_task.execution_task_suite_yaml_path) + execution_task_suite_yaml_file = f"{execution_task_suite_name}.yml" + execution_task_suite_yaml_path = os.path.join(execution_task_suite_yaml_dir, + execution_task_suite_yaml_file) + + # Decorate the command invocation options. + self._update_execution_task_suite_info(commands, execution_task_suite_yaml_path, + old_version) + commands = self._add_multiversion_commands(commands) + + # Store the result. + shrub_task = Task(name=sub_task_name, commands=commands, + dependencies=shrub_task.dependencies) + decorated_tasks.append( + ResmokeTask(shrub_task=shrub_task, resmoke_suite_name=resmoke_suite_name, + execution_task_suite_yaml_name=execution_task_suite_yaml_file, + execution_task_suite_yaml_path=execution_task_suite_yaml_path, + test_list=sub_task.test_list, excludes=sub_task.excludes)) + + return decorated_tasks + + def decorate_single_multiversion_tasks(self, sub_tasks: List[ResmokeTask]): + """Decorate a multiversion version of a task without all multiversion combinations.""" + decorated_sub_tasks = [] + for sub_task in sub_tasks: + shrub_task = sub_task.shrub_task + commands = self._add_multiversion_commands(shrub_task.commands) + shrub_task = Task(name=shrub_task.name, commands=commands, + dependencies=shrub_task.dependencies) + decorated_sub_tasks.append( + ResmokeTask(shrub_task=shrub_task, resmoke_suite_name=sub_task.resmoke_suite_name, + execution_task_suite_yaml_path=sub_task.execution_task_suite_yaml_path, + execution_task_suite_yaml_name=sub_task.execution_task_suite_yaml_name, + test_list=sub_task.test_list, excludes=sub_task.excludes)) + return decorated_sub_tasks + + @staticmethod + def _init_old_versions() -> List[str]: from buildscripts.resmokelib import multiversionconstants if multiversionconstants.LAST_LTS_FCV == multiversionconstants.LAST_CONTINUOUS_FCV: LOGGER.debug("Last-lts FCV and last-continuous FCV are equal") @@ -78,35 +162,40 @@ class MultiversionGenTaskDecorator: return _SuiteFixtureType.REPL return _SuiteFixtureType.OTHER - def _get_versions_combinations(self, fixture_type: str) -> List[str]: + @staticmethod + def _get_versions_combinations(fixture_type: str) -> List[str]: return { _SuiteFixtureType.SHELL: [""], - _SuiteFixtureType.SHARD: ["new-old-old-new"], - _SuiteFixtureType.REPL: ["new-new-old", "new-old-new", "old-new-new"], + _SuiteFixtureType.SHARD: ["new_old_old_new"], + _SuiteFixtureType.REPL: ["new_new_old", "new_old_new", "old_new_new"], _SuiteFixtureType.OTHER: [""], }[fixture_type] - def _build_name(self, base_name: str, *suffixes: str) -> str: - parts = [base_name] - parts.extend(suffixes) - return "_".join(part for part in parts if part != "") + @staticmethod + def _build_name(base_name: str, old_version: str, mixed_bin_versions: str) -> str: + return "_".join(part for part in [base_name, old_version, mixed_bin_versions] if part) - def _find_command(self, name: str, commands: List[Union[FunctionCall, ShrubCommand]] + @staticmethod + def _find_command(name: str, commands: List[Union[FunctionCall, ShrubCommand]] ) -> (Optional[int], Optional[FunctionCall]): for index, command in enumerate(commands): if hasattr(command, "name") and command.name == name: return index, command return None, None - def _update_suite_name(self, commands: List[Union[FunctionCall, ShrubCommand]], - suite_name: str): + def _update_execution_task_suite_info(self, commands: List[Union[FunctionCall, ShrubCommand]], + multiversion_suite_path: str, old_bin_version: str): index, run_test_func = self._find_command(RUN_GENERATED_TESTS, commands) if run_test_func is not None: run_test_vars = copy.deepcopy(run_test_func.parameters) - run_test_vars["suite"] = suite_name + run_test_vars["suite"] = multiversion_suite_path + run_test_vars["multiversion_exclude_tags_version"] = old_bin_version commands[index] = FunctionCall(RUN_GENERATED_TESTS, run_test_vars) + return run_test_vars["suite"] + return None - def _add_multiversion_commands(self, commands: List[Union[FunctionCall, ShrubCommand]] + @staticmethod + def _add_multiversion_commands(commands: List[Union[FunctionCall, ShrubCommand]] ) -> List[Union[FunctionCall, ShrubCommand]]: res = [ FunctionCall("git get project no modules"), diff --git a/buildscripts/task_generation/task_types/resmoke_tasks.py b/buildscripts/task_generation/task_types/resmoke_tasks.py index 36afe72c54f..43a62e346cd 100644 --- a/buildscripts/task_generation/task_types/resmoke_tasks.py +++ b/buildscripts/task_generation/task_types/resmoke_tasks.py @@ -3,14 +3,16 @@ from typing import Set, Any, Dict, NamedTuple, Optional, List import inject import structlog -from shrub.v2 import Task, TaskDependency +from shrub.v2 import Task, TaskDependency, FunctionCall -from buildscripts.patch_builds.task_generation import resmoke_commands from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG from buildscripts.task_generation.constants import ARCHIVE_DIST_TEST_DEBUG_TASK, EXCLUDES_TAGS_FILE_PATH, \ - BACKPORT_REQUIRED_TAG + BACKPORT_REQUIRED_TAG, CONFIGURE_EVG_CREDENTIALS, RUN_GENERATED_TESTS from buildscripts.task_generation.suite_split import GeneratedSuite from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions +from buildscripts.task_generation.task_types.models.resmoke_task_model import ResmokeTask +from buildscripts.task_generation.task_types.multiversion_decorator import MultiversionGenTaskDecorator, \ + MultiversionDecoratorParams from buildscripts.task_generation.timeout import TimeoutEstimate LOGGER = structlog.getLogger(__name__) @@ -35,6 +37,7 @@ class ResmokeGenTaskParams(NamedTuple): use_large_distro: Whether generated tasks should be run on a "large" distro. require_multiversion_setup: Requires downloading Multiversion binaries. + require_multiversion_version_combo: Requires combination of multiversion tasks. repeat_suites: How many times generated suites should be repeated. resmoke_args: Arguments to pass to resmoke in generated tasks. resmoke_jobs_max: Max number of jobs that resmoke should execute in parallel. @@ -45,10 +48,11 @@ class ResmokeGenTaskParams(NamedTuple): use_large_distro: bool large_distro_name: Optional[str] require_multiversion_setup: Optional[bool] + require_multiversion_version_combo: Optional[bool] repeat_suites: int resmoke_args: str resmoke_jobs_max: Optional[int] - config_location: Optional[str] + config_location: str class ResmokeGenTaskService: @@ -62,32 +66,38 @@ class ResmokeGenTaskService: :param gen_task_options: Global options for how tasks should be generated. """ self.gen_task_options = gen_task_options + self.multiversion_decorator = MultiversionGenTaskDecorator() # pylint: disable=no-value-for-parameter def generate_tasks(self, generated_suite: GeneratedSuite, - params: ResmokeGenTaskParams) -> Set[Task]: + params: ResmokeGenTaskParams) -> List[ResmokeTask]: """ Build a set of shrub task for all the sub tasks. :param generated_suite: Suite to generate tasks for. :param params: Parameters describing how tasks should be generated. - :return: Set of shrub tasks to generate the given suite. + :return: Set of ResmokeTask objects describing the tasks. """ - tasks = { + tasks = [ self._create_sub_task(index, suite.get_timeout_estimate(), generated_suite, params) for index, suite in enumerate(generated_suite.sub_suites) - } + ] if self.gen_task_options.create_misc_suite: - tasks.add( + tasks.append( self._create_sub_task(None, est_timeout=TimeoutEstimate.no_timeouts(), suite=generated_suite, params=params)) - if params.require_multiversion_setup: - # do the same thing as the fuzzer taks. - pass + if params.require_multiversion_version_combo: + mv_params = MultiversionDecoratorParams( + base_suite=generated_suite.suite_name, task=generated_suite.task_name, + variant=generated_suite.build_variant, num_tasks=len(tasks)) + tasks = self.multiversion_decorator.decorate_tasks_with_explicit_files(tasks, mv_params) + elif params.require_multiversion_setup: + tasks = self.multiversion_decorator.decorate_single_multiversion_tasks(tasks) + return tasks def _create_sub_task(self, index: Optional[int], est_timeout: TimeoutEstimate, - suite: GeneratedSuite, params: ResmokeGenTaskParams) -> Task: + suite: GeneratedSuite, params: ResmokeGenTaskParams) -> ResmokeTask: """ Create the sub task for the given suite. @@ -95,46 +105,54 @@ class ResmokeGenTaskService: :param est_timeout: timeout estimate. :param suite: Parent suite being created. :param params: Parameters describing how tasks should be generated. - :return: Shrub configuration for the sub-suite. + :return: ResmokeTask object describing the task. """ return self._generate_task( - suite.sub_suite_config_file(index), suite.sub_suite_task_name(index), est_timeout, - params, suite) + suite.sub_suite_config_file(index), suite.sub_suite_task_name(index), + [] if index is None else suite.sub_suite_test_list(index), est_timeout, params, suite, + excludes=suite.get_test_list() if index is None else []) - def _generate_task(self, sub_suite_file, sub_task_name: str, timeout_est: TimeoutEstimate, - params: ResmokeGenTaskParams, suite: GeneratedSuite) -> Task: + def _generate_task(self, sub_suite_file, sub_task_name: str, sub_suite_roots: List[str], + timeout_est: TimeoutEstimate, params: ResmokeGenTaskParams, + suite: GeneratedSuite, excludes: List[str]) -> ResmokeTask: """ Generate a shrub evergreen config for a resmoke task. :param sub_suite_file: Name of the suite file to run in the generated task. :param sub_task_name: Name of task to generate. + :param sub_suite_roots: List of files to run. :param timeout_est: Estimated runtime to use for calculating timeouts. :param params: Parameters describing how tasks should be generated. :param suite: Parent suite being created. - :return: Shrub configuration for the described task. + :param excludes: List of tests to exclude. + :return: ResmokeTask object describing the task. """ # pylint: disable=too-many-arguments LOGGER.debug("Generating task running suite", sub_task_name=sub_task_name, sub_suite_file=sub_suite_file) - # Some splits don't generate new physical config files. - if params.config_location is not None: - sub_suite_file_path = self.gen_task_options.suite_location(sub_suite_file) - else: - sub_suite_file_path = None + sub_suite_file_path = self.gen_task_options.suite_location(sub_suite_file) run_tests_vars = self._get_run_tests_vars(sub_suite_file_path=sub_suite_file_path, suite_file=suite.suite_name, task_name=suite.task_name, params=params) - require_multiversion_setup = params.require_multiversion_setup - timeout_cmd = timeout_est.generate_timeout_cmd(self.gen_task_options.is_patch, - params.repeat_suites, - self.gen_task_options.use_default_timeouts) - commands = resmoke_commands("run generated tests", run_tests_vars, timeout_cmd, - require_multiversion_setup) + timeout_info = timeout_est.generate_timeout_cmd(self.gen_task_options.is_patch, + params.repeat_suites, + self.gen_task_options.use_default_timeouts) + + commands = [ + timeout_info.cmd, + FunctionCall("do setup"), + FunctionCall(CONFIGURE_EVG_CREDENTIALS), + FunctionCall(RUN_GENERATED_TESTS, run_tests_vars), + ] - return Task(sub_task_name, commands, self._get_dependencies()) + shrub_task = Task(sub_task_name, [cmd for cmd in commands if cmd], self._get_dependencies()) + return ResmokeTask(shrub_task=shrub_task, resmoke_suite_name=suite.suite_name, + execution_task_suite_yaml_path=sub_suite_file_path, + execution_task_suite_yaml_name=sub_suite_file, test_list=sub_suite_roots, + excludes=excludes) @staticmethod def generate_resmoke_args(original_suite: str, task_name: str, @@ -149,7 +167,10 @@ class ResmokeGenTaskService: :return: arguments to pass to resmoke. """ - resmoke_args = f"--originSuite={original_suite} {params.resmoke_args}" + resmoke_args = f"--originSuite={original_suite}" + + if params.resmoke_args is not None: + resmoke_args += params.resmoke_args if params.repeat_suites and not string_contains_any_of_args(resmoke_args, ["repeatSuites", "repeat"]): diff --git a/buildscripts/tests/task_generation/task_types/test_multiversion_decorator.py b/buildscripts/tests/task_generation/task_types/test_multiversion_decorator.py index 6d4951e1b23..45f485e9194 100644 --- a/buildscripts/tests/task_generation/task_types/test_multiversion_decorator.py +++ b/buildscripts/tests/task_generation/task_types/test_multiversion_decorator.py @@ -7,29 +7,13 @@ import inject from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService from buildscripts.task_generation.task_types import multiversion_decorator as under_test -from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskParams +from buildscripts.task_generation.task_types.multiversion_decorator import MultiversionDecoratorParams # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access,no-value-for-parameter -def build_mock_fuzzer_params(): - return FuzzerGenTaskParams( - task_name="task_name", - variant="build_variant", - suite="resmoke_suite", - num_files=10, - num_tasks=5, - resmoke_args="args for resmoke", - npm_command="jstestfuzz", - jstestfuzz_vars="vars for jstestfuzz", - continue_on_failure=True, - resmoke_jobs_max=5, - should_shuffle=True, - timeout_secs=100, - require_multiversion_setup=True, - use_large_distro=None, - large_distro_name="large_distro", - config_location="config_location", - ) +def build_mock_params(): + return MultiversionDecoratorParams(task="dummy_task", base_suite="dummy_suite", + variant="dummy_variant", num_tasks=5) def build_mock_sub_tasks(): @@ -54,7 +38,7 @@ class TestDecorateFuzzerGenTask(unittest.TestCase): inject.clear() def run_test(self, fixture_type): - mock_params = build_mock_fuzzer_params() + mock_params = build_mock_params() mock_sub_tasks = build_mock_sub_tasks() multiversion_decorator = under_test.MultiversionGenTaskDecorator() expected_num_tasks = mock_params.num_tasks * len(multiversion_decorator.old_versions) * len( @@ -64,7 +48,8 @@ class TestDecorateFuzzerGenTask(unittest.TestCase): "_get_suite_fixture_type") as mock_get_suite_fixture_type: mock_get_suite_fixture_type.return_value = fixture_type - sub_tasks = multiversion_decorator.decorate_tasks(mock_sub_tasks, mock_params) + sub_tasks = multiversion_decorator.decorate_tasks_with_dynamically_generated_files( + mock_sub_tasks, mock_params) self.assertEqual(len(sub_tasks), expected_num_tasks) self.assertTrue( diff --git a/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py b/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py index b0306ec21d5..9ea74bde214 100644 --- a/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py +++ b/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py @@ -1,7 +1,10 @@ """Unit tests for resmoke_tasks.py.""" import unittest +import inject + import buildscripts.task_generation.task_types.resmoke_tasks as under_test +from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService from buildscripts.task_generation.suite_split import GeneratedSuite, SubSuite from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions from buildscripts.util.teststats import TestRuntime @@ -35,6 +38,7 @@ def build_mock_gen_params(repeat_suites=1, resmoke_args="resmoke args"): return under_test.ResmokeGenTaskParams( use_large_distro=False, require_multiversion_setup=False, + require_multiversion_version_combo=False, repeat_suites=repeat_suites, resmoke_args=resmoke_args, resmoke_jobs_max=None, @@ -59,6 +63,12 @@ def build_mock_suite(n_sub_suites, include_runtimes=True): class TestGenerateTask(unittest.TestCase): + def setUp(self) -> None: + def dependencies(binder: inject.Binder) -> None: + binder.bind(ResmokeProxyService, ResmokeProxyService()) + + inject.clear_and_configure(dependencies) + def test_evg_config_does_not_overwrite_repeatSuites_resmoke_arg_with_repeatSuites_default(self): mock_gen_options = build_mock_gen_options() params = build_mock_gen_params(resmoke_args="resmoke_args --repeatSuites=5") @@ -69,7 +79,7 @@ class TestGenerateTask(unittest.TestCase): for task in tasks: found_resmoke_cmd = False - for cmd in task.commands: + for cmd in task.shrub_task.commands: cmd_dict = cmd.as_dict() if cmd_dict.get("func") == "run generated tests": found_resmoke_cmd = True @@ -89,7 +99,7 @@ class TestGenerateTask(unittest.TestCase): for task in tasks: found_resmoke_cmd = False - for cmd in task.commands: + for cmd in task.shrub_task.commands: cmd_dict = cmd.as_dict() if cmd_dict.get("func") == "run generated tests": found_resmoke_cmd = True @@ -109,7 +119,8 @@ class TestGenerateTask(unittest.TestCase): tasks = resmoke_service.generate_tasks(suites, params) self.assertEqual(n_sub_suites + 1, len(tasks)) - for task in tasks: + for resmoke_task in tasks: + task = resmoke_task.shrub_task if "misc" in task.name: # Misc tasks should use default timeouts. continue @@ -127,7 +138,7 @@ class TestGenerateTask(unittest.TestCase): self.assertEqual(2, len(tasks)) for task in tasks: - for cmd in task.commands: + for cmd in task.shrub_task.commands: cmd_dict = cmd.as_dict() self.assertNotEqual("timeout.update", cmd_dict.get("command")) @@ -141,6 +152,6 @@ class TestGenerateTask(unittest.TestCase): self.assertEqual(2, len(tasks)) for task in tasks: - for cmd in task.commands: + for cmd in task.shrub_task.commands: cmd_dict = cmd.as_dict() self.assertNotEqual("timeout.update", cmd_dict.get("command")) diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml index 96af09bd574..39fd0c2d42f 100644 --- a/etc/backports_required_for_multiversion_tests.yml +++ b/etc/backports_required_for_multiversion_tests.yml @@ -110,26 +110,11 @@ last-continuous: test_file: jstests/sharding/retryable_writes_nested_shard_key.js - ticket: SERVER-54909 test_file: jstests/replsets/replSetGetStatus_member_wall_times.js + - ticket: SERVER-58744 + test_file: jstests/replsets/apply_ops_capped_collection.js # Tests that should only be excluded from particular suites should be listed under that suite. suites: - change_streams_multiversion: - - change_streams_sharded_collections_multiversion: - - concurrency_replication_multiversion: - - concurrency_sharded_replication_multiversion: - - replica_sets_multiversion: - - ticket: SERVER-58744 - test_file: jstests/replsets/apply_ops_capped_collection.js - - replica_sets_jscore_multiversion: - - sharding_jscore_multiversion: - - sharded_collections_jscore_multiversion: last-lts: # Tests that should be excluded from ALL multiversion suites should be listed here. @@ -380,23 +365,8 @@ last-lts: test_file: jstests/sharding/retryable_writes_nested_shard_key.js - ticket: SERVER-54909 test_file: jstests/replsets/replSetGetStatus_member_wall_times.js - - # Tests that should only be excluded from particular suites should be listed under that suite. - suites: - change_streams_multiversion: - - change_streams_sharded_collections_multiversion: - - concurrency_replication_multiversion: - - concurrency_sharded_replication_multiversion: - - replica_sets_multiversion: - ticket: SERVER-35649 test_file: jstests/replsets/disallow_adding_initialized_node1.js - replica_sets_jscore_multiversion: - - sharding_jscore_multiversion: - - sharded_collections_jscore_multiversion: + # Tests that should only be excluded from particular suites should be listed under that suite. + suites: diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 04e0dab9a79..b0dbe37e4eb 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -3769,58 +3769,22 @@ tasks: ## Tests that the multiversion test generation logic is not broken. - <<: *gen_task_template - name: multiversion_sanity_check_new_old_new_gen + name: multiversion_sanity_check_gen tags: ["multiversion", "multiversion_sanity_check"] commands: - func: "generate resmoke tasks" vars: - suite: multiversion_sanity_check - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=new-old-new --storageEngine=wiredTiger - -- <<: *gen_task_template - name: multiversion_sanity_check_old_new_new_gen - tags: ["multiversion", "multiversion_sanity_check"] - commands: - - func: "generate resmoke tasks" - vars: - suite: multiversion_sanity_check - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=old-new-new --storageEngine=wiredTiger - -- <<: *gen_task_template - name: multiversion_sanity_check_new_new_old_gen - tags: ["multiversion", "multiversion_sanity_check"] - commands: - - func: "generate resmoke tasks" - vars: - suite: multiversion_sanity_check - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=new-new-old --storageEngine=wiredTiger - -- <<: *gen_task_template - name: replica_sets_jscore_multiversion_passthrough_new_old_new_gen - tags: ["multiversion", "multiversion_passthrough"] - commands: - - func: "generate resmoke tasks" - vars: - suite: replica_sets_jscore_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=new-old-new --storageEngine=wiredTiger + resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template - name: replica_sets_jscore_multiversion_passthrough_old_new_new_gen + name: replica_sets_jscore_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: - func: "generate resmoke tasks" vars: - suite: replica_sets_jscore_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=old-new-new --storageEngine=wiredTiger + suite: replica_sets_jscore_passthrough + resmoke_args: --storageEngine=wiredTiger -- <<: *gen_task_template - name: replica_sets_jscore_multiversion_passthrough_new_new_old_gen - tags: ["multiversion", "multiversion_passthrough"] - commands: - - func: "generate resmoke tasks" - vars: - suite: replica_sets_jscore_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=new-new-old --storageEngine=wiredTiger # Check that the mutational fuzzer can parse JS files modified in a patch build. - name: lint_fuzzer_sanity_patch @@ -4062,34 +4026,16 @@ tasks: resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template - name: change_streams_multiversion_new_old_new_gen - tags: ["multiversion", "multiversion_passthrough"] - commands: - - func: "generate resmoke tasks" - vars: - suite: change_streams_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=new-old-new --storageEngine=wiredTiger - -- <<: *gen_task_template - name: change_streams_multiversion_old_new_new_gen - tags: ["multiversion", "multiversion_passthrough"] - commands: - - func: "generate resmoke tasks" - vars: - suite: change_streams_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=old-new-new --storageEngine=wiredTiger - -- <<: *gen_task_template - name: change_streams_multiversion_new_new_old_gen + name: change_streams_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: - func: "generate resmoke tasks" vars: - suite: change_streams_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=new-new-old --storageEngine=wiredTiger + suite: change_streams + resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template - name: change_streams_multiversion_downgrade_passthrough_gen + name: change_streams_downgrade_gen tags: ["multiversion_passthrough", "multiversion"] commands: - func: "generate resmoke tasks" @@ -4150,13 +4096,13 @@ tasks: resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template - name: change_streams_sharded_collections_multiversion_passthrough_gen + name: change_streams_sharded_collections_multiversion_gen tags: ["multiversion_passthrough", "multiversion"] commands: - func: "generate resmoke tasks" vars: - suite: change_streams_sharded_collections_multiversion - resmoke_args: --storageEngine=wiredTiger --oldBinVersion=last_continuous --mixedBinVersions="new-old-old-new" + suite: change_streams_sharded_collections_passthrough + resmoke_args: --storageEngine=wiredTiger - <<: *task_template name: change_streams_whole_db_passthrough @@ -4817,7 +4763,7 @@ tasks: - <<: *gen_task_template name: multiversion_auth_gen - tags: ["auth", "multiversion"] + tags: ["auth", "multiversion", "no_version_combination"] commands: - func: "generate resmoke tasks" vars: @@ -4825,7 +4771,7 @@ tasks: - <<: *gen_task_template name: multiversion_gen - tags: ["multiversion"] + tags: ["multiversion", "no_version_combination"] commands: - func: "generate resmoke tasks" vars: @@ -4836,7 +4782,7 @@ tasks: # build variants that enable this task. - <<: *gen_task_template name: feature_flag_multiversion_gen - tags: ["multiversion"] + tags: ["multiversion", "no_version_combination"] commands: - func: "generate resmoke tasks" vars: @@ -4916,7 +4862,8 @@ tasks: commands: - func: "generate resmoke tasks" vars: - resmoke_args: --storageEngine=wiredTiger --oldBinVersion=last_continuous --mixedBinVersions="new-old-old-new" + suite: sharded_collections_jscore_passthrough + resmoke_args: --storageEngine=wiredTiger - <<: *task_template name: sharding_jscore_passthrough @@ -4933,7 +4880,8 @@ tasks: commands: - func: "generate resmoke tasks" vars: - resmoke_args: --storageEngine=wiredTiger --oldBinVersion=last_continuous --mixedBinVersions="new-old-old-new" + suite: sharding_jscore_passthrough + resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template name: sharding_api_version_jscore_passthrough_gen @@ -5112,31 +5060,13 @@ tasks: resmoke_jobs_max: 1 - <<: *gen_task_template - name: concurrency_replication_multiversion_new_old_new_gen - tags: ["multiversion", "multiversion_passthrough"] - commands: - - func: "generate resmoke tasks" - vars: - suite: concurrency_replication_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=new-old-new --storageEngine=wiredTiger - -- <<: *gen_task_template - name: concurrency_replication_multiversion_old_new_new_gen + name: concurrency_replication_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: - func: "generate resmoke tasks" vars: - suite: concurrency_replication_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=old-new-new --storageEngine=wiredTiger - -- <<: *gen_task_template - name: concurrency_replication_multiversion_new_new_old_gen - tags: ["multiversion", "multiversion_passthrough"] - commands: - - func: "generate resmoke tasks" - vars: - suite: concurrency_replication_multiversion - resmoke_args: --oldBinVersion=last_continuous --mixedBinVersions=new-new-old --storageEngine=wiredTiger + suite: concurrency_replication + resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template name: concurrency_replication_causal_consistency_gen @@ -5222,8 +5152,8 @@ tasks: commands: - func: "generate resmoke tasks" vars: - suite: concurrency_sharded_replication_multiversion - resmoke_args: --storageEngine=wiredTiger --oldBinVersion=last_continuous --mixedBinVersions="new-old-old-new" + suite: concurrency_sharded_replication + resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template name: concurrency_sharded_replication_with_balancer_gen @@ -5563,8 +5493,8 @@ tasks: commands: - func: "generate resmoke tasks" vars: - resmoke_args: --storageEngine=wiredTiger --tagFile=generated_resmoke_config/multiversion_exclude_tags.yml - suite: replica_sets_multiversion + suite: replica_sets + resmoke_args: --storageEngine=wiredTiger - <<: *task_template name: sasl @@ -5586,13 +5516,13 @@ tasks: - <<: *gen_task_template name: sharding_multiversion_gen - tags: ["random_multiversion_ds", "multiversion"] + tags: ["random_multiversion_ds", "multiversion", "sharding_multiversion"] commands: - func: "generate resmoke tasks" vars: use_large_distro: "true" - resmoke_args: --storageEngine=wiredTiger --tagFile=generated_resmoke_config/multiversion_exclude_tags.yml - suite: sharding_multiversion + resmoke_args: --storageEngine=wiredTiger + suite: sharding - <<: *gen_task_template name: sharding_max_mirroring_gen @@ -5661,12 +5591,11 @@ tasks: - <<: *gen_task_template name: sharding_last_lts_mongos_and_mixed_shards_gen - tags: ["sharding", "common", "multiversion"] + tags: ["sharding", "common", "multiversion", "no_version_combination"] commands: - func: "generate resmoke tasks" vars: use_large_distro: "true" - resmoke_args: --tagFile=generated_resmoke_config/multiversion_exclude_tags.yml - <<: *gen_task_template name: sharding_update_v1_oplog_gen @@ -5801,22 +5730,30 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger +# Use explicit task definitions for retryable_writes_downgrade suites to avoid running +# with all Repl multiversion combinations. - <<: *gen_task_template - name: retryable_writes_downgrade_passthrough_gen - tags: ["multiversion_passthrough", "multiversion"] + name: retryable_writes_downgrade_last_continuous_gen + tags: ["multiversion_passthrough", "multiversion", "no_version_combination"] commands: - func: "generate resmoke tasks" vars: - suite: retryable_writes_downgrade_passthrough resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template - name: sharded_retryable_writes_downgrade_passthrough_gen + name: retryable_writes_downgrade_last_lts_gen + tags: ["multiversion_passthrough", "multiversion", "no_version_combination"] + commands: + - func: "generate resmoke tasks" + vars: + resmoke_args: --storageEngine=wiredTiger + +- <<: *gen_task_template + name: sharded_retryable_writes_downgrade_gen tags: ["multiversion_passthrough", "multiversion"] commands: - func: "generate resmoke tasks" vars: - suite: sharded_retryable_writes_downgrade_passthrough resmoke_args: --storageEngine=wiredTiger - <<: *gen_task_template @@ -9399,7 +9336,7 @@ buildvariants: - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .query_fuzzer - - name: .random_multiversion_ds + - name: .random_multiversion_ds !.sharding_multiversion - name: .read_write_concern .large distros: - rhel80-medium @@ -9518,7 +9455,7 @@ buildvariants: - name: multiversion_gen - name: powercycle_smoke - name: .query_fuzzer - - name: .random_multiversion_ds + - name: .random_multiversion_ds !.sharding_multiversion - name: .read_write_concern .large distros: - rhel80-medium @@ -9766,10 +9703,6 @@ buildvariants: run_on: - rhel80-small expansions: &enterprise-rhel-80-64-bit-multiversion-expansions-template - # TODO (SERVER-55857): Remove 'requires_fcv_51' once tests are filtered out using the - # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'. - test_flags: >- - --excludeWithAnyTags=requires_fcv_51,multiversion_incompatible compile_flags: >- -j$(grep -c ^processor /proc/cpuinfo) --ssl diff --git a/evergreen/burn_in_tests_generate.sh b/evergreen/burn_in_tests_generate.sh index fc346b91ff6..acd394b91a2 100644 --- a/evergreen/burn_in_tests_generate.sh +++ b/evergreen/burn_in_tests_generate.sh @@ -9,6 +9,7 @@ activate_venv # Multiversion exclusions can be used when selecting tests. PATH="$PATH:/data/multiversion" -$python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion=last_continuous --excludeTagsFilePath=multiversion_exclude_tags.yml +$python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion=last_continuous --excludeTagsFilePath=multiversion_exclude_tags_last_continuous.yml +$python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion=last_lts --excludeTagsFilePath=multiversion_exclude_tags_last_lts.yml PATH=$PATH:$HOME $python buildscripts/burn_in_tags.py --expansion-file ../expansions.yml diff --git a/evergreen/implicit_multiversions_tasks_generate.sh b/evergreen/implicit_multiversions_tasks_generate.sh index 0ca5e14a951..78eb852eccd 100644 --- a/evergreen/implicit_multiversions_tasks_generate.sh +++ b/evergreen/implicit_multiversions_tasks_generate.sh @@ -8,7 +8,6 @@ set -o errexit activate_venv PATH="$PATH:/data/multiversion" -# TODO SERVER-55857: remove this file and generate tags at execution time for suites defined in multiversion/ -if [[ "${require_multiversion_setup}" = "true" ]]; then - $python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion=last_continuous +if [[ "${require_multiversion_setup}" = "true" && -n "${multiversion_exclude_tags_version}" ]]; then + $python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion="${multiversion_exclude_tags_version}" fi diff --git a/jstests/core/mixed_version_replica_set.js b/jstests/core/mixed_version_replica_set.js index a4c51c03c38..d82c05883c0 100644 --- a/jstests/core/mixed_version_replica_set.js +++ b/jstests/core/mixed_version_replica_set.js @@ -9,9 +9,6 @@ const latestBinVersion = MongoRunner.getBinVersionFor("latest"); -const lastContiuousBinVersion = MongoRunner.getBinVersionFor("last-continuous"); -const lastLTSBinVersion = MongoRunner.getBinVersionFor("last-lts"); - if (testingReplication && TestData && TestData.mixedBinVersions) { const replSetStatus = db.adminCommand({"replSetGetStatus": 1}); const members = replSetStatus["members"]; @@ -23,8 +20,11 @@ if (testingReplication && TestData && TestData.mixedBinVersions) { const actualVersion = serverStatus["version"]; const expectedVersion = TestData.mixedBinVersions[i] === "new" ? latestBinVersion - : (TestData.mixedBinVersions[i] === "last-lts" ? lastLTSBinVersion - : lastContiuousBinVersion); + : MongoRunner.getBinVersionFor(TestData.multiversionBinVersion); + print(actualVersion, + expectedVersion, + MongoRunner.getBinVersionFor(TestData.multiversionBinVersion)); + print(TestData.multiversionBinVersion); assert(MongoRunner.areBinVersionsTheSame(actualVersion, expectedVersion)); } } else { |