diff options
| -rw-r--r-- | buildscripts/burn_in_tags.py | 2 | ||||
| -rw-r--r-- | buildscripts/burn_in_tests.py | 40 | ||||
| -rwxr-xr-x | buildscripts/evergreen_gen_multiversion_tests.py | 1 | ||||
| -rw-r--r-- | buildscripts/tests/test_burn_in_tests.py | 41 | ||||
| -rw-r--r-- | etc/evergreen.yml | 1 |
5 files changed, 58 insertions, 27 deletions
diff --git a/buildscripts/burn_in_tags.py b/buildscripts/burn_in_tags.py index 1ca028ec741..88639e4362f 100644 --- a/buildscripts/burn_in_tags.py +++ b/buildscripts/burn_in_tags.py @@ -146,7 +146,7 @@ def _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build repeat_tests_secs=config_options.repeat_tests_secs) create_generate_tasks_config(shrub_config, tests_by_task, gen_config, repeat_config, - evergreen_api, include_gen_task=False) + evergreen_api, evg_conf, include_gen_task=False) def _write_to_file(shrub_config): diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 56ec45261e7..fd2830c0f85 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -75,7 +75,8 @@ BURN_IN_TESTS_GEN_TASK = "burn_in_tests_gen" BURN_IN_TESTS_TASK = "burn_in_tests" MULTIVERSION_CONFIG_KEY = gen_multiversion.MULTIVERSION_CONFIG_KEY -MULTIVERSION_TAG = gen_multiversion.PASSTHROUGH_TAG +MULTIVERSION_PASSTHROUGH_TAG = gen_multiversion.PASSTHROUGH_TAG +RANDOM_MULTIVERSION_REPLSETS_TAG = gen_multiversion.RANDOM_REPLSETS_TAG BURN_IN_MULTIVERSION_TASK = gen_multiversion.BURN_IN_TASK TASK_PATH_SUFFIX = "/data/multiversion" @@ -555,10 +556,11 @@ def _get_task_runtime_history(evg_api: Optional[EvergreenApi], project: str, tas raise -def create_generate_tasks_config(evg_config: Configuration, tests_by_task: Dict, - generate_config: GenerateConfig, repeat_config: RepeatConfig, - evg_api: Optional[EvergreenApi], include_gen_task: bool = True, - task_prefix: str = "burn_in") -> Configuration: +def create_generate_tasks_config( + evg_config: Configuration, tests_by_task: Dict, generate_config: GenerateConfig, + repeat_config: RepeatConfig, evg_api: Optional[EvergreenApi], + evg_project_config: EvergreenProjectConfig, include_gen_task: bool = True, + task_prefix: str = "burn_in") -> Configuration: # pylint: disable=too-many-arguments,too-many-locals """ Create the config for the Evergreen generate.tasks file. @@ -575,13 +577,20 @@ def create_generate_tasks_config(evg_config: Configuration, tests_by_task: Dict, task_list = TaskList(evg_config) resmoke_options = repeat_config.generate_resmoke_options() for task in sorted(tests_by_task): - multiversion_path = tests_by_task[task].get("use_multiversion") - task_runtime_stats = _get_task_runtime_history(evg_api, generate_config.project, task, - generate_config.build_variant) - resmoke_args = tests_by_task[task]["resmoke_args"] test_list = tests_by_task[task]["tests"] - distro = tests_by_task[task].get("distro", generate_config.distro) for index, test in enumerate(test_list): + if task in evg_project_config.get_task_names_by_tag(RANDOM_MULTIVERSION_REPLSETS_TAG): + # Exclude files that should be blacklisted from multiversion testing. + files_to_exclude = gen_multiversion.get_exclude_files(task, TASK_PATH_SUFFIX) + if test in files_to_exclude: + LOGGER.debug("Files to exclude", files_to_exclude=files_to_exclude, test=test, + suite=task) + continue + multiversion_path = tests_by_task[task].get("use_multiversion") + task_runtime_stats = _get_task_runtime_history(evg_api, generate_config.project, task, + generate_config.build_variant) + resmoke_args = tests_by_task[task]["resmoke_args"] + distro = tests_by_task[task].get("distro", generate_config.distro) # Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join # here, just use the forward slash; otherwise the path separator will be treated as # the escape character on Windows. @@ -717,6 +726,7 @@ def create_tests_by_task(build_variant: str, repo: Repo, evg_conf: EvergreenProj # pylint: disable=too-many-arguments def create_generate_tasks_file(tests_by_task: Dict, generate_config: GenerateConfig, repeat_config: RepeatConfig, evg_api: Optional[EvergreenApi], + evg_project_config: EvergreenProjectConfig, task_prefix: str = 'burn_in', include_gen_task: bool = True) -> Dict: """ Create an Evergreen generate.tasks file to run the given tasks and tests. @@ -735,7 +745,7 @@ def create_generate_tasks_file(tests_by_task: Dict, generate_config: GenerateCon generate_config) else: evg_config = create_generate_tasks_config( - evg_config, tests_by_task, generate_config, repeat_config, evg_api, + evg_config, tests_by_task, generate_config, repeat_config, evg_api, evg_project_config, include_gen_task=include_gen_task, task_prefix=task_prefix) json_config = evg_config.to_map() @@ -821,16 +831,16 @@ def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmok if generate_tasks_file: if generate_config.use_multiversion: - multiversion_tasks = evg_conf.get_task_names_by_tag(MULTIVERSION_TAG) + multiversion_tasks = evg_conf.get_task_names_by_tag(MULTIVERSION_PASSTHROUGH_TAG) LOGGER.debug("Multiversion tasks by tag", tasks=multiversion_tasks, - tag=MULTIVERSION_TAG) - # We expect the number of suites with MULTIVERSION_TAG to be the same as in + tag=MULTIVERSION_PASSTHROUGH_TAG) + # We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in # multiversion_suites. Multiversion passthrough suites must include # MULTIVERSION_CONFIG_KEY as a root level key and must be set to true. multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY) assert len(multiversion_tasks) == len(multiversion_suites) json_config = create_generate_tasks_file(tests_by_task, generate_config, repeat_config, - evg_api) + evg_api, evg_conf) _write_json_file(json_config, generate_tasks_file) elif not no_exec: run_tests(tests_by_task, resmoke_cmd) diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py index dc161a8351c..0d8558b225c 100755 --- a/buildscripts/evergreen_gen_multiversion_tests.py +++ b/buildscripts/evergreen_gen_multiversion_tests.py @@ -52,6 +52,7 @@ SHARDED_MIXED_VERSION_CONFIGS = ["new-old-old-new"] BURN_IN_TASK = "burn_in_tests_multiversion" MULTIVERSION_CONFIG_KEY = "use_in_multiversion" PASSTHROUGH_TAG = "multiversion_passthrough" +RANDOM_REPLSETS_TAG = "random_multiversion_replica_sets" EXCLUDE_TAGS = f"{REQUIRES_FCV_TAG},multiversion_incompatible" # The directory in which BACKPORTS_REQUIRED_FILE resides. diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 3d530d47d56..2f693971fc0 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -512,17 +512,21 @@ TESTS_BY_TASK = { class TestCreateGenerateTasksConfig(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") def test_no_tasks_given(self): evg_config = Configuration() gen_config = MagicMock(run_build_variant="variant") repeat_config = MagicMock() - evg_config = under_test.create_generate_tasks_config(evg_config, {}, gen_config, - repeat_config, None) + evg_project_config = get_evergreen_config("etc/evergreen.yml") + + evg_config = under_test.create_generate_tasks_config( + evg_config, {}, gen_config, repeat_config, None, evg_project_config) evg_config_dict = evg_config.to_map() self.assertNotIn("tasks", evg_config_dict) + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") def test_one_task_one_test(self): n_tasks = 1 n_tests = 1 @@ -533,8 +537,10 @@ class TestCreateGenerateTasksConfig(unittest.TestCase): repeat_config.generate_resmoke_options.return_value = resmoke_options tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - evg_config = under_test.create_generate_tasks_config(evg_config, tests_by_task, gen_config, - repeat_config, None) + evg_project_config = get_evergreen_config("etc/evergreen.yml") + + evg_config = under_test.create_generate_tasks_config( + evg_config, tests_by_task, gen_config, repeat_config, None, evg_project_config) evg_config_dict = evg_config.to_map() tasks = evg_config_dict["tasks"] @@ -544,6 +550,7 @@ class TestCreateGenerateTasksConfig(unittest.TestCase): self.assertIn("--suites=suite_0", cmd[1]["vars"]["resmoke_args"]) self.assertIn("tests_0", cmd[1]["vars"]["resmoke_args"]) + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") def test_n_task_m_test(self): n_tasks = 3 n_tests = 5 @@ -552,12 +559,15 @@ class TestCreateGenerateTasksConfig(unittest.TestCase): repeat_config = MagicMock() tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - evg_config = under_test.create_generate_tasks_config(evg_config, tests_by_task, gen_config, - repeat_config, None) + evg_project_config = get_evergreen_config("etc/evergreen.yml") + + evg_config = under_test.create_generate_tasks_config( + evg_config, tests_by_task, gen_config, repeat_config, None, evg_project_config) evg_config_dict = evg_config.to_map() self.assertEqual(n_tasks * n_tests, len(evg_config_dict["tasks"])) + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") def test_multiversion_path_is_used(self): n_tasks = 1 n_tests = 1 @@ -569,8 +579,10 @@ class TestCreateGenerateTasksConfig(unittest.TestCase): multiversion_path = "multiversion_path" tests_by_task[first_task]["use_multiversion"] = multiversion_path - evg_config = under_test.create_generate_tasks_config(evg_config, tests_by_task, gen_config, - repeat_config, None) + evg_project_config = get_evergreen_config("etc/evergreen.yml") + + evg_config = under_test.create_generate_tasks_config( + evg_config, tests_by_task, gen_config, repeat_config, None, evg_project_config) evg_config_dict = evg_config.to_map() tasks = evg_config_dict["tasks"] @@ -678,6 +690,7 @@ class TestCreateMultiversionGenerateTasksConfig(unittest.TestCase): class TestCreateGenerateTasksFile(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch("buildscripts.burn_in_tests.create_generate_tasks_config") def test_gen_tasks_configuration_is_returned(self, gen_tasks_config_mock): evg_api = MagicMock() @@ -693,12 +706,14 @@ class TestCreateGenerateTasksFile(unittest.TestCase): } gen_tasks_config_mock.return_value = evg_config + evg_project_config = get_evergreen_config("etc/evergreen.yml") config = under_test.create_generate_tasks_file(tests_by_task, gen_config, repeat_config, - evg_api) + evg_api, evg_project_config) self.assertEqual(config, evg_config.to_map.return_value) + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch(ns("create_generate_tasks_config")) def test_gen_tasks_multiversion_configuration_is_returned(self, gen_tasks_config_mock): # pylint: disable=invalid-name evg_api = MagicMock() @@ -726,11 +741,13 @@ class TestCreateGenerateTasksFile(unittest.TestCase): } # yapf: disable gen_tasks_config_mock.return_value = evg_config + evg_project_config = get_evergreen_config("etc/evergreen.yml") config = under_test.create_generate_tasks_file(tests_by_task, gen_config, repeat_config, - evg_api) + evg_api, evg_project_config) self.assertEqual(config, evg_config.to_map.return_value) + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch("buildscripts.burn_in_tests.sys.exit") @patch("buildscripts.burn_in_tests.create_generate_tasks_config") def test_cap_on_task_generate(self, gen_tasks_config_mock, exit_mock): @@ -747,10 +764,12 @@ class TestCreateGenerateTasksFile(unittest.TestCase): } gen_tasks_config_mock.return_value = evg_config + evg_project_config = get_evergreen_config("etc/evergreen.yml") exit_mock.side_effect = ValueError("exiting") with self.assertRaises(ValueError): - under_test.create_generate_tasks_file(tests_by_task, gen_config, repeat_config, evg_api) + under_test.create_generate_tasks_file(tests_by_task, gen_config, repeat_config, evg_api, + evg_project_config) exit_mock.assert_called_once() diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 3633943b373..e3eb847f436 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -5409,6 +5409,7 @@ tasks: vars: pip_dir: ${workdir}/src/etc/pip - func: "upload pip requirements" + - func: "do multiversion setup" - func: "generate burn in tags" vars: max_revisions: 25 |