diff options
-rw-r--r-- | buildscripts/task_generation/evg_config_builder.py | 14 | ||||
-rw-r--r-- | buildscripts/task_generation/resmoke_proxy.py | 25 | ||||
-rw-r--r-- | buildscripts/task_generation/suite_split.py | 12 | ||||
-rw-r--r-- | buildscripts/task_generation/task_types/multiversion_tasks.py | 19 | ||||
-rw-r--r-- | buildscripts/task_generation/task_types/resmoke_tasks.py | 40 | ||||
-rw-r--r-- | buildscripts/tests/task_generation/test_suite_split.py | 13 | ||||
-rw-r--r-- | buildscripts/tests/test_selected_tests.py | 2 |
7 files changed, 71 insertions, 54 deletions
diff --git a/buildscripts/task_generation/evg_config_builder.py b/buildscripts/task_generation/evg_config_builder.py index 7e4ddc3b807..a1f9b299b98 100644 --- a/buildscripts/task_generation/evg_config_builder.py +++ b/buildscripts/task_generation/evg_config_builder.py @@ -1,6 +1,6 @@ """Builder for generating evergreen configuration.""" from threading import Lock -from typing import Optional, Set, List, Dict +from typing import Set, List, Dict import inject from shrub.v2 import ShrubProject, BuildVariant, ExistingTask, Task @@ -60,20 +60,17 @@ class EvgConfigBuilder: self.build_variants[build_variant] = BuildVariant(build_variant, activate=False) return self.build_variants[build_variant] - def _generate_suites_config(self, generated_suite: GeneratedSuite, - is_multiversion: bool = False) -> List[GeneratedFile]: + def _generate_suites_config(self, generated_suite: GeneratedSuite) -> List[GeneratedFile]: """ Generate the suites files and evergreen configuration for the generated task. :param generated_suite: Generated suite to create config files for. - :param is_multiversion: True if the generated suite is a multiversion suite. :return: The suites files and evergreen configuration for the generated task. """ test_list = generated_suite.get_test_list() return self.resmoke_proxy.render_suite_files( generated_suite.sub_suites, generated_suite.suite_name, generated_suite.filename, - test_list, self.gen_options.create_misc_suite, generated_suite.build_variant, - is_multiversion) + test_list, self.gen_options.create_misc_suite, generated_suite) def generate_suite(self, split_params: SuiteSplitParameters, gen_params: ResmokeGenTaskParams) -> None: @@ -102,8 +99,7 @@ class EvgConfigBuilder: build_variant = self.get_build_variant(generated_suite.build_variant) self.evg_config_gen_service.generate_multiversion_task(generated_suite, build_variant, gen_params) - self.generated_files.extend( - self._generate_suites_config(generated_suite, is_multiversion=True)) + self.generated_files.extend(self._generate_suites_config(generated_suite)) def add_multiversion_burn_in_test(self, split_params: SuiteSplitParameters, gen_params: MultiversionGenTaskParams) -> Set[Task]: @@ -118,7 +114,7 @@ class EvgConfigBuilder: build_variant = self.get_build_variant(generated_suite.build_variant) tasks = self.evg_config_gen_service.generate_multiversion_burnin_task( generated_suite, gen_params, build_variant) - self.generated_files.extend(self._generate_suites_config(generated_suite, True)) + self.generated_files.extend(self._generate_suites_config(generated_suite)) return tasks def generate_fuzzer(self, fuzzer_params: FuzzerGenTaskParams) -> FuzzerTask: diff --git a/buildscripts/task_generation/resmoke_proxy.py b/buildscripts/task_generation/resmoke_proxy.py index 4aa2037003b..487dd2a05cd 100644 --- a/buildscripts/task_generation/resmoke_proxy.py +++ b/buildscripts/task_generation/resmoke_proxy.py @@ -1,7 +1,7 @@ """A service to proxy requests to resmoke.""" import os from copy import deepcopy -from typing import List, Dict, Any, NamedTuple +from typing import List, Dict, Any, NamedTuple, TYPE_CHECKING import inject import structlog @@ -12,6 +12,9 @@ import buildscripts.resmokelib.suitesconfig as suitesconfig from buildscripts.task_generation.generated_config import GeneratedFile from buildscripts.util.fileops import read_yaml_file +if TYPE_CHECKING: + from buildscripts.task_generation.suite_split import GeneratedSuite, SubSuite + LOGGER = structlog.get_logger(__name__) HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost. @@ -71,9 +74,9 @@ class ResmokeProxyService: """ return read_yaml_file(os.path.join(self.resmoke_suite_dir, f"{suite_name}.yml")) - def render_suite_files(self, suites: List, suite_name: str, generated_suite_filename: str, - test_list: List[str], create_misc_suite: bool, build_variant: str, - is_multiversion: bool) -> List[GeneratedFile]: + def render_suite_files(self, suites: List["SubSuite"], suite_name: str, + generated_suite_filename: str, test_list: List[str], + create_misc_suite: bool, suite: "GeneratedSuite") -> List[GeneratedFile]: """ Render the given list of suites. @@ -85,24 +88,20 @@ class ResmokeProxyService: :param generated_suite_filename: The name to use as the file name for generated suite file. :param test_list: List of tests used in suites. :param create_misc_suite: Whether or not a _misc suite file should be created. - :param build_variant: Build variant suite file is being rendered for. - :param is_multiversion: True if the files being generated are for multiversion. + :param suite: Generated suite files belong to. :return: Dictionary of rendered resmoke config files. """ # pylint: disable=too-many-arguments source_config = self.read_suite_config(suite_name) - multiversion_str = "_multiversion" if is_multiversion else "" suite_configs = [ - GeneratedFile( - file_name= - f"{os.path.basename(suite.name(len(suites)))}{multiversion_str}_{build_variant}.yml", - content=suite.generate_resmoke_config(source_config)) for suite in suites + GeneratedFile(file_name=f"{suite.sub_suite_config_file(i)}.yml", + content=sub_suite.generate_resmoke_config(source_config)) + for i, sub_suite in enumerate(suites) ] if create_misc_suite: suite_configs.append( GeneratedFile( - file_name= - f"{generated_suite_filename}_misc{multiversion_str}_{build_variant}.yml", + file_name=f"{suite.sub_suite_config_file(None)}.yml", content=generate_resmoke_suite_config(source_config, generated_suite_filename, excludes=test_list))) diff --git a/buildscripts/task_generation/suite_split.py b/buildscripts/task_generation/suite_split.py index e444e43282e..78a2e0c0486 100644 --- a/buildscripts/task_generation/suite_split.py +++ b/buildscripts/task_generation/suite_split.py @@ -198,6 +198,18 @@ class GeneratedSuite(NamedTuple): """Get the number of sub-suites.""" return len(self.sub_suites) + def sub_suite_config_file(self, index: Optional[int] = None) -> str: + """ + Get the name of the file to store the resmoke configuration. + + :param index: Index of suite or None for '_misc' suite. + :return: Name resmoke configuration for subtask should stored. + """ + if index is not None: + return taskname.name_generated_task(self.display_task_name(), index, + len(self.sub_suites)) + return f"{self.display_task_name()}_misc" + class SuiteSplitParameters(NamedTuple): """ diff --git a/buildscripts/task_generation/task_types/multiversion_tasks.py b/buildscripts/task_generation/task_types/multiversion_tasks.py index 3b8bcd1778c..dcc9632c4a2 100644 --- a/buildscripts/task_generation/task_types/multiversion_tasks.py +++ b/buildscripts/task_generation/task_types/multiversion_tasks.py @@ -67,7 +67,7 @@ class MultiversionGenTaskService: """ sub_tasks = set() for version_config in params.mixed_version_configs: - for sub_suite in suite.sub_suites: + for index, sub_suite in enumerate(suite.sub_suites): # Generate the newly divided test suites sub_suite_name = sub_suite.name(len(suite)) sub_task_name = f"{sub_suite_name}_{version_config}_{suite.build_variant}" @@ -75,33 +75,30 @@ class MultiversionGenTaskService: sub_task_name = f"{params.name_prefix}:{sub_task_name}" sub_tasks.add( - self._generate_task(sub_task_name, sub_suite_name, version_config, params, - suite.build_variant)) + self._generate_task(sub_task_name, version_config, params, suite, index)) if params.create_misc_suite: # Also generate the misc task. misc_suite_name = f"{params.origin_suite}_misc" misc_task_name = f"{misc_suite_name}_{version_config}_{suite.build_variant}" sub_tasks.add( - self._generate_task(misc_task_name, misc_suite_name, version_config, params, - suite.build_variant)) + self._generate_task(misc_task_name, version_config, params, suite, None)) return sub_tasks # pylint: disable=too-many-arguments - def _generate_task(self, sub_task_name: str, sub_suite_name: str, mixed_version_config: str, - params: MultiversionGenTaskParams, build_variant: str) -> Task: + def _generate_task(self, sub_task_name: str, mixed_version_config: str, + params: MultiversionGenTaskParams, suite: GeneratedSuite, + index: Optional[int]) -> Task: """ Generate a sub task to be run with the provided suite and mixed version config. :param sub_task_name: Name of task being generated. - :param sub_suite_name: Name of suite to run. :param mixed_version_config: Versions task is being generated for. :param params: Parameters for how tasks should be generated. :return: Shrub configuration for task specified. """ - suite_file = self.gen_task_options.suite_location( - f"{sub_suite_name}_multiversion_{build_variant}.yml") + suite_file = self.gen_task_options.suite_location(suite.sub_suite_config_file(index)) run_tests_vars = { "resmoke_args": self._build_resmoke_args(suite_file, mixed_version_config, params), @@ -134,7 +131,7 @@ class MultiversionGenTaskService: return ( f"{params.resmoke_args} " - f" --suite={suite_file} " + f" --suite={suite_file}.yml " f" --mixedBinVersions={mixed_version_config}" f" --excludeWithAnyTags={EXCLUDE_TAGS},{params.parent_task_name}_{BACKPORT_REQUIRED_TAG} " f" --tagFile={tag_file_location} " diff --git a/buildscripts/task_generation/task_types/resmoke_tasks.py b/buildscripts/task_generation/task_types/resmoke_tasks.py index eabc9e072b3..7df492be8d4 100644 --- a/buildscripts/task_generation/task_types/resmoke_tasks.py +++ b/buildscripts/task_generation/task_types/resmoke_tasks.py @@ -47,16 +47,15 @@ class ResmokeGenTaskParams(NamedTuple): resmoke_jobs_max: Optional[int] config_location: str - def generate_resmoke_args(self, suite_file: str, suite_name: str, build_variant: str) -> str: + def generate_resmoke_args(self, suite_file: str, suite_name: str) -> str: """ Generate the resmoke args for the given suite. :param suite_file: File containing configuration for test suite. :param suite_name: Name of suite being generated. - :param build_variant: Build Variant being generated for. :return: arguments to pass to resmoke. """ - resmoke_args = (f"--suite={suite_file}_{build_variant}.yml --originSuite={suite_name} " + resmoke_args = (f"--suite={suite_file}.yml --originSuite={suite_name} " f" {self.resmoke_args}") if self.repeat_suites and not string_contains_any_of_args(resmoke_args, ["repeatSuites", "repeat"]): @@ -93,11 +92,10 @@ class ResmokeGenTaskService: if self.gen_task_options.create_misc_suite: # Add the misc suite - misc_suite_name = f"{os.path.basename(generated_suite.suite_name)}_misc" misc_task_name = f"{generated_suite.task_name}_misc_{generated_suite.build_variant}" tasks.add( - self._generate_task(misc_suite_name, misc_task_name, TimeoutEstimate.no_timeouts(), - params, generated_suite)) + self._generate_task(None, misc_task_name, TimeoutEstimate.no_timeouts(), params, + generated_suite)) return tasks @@ -113,16 +111,16 @@ class ResmokeGenTaskService: """ sub_task_name = taskname.name_generated_task(suite.task_name, sub_suite.index, len(suite), suite.build_variant) - return self._generate_task( - sub_suite.name(len(suite)), sub_task_name, sub_suite.get_timeout_estimate(), params, - suite) + return self._generate_task(sub_suite.index, sub_task_name, sub_suite.get_timeout_estimate(), + params, suite) - def _generate_task(self, sub_suite_name: str, sub_task_name: str, timeout_est: TimeoutEstimate, - params: ResmokeGenTaskParams, suite: GeneratedSuite) -> Task: + def _generate_task(self, sub_suite_index: Optional[int], sub_task_name: str, + timeout_est: TimeoutEstimate, params: ResmokeGenTaskParams, + suite: GeneratedSuite) -> Task: """ Generate a shrub evergreen config for a resmoke task. - :param sub_suite_name: Name of suite being generated. + :param sub_suite_index: Index of suite being generated. :param sub_task_name: Name of task to generate. :param timeout_est: Estimated runtime to use for calculating timeouts. :param params: Parameters describing how tasks should be generated. @@ -130,11 +128,11 @@ class ResmokeGenTaskService: :return: Shrub configuration for the described task. """ # pylint: disable=too-many-arguments - LOGGER.debug("Generating task", sub_suite=sub_suite_name) + LOGGER.debug("Generating task", suite=suite.display_task_name(), index=sub_suite_index) - target_suite_file = self.gen_task_options.suite_location(sub_suite_name) - run_tests_vars = self._get_run_tests_vars(target_suite_file, suite.suite_name, params, - suite.build_variant) + target_suite_file = self.gen_task_options.suite_location( + suite.sub_suite_config_file(sub_suite_index)) + run_tests_vars = self._get_run_tests_vars(target_suite_file, suite.suite_name, params) require_multiversion = params.require_multiversion timeout_cmd = timeout_est.generate_timeout_cmd(self.gen_task_options.is_patch, @@ -146,19 +144,21 @@ class ResmokeGenTaskService: return Task(sub_task_name, commands, self._get_dependencies()) @staticmethod - def _get_run_tests_vars(suite_file: str, suite_name: str, params: ResmokeGenTaskParams, - build_variant: str) -> Dict[str, Any]: + def _get_run_tests_vars( + suite_file: str, + suite_name: str, + params: ResmokeGenTaskParams, + ) -> Dict[str, Any]: """ Generate a dictionary of the variables to pass to the task. :param suite_file: Suite being generated. :param suite_name: Name of suite being generated :param params: Parameters describing how tasks should be generated. - :param build_variant: Build Variant being generated. :return: Dictionary containing variables and value to pass to generated task. """ variables = { - "resmoke_args": params.generate_resmoke_args(suite_file, suite_name, build_variant), + "resmoke_args": params.generate_resmoke_args(suite_file, suite_name), "gen_task_config_location": params.config_location, } diff --git a/buildscripts/tests/task_generation/test_suite_split.py b/buildscripts/tests/task_generation/test_suite_split.py index da6e7f9e16a..95536d111a3 100644 --- a/buildscripts/tests/task_generation/test_suite_split.py +++ b/buildscripts/tests/task_generation/test_suite_split.py @@ -105,6 +105,19 @@ class TestGeneratedSuite(unittest.TestCase): for test in test_list: self.assertIn(test, all_tests) + def test_sub_suite_config_file_should_generate_filename_for_sub_suites(self): + task_name = "task_name" + n_sub_suites = 42 + mock_sub_suites = [build_mock_sub_suite(i, []) for i in range(n_sub_suites)] + mock_suite = under_test.GeneratedSuite( + sub_suites=mock_sub_suites, build_variant="build_variant", task_name=f"{task_name}_gen", + suite_name="suite_name", filename="filename") + + self.assertEqual(mock_suite.sub_suite_config_file(34), "task_name_34") + self.assertEqual(mock_suite.sub_suite_config_file(0), "task_name_00") + self.assertEqual(mock_suite.sub_suite_config_file(3), "task_name_03") + self.assertEqual(mock_suite.sub_suite_config_file(None), "task_name_misc") + class TestSplitSuite(unittest.TestCase): def test_calculate_suites(self): diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py index cd774444d34..377ee59bc6b 100644 --- a/buildscripts/tests/test_selected_tests.py +++ b/buildscripts/tests/test_selected_tests.py @@ -136,7 +136,7 @@ class TestAcceptance(unittest.TestCase): # assert that generated suite files have the suite name and the variant name in the # filename, to prevent tasks on different variants from using the same suite file - self.assertIn("auth_0_enterprise-rhel-80-64-bit-dynamic-required.yml", files_to_generate) + self.assertIn("auth_enterprise-rhel-80-64-bit-dynamic-required_0.yml", files_to_generate) generated_evg_config_raw = [ gen_file.content for gen_file in generated_config.file_list |