diff options
| -rwxr-xr-x | buildscripts/evergreen_gen_multiversion_tests.py | 10 | ||||
| -rwxr-xr-x | buildscripts/evergreen_generate_resmoke_tasks.py | 92 | ||||
| -rw-r--r-- | buildscripts/tests/test_evergreen_gen_multiversion_tests.py | 9 | ||||
| -rw-r--r-- | buildscripts/tests/test_evergreen_generate_resmoke_tasks.py | 31 | ||||
| -rw-r--r-- | buildscripts/util/fileops.py | 15 | ||||
| -rw-r--r-- | etc/evergreen.yml | 16 | ||||
| -rw-r--r-- | etc/generate_subtasks_config.yml | 44 | ||||
| -rw-r--r-- | etc/pip/components/evergreen.req | 1 |
8 files changed, 180 insertions, 38 deletions
diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py index f803435785d..f8d7d15e9ff 100755 --- a/buildscripts/evergreen_gen_multiversion_tests.py +++ b/buildscripts/evergreen_gen_multiversion_tests.py @@ -23,7 +23,7 @@ from buildscripts.resmokelib.multiversionconstants import (LAST_STABLE_MONGO_BIN REQUIRES_FCV_TAG) import buildscripts.resmokelib.parser import buildscripts.util.taskname as taskname -from buildscripts.util.fileops import write_file_to_dir +from buildscripts.util.fileops import write_file_to_dir, read_yaml_file import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke from buildscripts.evergreen_generate_resmoke_tasks import Suite, ConfigOptions import buildscripts.evergreen_gen_fuzzer_tests as gen_fuzzer @@ -42,7 +42,7 @@ DEFAULT_CONFIG_VALUES = generate_resmoke.DEFAULT_CONFIG_VALUES CONFIG_DIR = DEFAULT_CONFIG_VALUES["generated_config_dir"] DEFAULT_CONFIG_VALUES["is_jstestfuzz"] = False TEST_SUITE_DIR = DEFAULT_CONFIG_VALUES["test_suites_dir"] -CONFIG_FILE = generate_resmoke.CONFIG_FILE +CONFIG_FILE = generate_resmoke.EVG_CONFIG_FILE CONFIG_FORMAT_FN = generate_resmoke.CONFIG_FORMAT_FN REPL_MIXED_VERSION_CONFIGS = ["new-old-new", "new-new-old", "old-new-new"] SHARDED_MIXED_VERSION_CONFIGS = ["new-old-old-new"] @@ -74,7 +74,7 @@ def enable_logging(): def is_suite_sharded(suite_dir: str, suite_name: str) -> bool: """Return true if a suite uses ShardedClusterFixture.""" - source_config = generate_resmoke.read_yaml(suite_dir, suite_name + ".yml") + source_config = read_yaml_file(os.path.join(suite_dir, suite_name + ".yml")) return source_config["executor"]["fixture"]["class"] == "ShardedClusterFixture" @@ -121,7 +121,7 @@ def get_last_stable_yaml(last_stable_commit_hash): with open(os.path.join(temp_dir, last_stable_file), "w") as fileh: fileh.write(response.text) - backports_required_last_stable = generate_resmoke.read_yaml(temp_dir, last_stable_file) + backports_required_last_stable = read_yaml_file(os.path.join(temp_dir, last_stable_file)) return backports_required_last_stable @@ -393,7 +393,7 @@ def generate_exclude_yaml(task_path_suffix: str, output: str) -> None: LOGGER.info(f"Cannot write to {output}. Not generating tag file.") return - backports_required_latest = generate_resmoke.read_yaml(ETC_DIR, BACKPORTS_REQUIRED_FILE) + backports_required_latest = read_yaml_file(os.path.join(ETC_DIR, BACKPORTS_REQUIRED_FILE)) # Get the state of the backports_required_for_multiversion_tests.yml file for the last-lts # binary we are running tests against. We do this by using the commit hash from the last-lts diff --git a/buildscripts/evergreen_generate_resmoke_tasks.py b/buildscripts/evergreen_generate_resmoke_tasks.py index e6bb26f6ca4..df1e45a0aed 100755 --- a/buildscripts/evergreen_generate_resmoke_tasks.py +++ b/buildscripts/evergreen_generate_resmoke_tasks.py @@ -24,6 +24,7 @@ import structlog import yaml from evergreen.api import EvergreenApi, RetryingEvergreenApi +from pydantic.main import BaseModel from shrub.v2 import Task, TaskDependency, BuildVariant, ExistingTask, ShrubProject @@ -34,7 +35,7 @@ if __name__ == "__main__" and __package__ is None: # pylint: disable=wrong-import-position import buildscripts.resmokelib.parser as _parser import buildscripts.resmokelib.suitesconfig as suitesconfig -from buildscripts.util.fileops import write_file_to_dir +from buildscripts.util.fileops import write_file_to_dir, read_yaml_file import buildscripts.util.read_config as read_config import buildscripts.util.taskname as taskname from buildscripts.util.teststats import HistoricTaskData, TestRuntime, normalize_test_name @@ -45,7 +46,8 @@ LOGGER = structlog.getLogger(__name__) AVG_SETUP_TIME = int(timedelta(minutes=5).total_seconds()) DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites") -CONFIG_FILE = "./.evergreen.yml" +EVG_CONFIG_FILE = "./.evergreen.yml" +GENERATE_CONFIG_FILE = "etc/generate_subtasks_config.yml" MIN_TIMEOUT_SECONDS = int(timedelta(minutes=5).total_seconds()) MAX_EXPECTED_TIMEOUT = int(timedelta(hours=48).total_seconds()) LOOKBACK_DURATION_DAYS = 14 @@ -58,6 +60,21 @@ HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost. # {suite_file}. """ +NO_LARGE_DISTRO_ERR = """ +*************************************************************************************** +It appears we are trying to generate a task marked as requiring a large distro, but the +build variant has not specified a large build variant. In order to resolve this error, +you need to: + +(1) add a "large_distro_name" expansion to this build variant ("{build_variant}"). + + -- or -- + +(2) add this build variant ("{build_variant}") to the "build_variant_large_distro_exception" +list in the "etc/generate_subtasks_config.yml" file. +*************************************************************************************** +""" + REQUIRED_CONFIG_KEYS = { "build_variant", "fallback_num_sub_suites", @@ -87,6 +104,22 @@ CONFIG_FORMAT_FN = { } +class GenerationConfiguration(BaseModel): + """Configuration for generating sub-tasks.""" + + build_variant_large_distro_exceptions: Set[str] + + @classmethod + def from_yaml_file(cls, path: str) -> "GenerationConfiguration": + """Read the generation configuration from the given file.""" + return cls(**read_yaml_file(path)) + + @classmethod + def default_config(cls) -> "GenerationConfiguration": + """Create a default configuration.""" + return cls(build_variant_large_distro_exceptions=set()) + + class ConfigOptions(object): """Retrieve configuration from a config file.""" @@ -229,18 +262,6 @@ def write_file_dict(directory: str, file_dict: Dict[str, str]) -> None: write_file_to_dir(directory, name, contents) -def read_yaml(directory: str, filename: str) -> Dict: - """ - Read the given yaml file. - - :param directory: Directory containing file. - :param filename: Name of file to read. - :return: Yaml contents of file. - """ - with open(os.path.join(directory, filename), "r") as fileh: - return yaml.safe_load(fileh) - - def split_if_exists(str_to_split): """Split the given string on "," if it is not None.""" if str_to_split: @@ -411,7 +432,8 @@ def render_suite_files(suites: List, suite_name: str, test_list: List[str], suit :param create_misc_suite: Whether or not a _misc suite file should be created. :return: Dictionary of rendered resmoke config files. """ - source_config = read_yaml(suite_dir, suite_name + ".yml") + # pylint: disable=too-many-arguments + source_config = read_yaml_file(os.path.join(suite_dir, suite_name + ".yml")) suite_configs = { f"{os.path.basename(suite.name)}.yml": suite.generate_resmoke_config(source_config) for suite in suites @@ -616,24 +638,38 @@ class Suite(object): class EvergreenConfigGenerator(object): """Generate evergreen configurations.""" - def __init__(self, suites: List[Suite], options: ConfigOptions, evg_api: EvergreenApi): + def __init__(self, suites: List[Suite], options: ConfigOptions, evg_api: EvergreenApi, + generate_config: Optional[GenerationConfiguration] = None) -> None: """ Create new EvergreenConfigGenerator object. :param suites: The suite the Evergreen config will be generated for. :param options: The ConfigOptions object containing the config file values. :param evg_api: Evergreen API object. + :param generate_config: Configuration about how generation should be done. """ self.suites = suites self.options = options + self.gen_config = GenerationConfiguration.default_config() + if generate_config: + self.gen_config = generate_config self.evg_api = evg_api self.task_specs = [] self.task_names = [] self.build_tasks = None - def _get_distro(self) -> Optional[Sequence[str]]: - """Get the distros that the tasks should be run on.""" - if self.options.use_large_distro and self.options.large_distro_name: + def _get_distro(self, build_variant: str) -> Optional[Sequence[str]]: + """ + Get the distros that the tasks should be run on. + + :param build_variant: Name of build variant being generated. + :return: List of distros to run on. + """ + if self.options.use_large_distro: + if (build_variant not in self.gen_config.build_variant_large_distro_exceptions + and not self.options.large_distro_name): + print(NO_LARGE_DISTRO_ERR.format(build_variant=build_variant)) + raise ValueError("Invalid Evergreen Configuration") return [self.options.large_distro_name] return None @@ -777,7 +813,7 @@ class EvergreenConfigGenerator(object): tasks = self._generate_all_tasks() generating_task = {ExistingTask(task_name) for task_name in self.options.gen_task_set} - distros = self._get_distro() + distros = self._get_distro(build_variant.name) build_variant.display_task(self.options.display_task_name, execution_tasks=tasks, execution_existing_tasks=generating_task, distros=distros) @@ -785,15 +821,20 @@ class EvergreenConfigGenerator(object): class GenerateSubSuites(object): """Orchestrate the execution of generate_resmoke_suites.""" - def __init__(self, evergreen_api: EvergreenApi, config_options: ConfigOptions): + def __init__(self, evergreen_api: EvergreenApi, config_options: ConfigOptions, + generate_config: Optional[GenerationConfiguration] = None) -> None: """ Initialize the object. :param evergreen_api: Evergreen API client. :param config_options: Generation configuration options. + :param generate_config: Configuration for how generate tasks. """ self.evergreen_api = evergreen_api self.config_options = config_options + self.generate_options = GenerationConfiguration.default_config() + if generate_config: + self.generate_options = generate_config self.test_list = [] # Populate config values for methods like list_tests() @@ -916,8 +957,8 @@ class GenerateSubSuites(object): :param suites: Suites to add. :param build_variant: Build variant to add suite to. """ - EvergreenConfigGenerator(suites, self.config_options, self.evergreen_api) \ - .generate_config(build_variant) + EvergreenConfigGenerator(suites, self.config_options, self.evergreen_api, + self.generate_options).generate_config(build_variant) def generate_task_config(self, suites: List[Suite]) -> BuildVariant: """ @@ -983,7 +1024,7 @@ def filter_specified_tests(specified_tests: Set[str], tests_runtimes: List[TestR @click.command() @click.option("--expansion-file", type=str, required=True, help="Location of expansions file generated by evergreen.") -@click.option("--evergreen-config", type=str, default=CONFIG_FILE, +@click.option("--evergreen-config", type=str, default=EVG_CONFIG_FILE, help="Location of evergreen configuration file.") @click.option("--verbose", is_flag=True, default=False, help="Enable verbose logging.") def main(expansion_file, evergreen_config, verbose): @@ -998,10 +1039,11 @@ def main(expansion_file, evergreen_config, verbose): """ enable_logging(verbose) evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config) + generate_config = GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE) config_options = ConfigOptions.from_file(expansion_file, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES, CONFIG_FORMAT_FN) - GenerateSubSuites(evg_api, config_options).run() + GenerateSubSuites(evg_api, config_options, generate_config).run() if __name__ == "__main__": diff --git a/buildscripts/tests/test_evergreen_gen_multiversion_tests.py b/buildscripts/tests/test_evergreen_gen_multiversion_tests.py index 20da023bf48..62f683dc6f8 100644 --- a/buildscripts/tests/test_evergreen_gen_multiversion_tests.py +++ b/buildscripts/tests/test_evergreen_gen_multiversion_tests.py @@ -1,18 +1,15 @@ ''' Tests for the multiversion generators ''' import os -import shutil import unittest from tempfile import TemporaryDirectory, NamedTemporaryFile from mock import patch, MagicMock -from shrub.v2 import BuildVariant, ShrubProject -from shrub.variant import DisplayTaskDefinition from click.testing import CliRunner from buildscripts import evergreen_gen_multiversion_tests as under_test -from buildscripts.evergreen_generate_resmoke_tasks import read_yaml import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke +from buildscripts.util.fileops import read_yaml_file # pylint: disable=missing-docstring, no-self-use @@ -79,7 +76,7 @@ class TestGenerateExcludeYaml(unittest.TestCase): self._tmpdir.cleanup() def assert_contents(self, expected): - actual = read_yaml(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE) + actual = read_yaml_file(os.path.join(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE)) self.assertEqual(actual, expected) def patch_and_run(self, latest, last_stable): @@ -93,7 +90,7 @@ class TestGenerateExcludeYaml(unittest.TestCase): with patch.multiple('buildscripts.evergreen_gen_multiversion_tests', **mock_multiversion_methods): - with patch('buildscripts.evergreen_generate_resmoke_tasks.read_yaml', + with patch('buildscripts.evergreen_gen_multiversion_tests.read_yaml_file', return_value=latest) as mock_read_yaml: output = os.path.join(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE) diff --git a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py index 004527dec4a..d4c83ac76a2 100644 --- a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py +++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py @@ -661,6 +661,37 @@ class EvergreenConfigGeneratorTest(unittest.TestCase): self.assertEqual(options.large_distro_name, config["buildvariants"][0]["tasks"][0]["distros"][0]) + def test_build_variant_without_large_distro_defined_fails(self): + options = self.generate_mock_options() + options.use_large_distro = "true" + options.large_distro_name = None + suites = self.generate_mock_suites(3) + build_variant = BuildVariant("variant") + + generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock()) + with self.assertRaises(ValueError): + generator.generate_config(build_variant) + + def test_build_variant_without_large_distro_defined_can_be_ignored(self): + options = self.generate_mock_options() + options.use_large_distro = "true" + options.large_distro_name = None + suites = self.generate_mock_suites(3) + build_variant = BuildVariant("variant") + generate_config = under_test.GenerationConfiguration( + build_variant_large_distro_exceptions={"variant"}) + + generator = under_test.EvergreenConfigGenerator(suites, options, MagicMock(), + generate_config) + generator.generate_config(build_variant) + + shrub_project = ShrubProject.empty().add_build_variant(build_variant) + config = shrub_project.as_dict() + + self.assertEqual(len(config["tasks"]), len(suites) + 1) + self.assertEqual(options.large_distro_name, + config["buildvariants"][0]["tasks"][0]["distros"][0]) + def test_selecting_tasks(self): is_task_dependency = under_test.EvergreenConfigGenerator._is_task_dependency self.assertFalse(is_task_dependency("sharding", "sharding")) diff --git a/buildscripts/util/fileops.py b/buildscripts/util/fileops.py index e26e67c6593..2171452d343 100644 --- a/buildscripts/util/fileops.py +++ b/buildscripts/util/fileops.py @@ -1,6 +1,8 @@ """Utility to support file operations.""" - import os +from typing import Dict, Any + +import yaml def create_empty(path): @@ -52,3 +54,14 @@ def write_file_to_dir(directory: str, file: str, contents: str) -> None: os.makedirs(directory) write_file(os.path.join(directory, file), contents) + + +def read_yaml_file(path: str) -> Dict[str, Any]: + """ + Read the yaml file at the given path and return the contents. + + :param path: Path to file to read. + :return: Contents of given file. + """ + with open(path) as file_handle: + return yaml.safe_load(file_handle) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index fa1ede3301f..f332461b4b7 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -8779,6 +8779,7 @@ buildvariants: # generated suites. Once everything is converted to generated suites, we should remove the # '--repeatSuites=10' from the test_flags. This will be done in SERVER-38817. scons_cache_scope: shared + large_distro_name: rhel62-medium tasks: - name: compile_all_run_unittests_TG distros: @@ -8810,6 +8811,7 @@ buildvariants: test_flags: --excludeWithAnyTags=requires_http_client target_resmoke_time: 15 max_sub_suites: 3 + large_distro_name: rhel62-medium tasks: - name: compile_all_run_unittests_TG - name: .aggregation !.encrypt @@ -8854,6 +8856,7 @@ buildvariants: # exclude those tests as well. test_flags: --nojournal --excludeWithAnyTags=requires_journaling,requires_replication,requires_sharding,uses_transactions,requires_http_client scons_cache_scope: shared + large_distro_name: rhel62-medium tasks: - name: compile_all_run_unittests_TG distros: @@ -8890,6 +8893,7 @@ buildvariants: packager_distro: ubuntu1804 repo_edition: org scons_cache_scope: shared + large_distro_name: ubuntu1804-build tasks: - name: compile_all_run_unittests_TG distros: @@ -8946,6 +8950,7 @@ buildvariants: packager_distro: ubuntu1804 repo_edition: enterprise scons_cache_scope: shared + large_distro_name: ubuntu1804-build tasks: - name: compile_all_run_unittests_TG distros: @@ -9002,6 +9007,7 @@ buildvariants: packager_distro: ubuntu1604 repo_edition: org scons_cache_scope: shared + large_distro_name: ubuntu1604-build tasks: - name: compile_all_run_unittests_TG distros: @@ -9309,6 +9315,7 @@ buildvariants: packager_distro: ubuntu2004 repo_edition: org scons_cache_scope: shared + large_distro_name: ubuntu2004-large tasks: - name: compile_all_run_unittests_TG distros: @@ -9364,6 +9371,7 @@ buildvariants: packager_distro: ubuntu2004 repo_edition: enterprise scons_cache_scope: shared + large_distro_name: ubuntu2004-large tasks: - name: compile_all_run_unittests_TG distros: @@ -11862,6 +11870,7 @@ buildvariants: packager_distro: debian92 repo_edition: enterprise scons_cache_scope: shared + large_distro_name: debian92-build tasks: - name: compile_all_run_unittests_TG distros: @@ -11904,6 +11913,7 @@ buildvariants: packager_distro: debian92 repo_edition: org scons_cache_scope: shared + large_distro_name: debian92-build tasks: - name: compile_all_run_unittests_TG distros: @@ -11958,6 +11968,7 @@ buildvariants: packager_distro: debian10 repo_edition: enterprise scons_cache_scope: shared + large_distro_name: debian10-build tasks: - name: compile_all_run_unittests_TG distros: @@ -12000,6 +12011,7 @@ buildvariants: packager_distro: debian10 repo_edition: org scons_cache_scope: shared + large_distro_name: debian10-build tasks: - name: compile_all_run_unittests_TG distros: @@ -12493,6 +12505,7 @@ buildvariants: --mongosSetParameters="{replicaSetMonitorProtocol: scanning}" --mongodSetParameters="{replicaSetMonitorProtocol: scanning}" --excludeWithAnyTags=requires_streamable_rsm + large_distro_name: ubuntu1604-build tasks: - name: compile_all_run_unittests_TG @@ -12524,7 +12537,7 @@ buildvariants: --mongosSetParameters="{replicaSetMonitorProtocol: sdam}" --mongodSetParameters="{replicaSetMonitorProtocol: sdam}" --excludeWithAnyTags=requires_streamable_rsm - + large_distro_name: ubuntu1604-build tasks: - name: compile_all_run_unittests_TG distros: @@ -12553,6 +12566,7 @@ buildvariants: multiversion_edition: enterprise test_flags: |- # Set the taskExecutorPoolSize for all tests --mongosSetParameters="taskExecutorPoolSize: 4" + large_distro_name: ubuntu1604-build tasks: - name: compile_all_run_unittests_TG distros: diff --git a/etc/generate_subtasks_config.yml b/etc/generate_subtasks_config.yml new file mode 100644 index 00000000000..326067b0ef1 --- /dev/null +++ b/etc/generate_subtasks_config.yml @@ -0,0 +1,44 @@ +build_variant_large_distro_exceptions: + - amazon + - amazon2 + - debian10 + - debian92 + - enterprise-amazon2 + - enterprise-debian10-64 + - enterprise-debian92-64 + - enterprise-linux-64-amazon-ami + - enterprise-macos + - enterprise-rhel-62-64-bit-coverage + - enterprise-rhel-67-s390x + - enterprise-rhel-70-64-bit + - enterprise-rhel-70-64-bit-no-libunwind + - enterprise-rhel-71-ppc64le + - enterprise-rhel-71-ppc64le-inmem + - enterprise-rhel-72-s390x + - enterprise-rhel-72-s390x-inmem + - enterprise-rhel-80-64-bit + - enterprise-rhel-81-ppc64le + - enterprise-rhel-82-arm64 + - enterprise-suse12-64 + - enterprise-suse12-s390x + - enterprise-suse15-64 + - enterprise-ubuntu1604-64 + - enterprise-ubuntu1604-arm64 + - enterprise-ubuntu1804-arm64 + - enterprise-ubuntu1804-ppc64le + - enterprise-ubuntu1804-s390x + - enterprise-ubuntu2004-arm64 + - hot_backups-rhel-70-64-bit + - linux-64-ephemeralForTest + - macos + - rhel62 + - rhel70 + - rhel80 + - rhel-82-arm64 + - suse12 + - suse15 + - ubi7 + - ubuntu1604-container + - ubuntu1604-debug + - ubuntu1804-debug-asan + - ubuntu1804-debug-ubsan diff --git a/etc/pip/components/evergreen.req b/etc/pip/components/evergreen.req index 844c3088b2e..165015f1fe5 100644 --- a/etc/pip/components/evergreen.req +++ b/etc/pip/components/evergreen.req @@ -2,4 +2,5 @@ click ~= 7.0 dataclasses; python_version < "3.7" GitPython ~= 2.1.11 psutil +pydantic ~= 1.7.3 structlog ~= 19.1.0 |