diff options
author | David Bradford <david.bradford@mongodb.com> | 2021-06-11 19:43:08 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-07-07 19:38:59 +0000 |
commit | f7af7fe8600b5245bdfefd03081948a7525d39f4 (patch) | |
tree | f294aaa7c46f5ce157ef1e45f9e5ad8d1b588afe | |
parent | 25e6b31298e882b06b4afb6c72ada5d22e9aaeec (diff) | |
download | mongo-f7af7fe8600b5245bdfefd03081948a7525d39f4.tar.gz |
SERVER-57003: Generate resmoke tasks at build variant granularity
26 files changed, 1404 insertions, 1324 deletions
diff --git a/buildscripts/burn_in_tests_multiversion.py b/buildscripts/burn_in_tests_multiversion.py index 298da0ffe0d..5713c0309df 100644 --- a/buildscripts/burn_in_tests_multiversion.py +++ b/buildscripts/burn_in_tests_multiversion.py @@ -13,8 +13,6 @@ import structlog from structlog.stdlib import LoggerFactory from evergreen.api import EvergreenApi, RetryingEvergreenApi -import buildscripts.evergreen_gen_multiversion_tests as gen_multiversion -import buildscripts.evergreen_generate_resmoke_tasks as gen_resmoke from buildscripts.burn_in_tests import EVERGREEN_FILE, \ DEFAULT_REPO_LOCATIONS, create_tests_by_task, TaskInfo from buildscripts.ciconfig.evergreen import parse_evergreen_file, EvergreenProjectConfig @@ -36,9 +34,9 @@ from buildscripts.util.cmdutils import enable_logging structlog.configure(logger_factory=LoggerFactory()) LOGGER = structlog.getLogger(__name__) -MULTIVERSION_CONFIG_KEY = gen_multiversion.MULTIVERSION_CONFIG_KEY -MULTIVERSION_PASSTHROUGH_TAG = gen_multiversion.PASSTHROUGH_TAG -BURN_IN_MULTIVERSION_TASK = gen_multiversion.BURN_IN_TASK +MULTIVERSION_CONFIG_KEY = "use_in_multiversion" +MULTIVERSION_PASSTHROUGH_TAG = "multiversion_passthrough" +BURN_IN_MULTIVERSION_TASK = "burn_in_tests_multiversion" DEFAULT_CONFIG_DIR = "generated_resmoke_config" DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites") @@ -282,8 +280,7 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file, binder.bind(EvergreenProjectConfig, evg_conf) binder.bind(GenTaskOptions, gen_task_options) binder.bind(EvergreenApi, evg_api) - binder.bind(GenerationConfiguration, - GenerationConfiguration.from_yaml_file(gen_resmoke.GENERATE_CONFIG_FILE)) + binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) binder.bind(EvergreenFileChangeDetector, EvergreenFileChangeDetector(task_id, evg_api)) diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 877acf42026..8d96bd03055 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -148,6 +148,15 @@ class Task(object): """Return True if 'generate fuzzer tasks' command is found.""" return self.generate_fuzzer_tasks_command() is not None + def generate_implicit_multiversion_command(self): + """Return the 'generate implicit multiversion tasks' command if found, or None.""" + func = self.find_func_command("generate implicit multiversion tasks") + return func + + def is_generate_implicit_multiversion_task(self) -> bool: + """Return True if 'generate implicit multiverion tasks' command is found.""" + return self.generate_implicit_multiversion_command() is not None + @property def run_tests_command(self): """Return the 'run tests' command if found, or None.""" diff --git a/buildscripts/evergreen_activate_gen_tasks.py b/buildscripts/evergreen_activate_gen_tasks.py new file mode 100755 index 00000000000..e42395b5a13 --- /dev/null +++ b/buildscripts/evergreen_activate_gen_tasks.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Activate an evergreen task in the existing build.""" +import os +import sys + +import click +import structlog +from pydantic.main import BaseModel +from evergreen.api import RetryingEvergreenApi, EvergreenApi + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# pylint: disable=wrong-import-position +from buildscripts.util.cmdutils import enable_logging +from buildscripts.util.fileops import read_yaml_file +from buildscripts.util.taskname import remove_gen_suffix +# pylint: enable=wrong-import-position + +LOGGER = structlog.getLogger(__name__) + +EVG_CONFIG_FILE = "./.evergreen.yml" + + +class EvgExpansions(BaseModel): + """ + Evergreen expansions file contents. + + build_id: ID of build being run. + task_name: Name of task creating the generated configuration. + """ + + build_id: str + task_name: str + + @classmethod + def from_yaml_file(cls, path: str) -> "EvgExpansions": + """Read the generation configuration from the given file.""" + return cls(**read_yaml_file(path)) + + @property + def task(self) -> str: + """Get the task being generated.""" + return remove_gen_suffix(self.task_name) + + +def activate_task(build_id: str, task_name: str, evg_api: EvergreenApi) -> None: + """ + Activate the given task in the specified build. + + :param build_id: Build to activate task in. + :param task_name: Name of task to activate. + :param evg_api: Evergreen API client. + """ + build = evg_api.build_by_id(build_id) + task_list = build.get_tasks() + for task in task_list: + if task.display_name == task_name: + LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) + evg_api.configure_task(task.task_id, activated=True) + + +@click.command() +@click.option("--expansion-file", type=str, required=True, + help="Location of expansions file generated by evergreen.") +@click.option("--evergreen-config", type=str, default=EVG_CONFIG_FILE, + help="Location of evergreen configuration file.") +@click.option("--verbose", is_flag=True, default=False, help="Enable verbose logging.") +def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None: + """ + Activate the associated generated executions based in the running build. + + The `--expansion-file` should contain all the configuration needed to generate the tasks. + \f + :param expansion_file: Configuration file. + :param evergreen_config: Evergreen configuration file. + :param verbose: Use verbose logging. + """ + enable_logging(verbose) + expansions = EvgExpansions.from_yaml_file(expansion_file) + evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config) + + activate_task(expansions.build_id, expansions.task, evg_api) + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py new file mode 100644 index 00000000000..488b98e65fb --- /dev/null +++ b/buildscripts/evergreen_gen_build_variant.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Generate configuration for a build variant.""" +import os +import re +from concurrent.futures import ThreadPoolExecutor as Executor +from datetime import datetime, timedelta +from time import perf_counter +from typing import Optional, Any, List, Set + +import click +import inject +import structlog +from pydantic import BaseModel +from evergreen import EvergreenApi, RetryingEvergreenApi +from evergreen import Task as EvgTask + +from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, parse_evergreen_file, Task, \ + Variant +from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder +from buildscripts.task_generation.gen_config import GenerationConfiguration +from buildscripts.task_generation.gen_task_validation import GenTaskValidationService +from buildscripts.task_generation.multiversion_util import MultiversionUtilService, \ + SHARDED_MIXED_VERSION_CONFIGS, REPL_MIXED_VERSION_CONFIGS +from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig +from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters +from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \ + greedy_division, round_robin_fallback +from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskParams +from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions +from buildscripts.task_generation.task_types.multiversion_tasks import MultiversionGenTaskParams +from buildscripts.task_generation.task_types.resmoke_tasks import ResmokeGenTaskParams +from buildscripts.util.cmdutils import enable_logging +from buildscripts.util.fileops import read_yaml_file +from buildscripts.util.taskname import remove_gen_suffix + +LOGGER = structlog.get_logger(__name__) + +DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites") +MAX_WORKERS = 16 +LOOKBACK_DURATION_DAYS = 14 +MAX_TASK_PRIORITY = 99 +GENERATED_CONFIG_DIR = "generated_resmoke_config" +GEN_PARENT_TASK = "generator_tasks" +EXPANSION_RE = re.compile(r"\${(?P<id>[a-zA-Z0-9_]+)(\|(?P<default>.*))?}") + + +class EvgExpansions(BaseModel): + """ + Evergreen expansions needed to generate tasks. + + build_id: Build ID being run under. + build_variant: Build variant being generated. + gen_task_gran: Granularity of how tasks are being generated. + is_patch: Whether generation is part of a patch build. + project: Evergreen project being run under. + max_test_per_suite: Maximum amount of tests to include in a suite. + max_sub_suites: Maximum number of sub-suites to generate per task. + revision: Git revision being run against. + task_name: Name of task running. + target_resmoke_time: Target time of generated sub-suites. + task_id: ID of task being run under. + """ + + build_id: str + build_variant: str + is_patch: Optional[bool] + project: str + max_tests_per_suite: Optional[int] = 100 + max_sub_suites: Optional[int] = 5 + revision: str + task_name: str + target_resmoke_time: Optional[int] = None + task_id: str + + @classmethod + def from_yaml_file(cls, path: str) -> "EvgExpansions": + """ + Read the evergreen expansions from the given YAML file. + + :param path: Path to expansions YAML file. + :return: Expansions read from file. + """ + return cls(**read_yaml_file(path)) + + def build_suite_split_config(self, start_date: datetime, + end_date: datetime) -> SuiteSplitConfig: + """ + Get the configuration for splitting suites based on Evergreen expansions. + + :param start_date: Start date for historic stats lookup. + :param end_date: End date for historic stats lookup. + :return: Configuration to use for splitting suites. + """ + return SuiteSplitConfig( + evg_project=self.project, + target_resmoke_time=self.target_resmoke_time if self.target_resmoke_time else 60, + max_sub_suites=self.max_sub_suites, + max_tests_per_suite=self.max_tests_per_suite, + start_date=start_date, + end_date=end_date, + ) + + def build_evg_config_gen_options(self) -> GenTaskOptions: + """ + Get the configuration for generating tasks from Evergreen expansions. + + :return: Configuration to use for splitting suites. + """ + return GenTaskOptions( + create_misc_suite=True, + is_patch=self.is_patch, + generated_config_dir=GENERATED_CONFIG_DIR, + use_default_timeouts=False, + ) + + def config_location(self) -> str: + """Location where generated configuration is stored.""" + task = remove_gen_suffix(self.task_name) + return f"{self.build_variant}/{self.revision}/generate_tasks/{task}_gen-{self.build_id}.tgz" + + +def translate_run_var(run_var: str, build_variant: Variant) -> Any: + """ + Translate the given "run_var" into an actual value. + + Run_vars can contain evergreen expansions, in which case, the expansion (and possible default + value) need to be translated into a value we can use. + + :param run_var: Run var to translate. + :param build_variant: Build variant configuration. + :return: Value of run_var. + """ + match = EXPANSION_RE.search(run_var) + if match: + value = build_variant.expansion(match.group("id")) + if value is None: + value = match.group("default") + return value + return run_var + + +def get_version_configs(is_sharded: bool) -> List[str]: + """Get the version configurations to use.""" + if is_sharded: + return SHARDED_MIXED_VERSION_CONFIGS + return REPL_MIXED_VERSION_CONFIGS + + +class GenerateBuildVariantOrchestrator: + """Orchestrator for generating tasks in a build variant.""" + + # pylint: disable=too-many-arguments + @inject.autoparams() + def __init__( + self, + gen_task_validation: GenTaskValidationService, + gen_task_options: GenTaskOptions, + evg_project_config: EvergreenProjectConfig, + evg_expansions: EvgExpansions, + multiversion_util: MultiversionUtilService, + evg_api: EvergreenApi, + ) -> None: + """ + Initialize the orchestrator. + + :param gen_task_validation: Service to validate task generation. + :param gen_task_options: Options for how tasks should be generated. + :param evg_project_config: Configuration for Evergreen Project. + :param evg_expansions: Evergreen expansions for running task. + :param multiversion_util: Multiversion utility service. + :param evg_api: Evergreen API client. + """ + self.gen_task_validation = gen_task_validation + self.gen_task_options = gen_task_options + self.evg_project_config = evg_project_config + self.evg_expansions = evg_expansions + self.multiversion_util = multiversion_util + self.evg_api = evg_api + + def get_build_variant_expansion(self, build_variant_name: str, expansion: str) -> Any: + """ + Get the value of the given expansion for the specified build variant. + + :param build_variant_name: Build Variant to query. + :param expansion: Expansion to query. + :return: Value of given expansion. + """ + build_variant = self.evg_project_config.get_variant(build_variant_name) + return build_variant.expansion(expansion) + + @staticmethod + def task_def_to_split_params(task_def: Task, build_variant: str) -> SuiteSplitParameters: + """ + Build parameters for how a task should be split based on its task definition. + + :param task_def: Task definition in evergreen project config. + :param build_variant: Name of Build Variant being generated. + :return: Parameters for how task should be split. + """ + task = remove_gen_suffix(task_def.name) + run_func = task_def.generate_resmoke_tasks_command + if run_func is None: + run_func = task_def.generate_implicit_multiversion_command() + run_vars = run_func.get("vars", {}) + + suite = run_vars.get("suite", task) + return SuiteSplitParameters( + build_variant=build_variant, + task_name=task, + suite_name=suite, + filename=suite, + ) + + def task_def_to_gen_params(self, task_def: Task, build_variant: str) -> ResmokeGenTaskParams: + """ + Build parameters for how a task should be generated based on its task definition. + + :param task_def: Task definition in evergreen project config. + :param build_variant: Name of Build Variant being generated. + :return: Parameters for how task should be generated. + """ + run_func = task_def.generate_resmoke_tasks_command + run_vars = run_func["vars"] + + return ResmokeGenTaskParams( + use_large_distro=run_vars.get("use_large_distro"), + require_multiversion=run_vars.get("require_multiversion"), + repeat_suites=1, + resmoke_args=run_vars.get("resmoke_args"), + resmoke_jobs_max=run_vars.get("resmoke_jobs_max"), + large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), + config_location=self.evg_expansions.config_location(), + ) + + def task_def_to_mv_gen_params(self, task_def: Task, build_variant: str, is_sharded: bool, + version_config: List[str]) -> MultiversionGenTaskParams: + """ + Build parameters for how a task should be generated based on its task definition. + + :param task_def: Task definition in evergreen project config. + :param build_variant: Name of Build Variant being generated. + :return: Parameters for how task should be generated. + """ + run_func = task_def.generate_implicit_multiversion_command() + run_vars = run_func["vars"] + + task = remove_gen_suffix(task_def.name) + + return MultiversionGenTaskParams( + mixed_version_configs=version_config, + is_sharded=is_sharded, + resmoke_args=run_vars.get("resmoke_args"), + parent_task_name=task, + origin_suite=run_vars.get("suite", task), + use_large_distro=run_vars.get("use_large_distro"), + large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), + config_location=self.evg_expansions.config_location(), + ) + + def task_def_to_fuzzer_params( + self, task_def: Task, build_variant: str, is_sharded: Optional[bool] = None, + version_config: Optional[List[str]] = None) -> FuzzerGenTaskParams: + """ + Build parameters for how a fuzzer task should be generated based on its task definition. + + :param task_def: Task definition in evergreen project config. + :param build_variant: Name of Build Variant being generated. + :return: Parameters for how a fuzzer task should be generated. + """ + variant = self.evg_project_config.get_variant(build_variant) + if is_sharded is not None or version_config is not None: + run_func = task_def.generate_implicit_multiversion_command() + else: + run_func = task_def.generate_fuzzer_tasks_command() + run_vars = run_func["vars"] + run_vars = {k: translate_run_var(v, variant) for k, v in run_vars.items()} + + return FuzzerGenTaskParams( + task_name=run_vars.get("name"), + variant=build_variant, + suite=run_vars.get("suite"), + num_files=int(run_vars.get("num_files")), + num_tasks=int(run_vars.get("num_tasks")), + resmoke_args=run_vars.get("resmoke_args"), + npm_command=run_vars.get("npm_command", "jstestfuzz"), + jstestfuzz_vars=run_vars.get("jstestfuzz_vars", ""), + continue_on_failure=run_vars.get("continue_on_failure"), + resmoke_jobs_max=run_vars.get("resmoke_jobs_max"), + should_shuffle=run_vars.get("should_shuffle"), + timeout_secs=run_vars.get("timeout_secs"), + require_multiversion=run_vars.get("require_multiversion"), + use_large_distro=run_vars.get("use_large_distro", False), + large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), + config_location=self.evg_expansions.config_location(), + is_sharded=is_sharded, + version_config=version_config, + ) + + def generate(self, task_id: str, build_variant_name: str, output_file: str) -> None: + """ + Write task configuration for a build variant to disk. + + :param task_id: ID of running task. + :param build_variant_name: Name of build variant to generate. + :param output_file: Filename to write generated configuration to. + """ + if not self.gen_task_validation.should_task_be_generated(task_id): + LOGGER.info("Not generating configuration due to previous successful generation.") + return + + builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter + builder = self.generate_build_variant(builder, build_variant_name) + + generated_config = builder.build(output_file) + generated_config.write_all_to_dir(self.gen_task_options.generated_config_dir) + + # pylint: disable=too-many-locals + def generate_build_variant(self, builder: EvgConfigBuilder, + build_variant_name: str) -> EvgConfigBuilder: + """ + Generate task configuration for a build variant. + + :param builder: Evergreen configuration builder to use. + :param build_variant_name: Name of build variant to generate. + :return: Evergreen configuration builder with build variant configuration. + """ + LOGGER.info("Generating config", build_variant=build_variant_name) + start_time = perf_counter() + task_list = self.evg_project_config.get_variant(build_variant_name).task_names + tasks_to_hide = set() + with Executor(max_workers=MAX_WORKERS) as exe: + jobs = [] + for task_name in task_list: + task_def = self.evg_project_config.get_task(task_name) + if task_def.is_generate_resmoke_task: + tasks_to_hide.add(task_name) + split_params = self.task_def_to_split_params(task_def, build_variant_name) + gen_params = self.task_def_to_gen_params(task_def, build_variant_name) + jobs.append(exe.submit(builder.generate_suite, split_params, gen_params)) + elif task_def.is_generate_fuzzer_task(): + tasks_to_hide.add(task_name) + fuzzer_params = self.task_def_to_fuzzer_params(task_def, build_variant_name) + jobs.append(exe.submit(builder.generate_fuzzer, fuzzer_params)) + elif task_def.is_generate_implicit_multiversion_task(): + tasks_to_hide.add(task_name) + run_func = task_def.generate_implicit_multiversion_command() + run_vars = run_func["vars"] + is_jstestjuzz = run_vars.get("is_jstestfuzz", False) + suite = run_vars["suite"] + is_sharded = self.multiversion_util.is_suite_sharded(suite) + version_list = get_version_configs(is_sharded) + if is_jstestjuzz: + fuzzer_params = self.task_def_to_fuzzer_params( + task_def, build_variant_name, is_sharded, version_list) + jobs.append(exe.submit(builder.generate_fuzzer, fuzzer_params)) + else: + split_params = self.task_def_to_split_params(task_def, build_variant_name) + gen_params = self.task_def_to_mv_gen_params(task_def, build_variant_name, + is_sharded, version_list) + jobs.append( + exe.submit(builder.add_multiversion_suite, split_params, gen_params)) + + [j.result() for j in jobs] # pylint: disable=expression-not-assigned + + end_time = perf_counter() + duration = end_time - start_time + + LOGGER.info("Finished BV", build_variant=build_variant_name, duration=duration, + task_count=len(tasks_to_hide)) + + builder.add_display_task(GEN_PARENT_TASK, tasks_to_hide, build_variant_name) + self.adjust_gen_tasks_priority(tasks_to_hide) + return builder + + def adjust_task_priority(self, task: EvgTask) -> None: + """ + Increase the priority of the given task by 1. + + :param task: Task to increase priority of. + """ + priority = min(task.priority + 1, MAX_TASK_PRIORITY) + LOGGER.info("Configure task", task_id=task.task_id, priority=priority) + self.evg_api.configure_task(task.task_id, priority=priority) + + def adjust_gen_tasks_priority(self, gen_tasks: Set[str]) -> None: + """ + Increase the priority of any "_gen" tasks. + + We want to minimize the time it tasks for the "_gen" tasks to activate the generated + sub-tasks. We will do that by increase the priority of the "_gen" tasks. + + :param gen_tasks: Set of "_gen" tasks that were found. + """ + build = self.evg_api.build_by_id(self.evg_expansions.build_id) + task_list = build.get_tasks() + + with Executor(max_workers=MAX_WORKERS) as exe: + jobs = [ + exe.submit(self.adjust_task_priority, task) for task in task_list + if task.display_name in gen_tasks + ] + + [j.result() for j in jobs] # pylint: disable=expression-not-assigned + + +@click.command() +@click.option("--expansion-file", type=str, required=True, + help="Location of expansions file generated by evergreen.") +@click.option("--evg-api-config", type=str, required=True, + help="Location of evergreen api configuration.") +@click.option("--evg-project-config", type=str, default="etc/evergreen.yml", + help="Location of Evergreen project configuration.") +@click.option("--output-file", type=str, help="Name of output file to write.") +@click.option("--verbose", is_flag=True, default=False, help="Enable verbose logging.") +def main(expansion_file: str, evg_api_config: str, evg_project_config: str, output_file: str, + verbose: bool) -> None: + """ + Generate task configuration for a build variant. + \f + :param expansion_file: Location of evergreen expansions for task. + :param evg_api_config: Location of file containing evergreen API authentication information. + :param evg_project_config: Location of file containing evergreen project configuration. + :param output_file: Location to write generated configuration to. + :param verbose: Should verbose logging be used. + """ + enable_logging(verbose) + + end_date = datetime.utcnow().replace(microsecond=0) + start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS) + + evg_expansions = EvgExpansions.from_yaml_file(expansion_file) + + # pylint: disable=no-value-for-parameter + def dependencies(binder: inject.Binder) -> None: + binder.bind(EvgExpansions, evg_expansions) + binder.bind(SuiteSplitConfig, evg_expansions.build_suite_split_config(start_date, end_date)) + binder.bind(SplitStrategy, greedy_division) + binder.bind(FallbackStrategy, round_robin_fallback) + binder.bind(GenTaskOptions, evg_expansions.build_evg_config_gen_options()) + binder.bind(EvergreenApi, RetryingEvergreenApi.get_api(config_file=evg_api_config)) + binder.bind(EvergreenProjectConfig, parse_evergreen_file(evg_project_config)) + binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) + binder.bind(ResmokeProxyConfig, + ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) + + inject.configure(dependencies) + + orchestrator = GenerateBuildVariantOrchestrator() # pylint: disable=no-value-for-parameter + start_time = perf_counter() + orchestrator.generate(evg_expansions.task_id, evg_expansions.build_variant, output_file) + end_time = perf_counter() + + LOGGER.info("Total runtime", duration=end_time - start_time) + + +if __name__ == '__main__': + main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/evergreen_gen_fuzzer_tests.py b/buildscripts/evergreen_gen_fuzzer_tests.py deleted file mode 100755 index b0c41011741..00000000000 --- a/buildscripts/evergreen_gen_fuzzer_tests.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 -"""Generate fuzzer tests to run in evergreen in parallel.""" -import os -from typing import Optional - -import click -import inject -from pydantic import BaseModel -from evergreen import EvergreenApi, RetryingEvergreenApi - -from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder -from buildscripts.task_generation.gen_config import GenerationConfiguration -from buildscripts.task_generation.gen_task_service import GenTaskOptions, FuzzerGenTaskParams -from buildscripts.task_generation.gen_task_validation import GenTaskValidationService -from buildscripts.task_generation.generated_config import GeneratedConfiguration -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig -from buildscripts.task_generation.suite_split import SuiteSplitService -from buildscripts.util.cmdutils import enable_logging -from buildscripts.util.fileops import read_yaml_file - -CONFIG_DIRECTORY = "generated_resmoke_config" -GEN_PARENT_TASK = "generator_tasks" -GENERATE_CONFIG_FILE = "etc/generate_subtasks_config.yml" -DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites") -EVG_CONFIG_FILE = "./.evergreen.yml" - - -class EvgExpansions(BaseModel): - """ - Evergreen expansions to read for configuration. - - build_id: ID of build being run. - build_variant: Build Variant being run on. - continue_on_failure: Should tests continue after encountering a failure. - is_patch: Are tests being run in a patch build. - jstestfuzz_vars: Variable to pass to jstestfuzz command. - large_distro_name: Name of "large" distro to use. - name: Name of task to generate. - npm_command: NPM command to generate fuzzer tests. - num_files: Number of fuzzer files to generate. - num_tasks: Number of sub-tasks to generate. - resmoke_args: Arguments to pass to resmoke. - resmoke_jobs_max: Max number of jobs resmoke should execute in parallel. - revision: git revision being run against. - should_shuffle: Should remove shuffle tests before executing. - suite: Resmoke suite to run the tests. - task_id: ID of task currently being executed. - require_multiversion: Requires downloading Multiversion binaries. - timeout_secs: Timeout to set for task execution. - use_large_distro: Should tasks be generated to run on a large distro. - """ - - build_id: str - build_variant: str - continue_on_failure: bool - is_patch: Optional[bool] - jstestfuzz_vars: Optional[str] - large_distro_name: Optional[str] - name: str - npm_command: Optional[str] - num_files: int - num_tasks: int - resmoke_args: str - resmoke_jobs_max: int - revision: str - should_shuffle: bool - suite: str - task_id: str - timeout_secs: int - use_large_distro: Optional[bool] - require_multiversion: Optional[bool] - - @classmethod - def from_yaml_file(cls, path: str) -> "EvgExpansions": - """ - Read the generation configuration from the given file. - - :param path: Path to file. - :return: Parse evergreen expansions. - """ - return cls(**read_yaml_file(path)) - - def gen_task_options(self) -> GenTaskOptions: - """Determine the options for generating tasks based on the given expansions.""" - return GenTaskOptions( - is_patch=self.is_patch, - create_misc_suite=True, - generated_config_dir=CONFIG_DIRECTORY, - use_default_timeouts=False, - ) - - def fuzzer_gen_task_params(self) -> FuzzerGenTaskParams: - """Determine the parameters for generating fuzzer tasks based on the given expansions.""" - return FuzzerGenTaskParams( - task_name=self.name, num_files=self.num_files, num_tasks=self.num_tasks, - resmoke_args=self.resmoke_args, npm_command=self.npm_command or "jstestfuzz", - jstestfuzz_vars=self.jstestfuzz_vars, variant=self.build_variant, - continue_on_failure=self.continue_on_failure, resmoke_jobs_max=self.resmoke_jobs_max, - should_shuffle=self.should_shuffle, timeout_secs=self.timeout_secs, - require_multiversion=self.require_multiversion, suite=self.suite, - use_large_distro=self.use_large_distro, large_distro_name=self.large_distro_name, - config_location= - f"{self.build_variant}/{self.revision}/generate_tasks/{self.name}_gen-{self.build_id}.tgz" - ) - - -class EvgGenFuzzerOrchestrator: - """Orchestrate the generation of fuzzer tasks.""" - - @inject.autoparams() - def __init__(self, validation_service: GenTaskValidationService) -> None: - """ - Initialize the orchestrator. - - :param validation_service: Validation Service for generating tasks. - """ - self.validation_service = validation_service - - @staticmethod - def generate_config(fuzzer_params: FuzzerGenTaskParams) -> GeneratedConfiguration: - """ - Generate a fuzzer task based on the given parameters. - - :param fuzzer_params: Parameters describing how fuzzer should be generated. - :return: Configuration to generate the specified fuzzer. - """ - builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter - - builder.generate_fuzzer(fuzzer_params) - builder.add_display_task(GEN_PARENT_TASK, {f"{fuzzer_params.task_name}_gen"}, - fuzzer_params.variant) - return builder.build(fuzzer_params.task_name + ".json") - - def generate_fuzzer(self, task_id: str, fuzzer_params: FuzzerGenTaskParams) -> None: - """ - Save the configuration to generate the specified fuzzer to disk. - - :param task_id: ID of task doing the generation. - :param fuzzer_params: Parameters describing how fuzzer should be generated. - """ - if not self.validation_service.should_task_be_generated(task_id): - print("Not generating configuration due to previous successful generation.") - return - - generated_config = self.generate_config(fuzzer_params) - generated_config.write_all_to_dir(CONFIG_DIRECTORY) - - -@click.command() -@click.option("--expansion-file", type=str, required=True, - help="Location of expansions file generated by evergreen.") -@click.option("--evergreen-config", type=str, default=EVG_CONFIG_FILE, - help="Location of evergreen configuration file.") -@click.option("--verbose", is_flag=True, default=False, help="Enable verbose logging.") -def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None: - """Generate fuzzer tests to run in evergreen.""" - enable_logging(verbose) - - evg_expansions = EvgExpansions.from_yaml_file(expansion_file) - - def dependencies(binder: inject.Binder) -> None: - binder.bind(SuiteSplitService, None) - binder.bind(GenTaskOptions, evg_expansions.gen_task_options()) - binder.bind(EvergreenApi, RetryingEvergreenApi.get_api(config_file=evergreen_config)) - binder.bind(GenerationConfiguration, - GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE)) - binder.bind(ResmokeProxyConfig, - ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) - - inject.configure(dependencies) - - gen_fuzzer_orchestrator = EvgGenFuzzerOrchestrator() # pylint: disable=no-value-for-parameter - gen_fuzzer_orchestrator.generate_fuzzer(evg_expansions.task_id, - evg_expansions.fuzzer_gen_task_params()) - - -if __name__ == '__main__': - main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py index 05b6cdacb18..e4cfcaf2ec3 100755 --- a/buildscripts/evergreen_gen_multiversion_tests.py +++ b/buildscripts/evergreen_gen_multiversion_tests.py @@ -1,65 +1,34 @@ #!/usr/bin/env python3 """Generate multiversion tests to run in evergreen in parallel.""" -from datetime import datetime, timedelta import os import re import tempfile -from typing import Optional, List from collections import defaultdict from sys import platform from subprocess import check_output -import inject import requests import click import structlog -from pydantic import BaseModel - -from shrub.v2 import ExistingTask -from evergreen.api import RetryingEvergreenApi, EvergreenApi from buildscripts.resmokelib.multiversionconstants import ( LAST_LTS_MONGO_BINARY, LAST_CONTINUOUS_MONGO_BINARY, REQUIRES_FCV_TAG) -from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder -from buildscripts.task_generation.gen_config import GenerationConfiguration -from buildscripts.task_generation.generated_config import GeneratedConfiguration -from buildscripts.task_generation.multiversion_util import MultiversionUtilService -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig -from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters -from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \ - greedy_division, round_robin_fallback -from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskParams -from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions -from buildscripts.task_generation.task_types.multiversion_tasks import MultiversionGenTaskParams from buildscripts.util.cmdutils import enable_logging from buildscripts.util.fileops import read_yaml_file -import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke -import buildscripts.evergreen_gen_fuzzer_tests as gen_fuzzer import buildscripts.ciconfig.tags as _tags # pylint: disable=len-as-condition -from buildscripts.util.taskname import remove_gen_suffix LOGGER = structlog.getLogger(__name__) DEFAULT_CONFIG_DIR = "generated_resmoke_config" CONFIG_DIR = DEFAULT_CONFIG_DIR -DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites") -LOOKBACK_DURATION_DAYS = 14 -CONFIG_FILE = generate_resmoke.EVG_CONFIG_FILE -REPL_MIXED_VERSION_CONFIGS = ["new-old-new", "new-new-old", "old-new-new"] -SHARDED_MIXED_VERSION_CONFIGS = ["new-old-old-new"] - -BURN_IN_TASK = "burn_in_tests_multiversion" -MULTIVERSION_CONFIG_KEY = "use_in_multiversion" -PASSTHROUGH_TAG = "multiversion_passthrough" + BACKPORT_REQUIRED_TAG = "backport_required_multiversion" EXCLUDE_TAGS = f"{REQUIRES_FCV_TAG},multiversion_incompatible,{BACKPORT_REQUIRED_TAG}" EXCLUDE_TAGS_FILE = "multiversion_exclude_tags.yml" -GEN_PARENT_TASK = "generator_tasks" -ASAN_SIGNATURE = "detect_leaks=1" # The directory in which BACKPORTS_REQUIRED_FILE resides. ETC_DIR = "etc" @@ -67,158 +36,6 @@ BACKPORTS_REQUIRED_FILE = "backports_required_for_multiversion_tests.yml" BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/mongodb/mongo" -class EvgExpansions(BaseModel): - """Evergreen expansions file contents.""" - - project: str - target_resmoke_time: int = 60 - max_sub_suites: int = 5 - max_tests_per_suite: int = 100 - san_options: Optional[str] - task_name: str - suite: Optional[str] - num_files: Optional[int] - num_tasks: Optional[int] - resmoke_args: Optional[str] - npm_command: Optional[str] - jstestfuzz_vars: Optional[str] - build_variant: str - continue_on_failure: Optional[bool] - resmoke_jobs_max: Optional[int] - should_shuffle: Optional[bool] - timeout_secs: Optional[int] - require_multiversion: Optional[bool] - use_large_distro: Optional[bool] - large_distro_name: Optional[str] - revision: str - build_id: str - create_misc_suite: bool = True - is_patch: bool = False - is_jstestfuzz: bool = False - - @property - def task(self) -> str: - """Get the name of the task.""" - return remove_gen_suffix(self.task_name) - - @classmethod - def from_yaml_file(cls, path: str) -> "EvgExpansions": - """Read the evergreen expansions from the given file.""" - return cls(**read_yaml_file(path)) - - def config_location(self) -> str: - """Get the location to store the configuration.""" - return f"{self.build_variant}/{self.revision}/generate_tasks/{self.task}_gen-{self.build_id}.tgz" - - def is_asan_build(self) -> bool: - """Determine if this task is an ASAN build.""" - san_options = self.san_options - if san_options: - return ASAN_SIGNATURE in san_options - return False - - def get_generation_options(self) -> GenTaskOptions: - """Get options for how tasks should be generated.""" - return GenTaskOptions( - create_misc_suite=self.create_misc_suite, - is_patch=self.is_patch, - generated_config_dir=DEFAULT_CONFIG_DIR, - use_default_timeouts=False, - ) - - def get_fuzzer_params(self, version_config: str, is_sharded: bool) -> FuzzerGenTaskParams: - """ - Get parameters to generate fuzzer tasks. - - :param version_config: Version configuration to generate for. - :param is_sharded: If configuration is for sharded tests. - :return: Parameters to generate fuzzer tasks. - """ - name = f"{self.suite}_multiversion_{version_config}" - add_resmoke_args = get_multiversion_resmoke_args(is_sharded) - resmoke_args = f"{self.resmoke_args or ''} --mixedBinVersions={version_config} {add_resmoke_args}" - - return FuzzerGenTaskParams( - num_files=self.num_files, - num_tasks=self.num_tasks, - resmoke_args=resmoke_args, - npm_command=self.npm_command, - jstestfuzz_vars=self.jstestfuzz_vars, - task_name=name, - variant=self.build_variant, - continue_on_failure=self.continue_on_failure, - resmoke_jobs_max=self.resmoke_jobs_max, - should_shuffle=self.should_shuffle, - timeout_secs=self.timeout_secs, - require_multiversion=self.require_multiversion, - suite=self.suite or self.task, - use_large_distro=self.use_large_distro, - large_distro_name=self.large_distro_name, - config_location=self.config_location(), - ) - - def get_split_params(self) -> SuiteSplitParameters: - """Get the parameters specified to split suites.""" - return SuiteSplitParameters( - task_name=self.task_name, - suite_name=self.suite or self.task, - filename=self.suite or self.task, - test_file_filter=None, - build_variant=self.build_variant, - is_asan=self.is_asan_build(), - ) - - def get_split_config(self, start_date: datetime, end_date: datetime) -> SuiteSplitConfig: - """ - Get the configuration specifed to split suites. - - :param start_date: Start date for historic results query. - :param end_date: End date for historic results query. - :return: Configuration to use for splitting suites. - """ - return SuiteSplitConfig( - evg_project=self.project, - target_resmoke_time=self.target_resmoke_time, - max_sub_suites=self.max_sub_suites, - max_tests_per_suite=self.max_tests_per_suite, - start_date=start_date, - end_date=end_date, - ) - - def get_generation_params(self, is_sharded: bool) -> MultiversionGenTaskParams: - """ - Get the parameters to use to generating multiversion tasks. - - :param is_sharded: True if a sharded sutie is being generated. - :return: Parameters to use for generating multiversion tasks. - """ - version_config_list = get_version_configs(is_sharded) - return MultiversionGenTaskParams( - mixed_version_configs=version_config_list, - is_sharded=is_sharded, - resmoke_args=self.resmoke_args, - parent_task_name=self.task, - origin_suite=self.suite or self.task, - use_large_distro=self.use_large_distro, - large_distro_name=self.large_distro_name, - config_location=self.config_location(), - ) - - -def get_version_configs(is_sharded: bool) -> List[str]: - """Get the version configurations to use.""" - if is_sharded: - return SHARDED_MIXED_VERSION_CONFIGS - return REPL_MIXED_VERSION_CONFIGS - - -def get_multiversion_resmoke_args(is_sharded: bool) -> str: - """Return resmoke args used to configure a cluster for multiversion testing.""" - if is_sharded: - return "--numShards=2 --numReplSetNodes=2 " - return "--numReplSetNodes=3 --linearChain=on " - - def get_backports_required_hash_for_shell_version(mongo_shell_path=None): """Parse the last-lts shell binary to get the commit hash.""" if platform.startswith("win"): @@ -258,128 +75,12 @@ def get_last_lts_yaml(commit_hash): return backports_required_last_lts -class MultiVersionGenerateOrchestrator: - """An orchestrator for generating multiversion tasks.""" - - @inject.autoparams() - def __init__(self, evg_api: EvergreenApi, multiversion_util: MultiversionUtilService, - gen_task_options: GenTaskOptions) -> None: - """ - Initialize the orchestrator. - - :param evg_api: Evergreen API client. - :param multiversion_util: Multiverison utilities service. - :param gen_task_options: Options to use for generating tasks. - """ - self.evg_api = evg_api - self.multiversion_util = multiversion_util - self.gen_task_options = gen_task_options - - def generate_fuzzer(self, evg_expansions: EvgExpansions) -> GeneratedConfiguration: - """ - Generate configuration for the fuzzer task specified by the expansions. - - :param evg_expansions: Evergreen expansions describing what to generate. - :return: Configuration to generate the specified task. - """ - suite = evg_expansions.suite - is_sharded = self.multiversion_util.is_suite_sharded(suite) - version_config_list = get_version_configs(is_sharded) - - builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter - - fuzzer_task_set = set() - for version_config in version_config_list: - fuzzer_params = evg_expansions.get_fuzzer_params(version_config, is_sharded) - fuzzer_task = builder.generate_fuzzer(fuzzer_params) - fuzzer_task_set = fuzzer_task_set.union(fuzzer_task.sub_tasks) - - existing_tasks = {ExistingTask(task) for task in fuzzer_task_set} - existing_tasks.add({ExistingTask(f"{suite}_multiversion_gen")}) - builder.add_display_task(evg_expansions.task, existing_tasks, evg_expansions.build_variant) - return builder.build(f"{evg_expansions.task}.json") - - def generate_resmoke_suite(self, evg_expansions: EvgExpansions) -> GeneratedConfiguration: - """ - Generate configuration for the resmoke task specified by the expansions. - - :param evg_expansions: Evergreen expansions describing what to generate. - :return: Configuration to generate the specified task. - """ - suite = evg_expansions.suite or evg_expansions.task - is_sharded = self.multiversion_util.is_suite_sharded(suite) - - split_params = evg_expansions.get_split_params() - gen_params = evg_expansions.get_generation_params(is_sharded) - - builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter - builder.add_multiversion_suite(split_params, gen_params) - builder.add_display_task(GEN_PARENT_TASK, {f"{split_params.task_name}"}, - evg_expansions.build_variant) - return builder.build(f"{evg_expansions.task}.json") - - def generate(self, evg_expansions: EvgExpansions) -> None: - """ - Generate configuration for the specified task and save it to disk. - - :param evg_expansions: Evergreen expansions describing what to generate. - """ - if evg_expansions.is_jstestfuzz: - generated_config = self.generate_fuzzer(evg_expansions) - else: - generated_config = self.generate_resmoke_suite(evg_expansions) - generated_config.write_all_to_dir(DEFAULT_CONFIG_DIR) - - @click.group() def main(): """Serve as an entry point for the 'run' and 'generate-exclude-tags' commands.""" pass -@main.command("run") -@click.option("--expansion-file", type=str, required=True, - help="Location of expansions file generated by evergreen.") -@click.option("--evergreen-config", type=str, default=CONFIG_FILE, - help="Location of evergreen configuration file.") -def run_generate_tasks(expansion_file: str, evergreen_config: Optional[str] = None): - """ - Create a configuration for generate tasks to create sub suites for the specified resmoke suite. - - Tests using ReplicaSetFixture will be generated to use 3 nodes and linear_chain=True. - Tests using ShardedClusterFixture will be generated to use 2 shards with 2 nodes each. - The different binary version configurations tested are stored in REPL_MIXED_VERSION_CONFIGS - and SHARDED_MIXED_VERSION_CONFIGS. - - The `--expansion-file` should contain all the configuration needed to generate the tasks. - \f - :param expansion_file: Configuration file. - :param evergreen_config: Evergreen configuration file. - """ - enable_logging(False) - - end_date = datetime.utcnow().replace(microsecond=0) - start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS) - - evg_expansions = EvgExpansions.from_yaml_file(expansion_file) - - def dependencies(binder: inject.Binder) -> None: - binder.bind(SuiteSplitConfig, evg_expansions.get_split_config(start_date, end_date)) - binder.bind(SplitStrategy, greedy_division) - binder.bind(FallbackStrategy, round_robin_fallback) - binder.bind(GenTaskOptions, evg_expansions.get_generation_options()) - binder.bind(EvergreenApi, RetryingEvergreenApi.get_api(config_file=evergreen_config)) - binder.bind(GenerationConfiguration, - GenerationConfiguration.from_yaml_file(gen_fuzzer.GENERATE_CONFIG_FILE)) - binder.bind(ResmokeProxyConfig, - ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) - - inject.configure(dependencies) - - generate_orchestrator = MultiVersionGenerateOrchestrator() # pylint: disable=no-value-for-parameter - generate_orchestrator.generate(evg_expansions) - - @main.command("generate-exclude-tags") @click.option("--output", type=str, default=os.path.join(CONFIG_DIR, EXCLUDE_TAGS_FILE), show_default=True, help="Where to output the generated tags.") diff --git a/buildscripts/evergreen_generate_resmoke_tasks.py b/buildscripts/evergreen_generate_resmoke_tasks.py deleted file mode 100755 index 9a05eb14ca7..00000000000 --- a/buildscripts/evergreen_generate_resmoke_tasks.py +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env python3 -""" -Resmoke Test Suite Generator. - -Analyze the evergreen history for tests run under the given task and create new evergreen tasks -to attempt to keep the task runtime under a specified amount. -""" -from datetime import timedelta, datetime -import os -import sys -from typing import Optional - -import click -import inject -import structlog - -from pydantic.main import BaseModel -from evergreen.api import EvergreenApi, RetryingEvergreenApi - -# Get relative imports to work when the package is not installed on the PYTHONPATH. -from buildscripts.task_generation.gen_task_validation import GenTaskValidationService -from buildscripts.util.taskname import remove_gen_suffix - -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -# pylint: disable=wrong-import-position -from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder -from buildscripts.task_generation.gen_config import GenerationConfiguration -from buildscripts.task_generation.gen_task_service import GenTaskOptions, ResmokeGenTaskParams -from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \ - greedy_division, round_robin_fallback -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig -from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters -from buildscripts.util.cmdutils import enable_logging -from buildscripts.util.fileops import read_yaml_file -# pylint: enable=wrong-import-position - -LOGGER = structlog.getLogger(__name__) - -DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites") -EVG_CONFIG_FILE = "./.evergreen.yml" -GENERATE_CONFIG_FILE = "etc/generate_subtasks_config.yml" -LOOKBACK_DURATION_DAYS = 14 -GEN_SUFFIX = "_gen" -GEN_PARENT_TASK = "generator_tasks" -GENERATED_CONFIG_DIR = "generated_resmoke_config" -ASAN_SIGNATURE = "detect_leaks=1" - -DEFAULT_MAX_SUB_SUITES = 5 -DEFAULT_MAX_TESTS_PER_SUITE = 100 -DEFAULT_TARGET_RESMOKE_TIME = 60 - - -class EvgExpansions(BaseModel): - """ - Evergreen expansions file contents. - - build_id: ID of build being run. - build_variant: Build variant task is being generated under. - is_patch: Is this part of a patch build. - large_distro_name: Name of distro to use for 'large' tasks. - max_sub_suites: Max number of sub-suites to create for a single task. - max_tests_per_suite: Max number of tests to include in a single sub-suite. - project: Evergreen project being run in. - resmoke_args: Arguments to pass to resmoke for generated tests. - resmoke_jobs_max: Max number of jobs for resmoke to run in parallel. - resmoke_repeat_suites: Number of times resmoke should repeat each suite. - revision: Git revision being run against. - san_options: SAN options build variant is running under. - suite: Name of test suite being generated. - target_resmoke_time: Target time (in minutes) to keep sub-suite under. - task_id: ID of task creating the generated configuration. - task_name: Name of task creating the generated configuration. - use_large_distro: Should the generated tasks run on "large" distros. - require_multiversion: Requires downloading Multiversion binaries. - """ - - build_id: str - build_variant: str - is_patch: Optional[bool] - large_distro_name: Optional[str] - max_sub_suites: int = DEFAULT_MAX_SUB_SUITES - max_tests_per_suite: int = DEFAULT_MAX_TESTS_PER_SUITE - project: str - resmoke_args: str = "" - resmoke_jobs_max: Optional[int] - resmoke_repeat_suites: int = 1 - revision: str - san_options: Optional[str] - suite: Optional[str] - target_resmoke_time: int = DEFAULT_TARGET_RESMOKE_TIME - task_id: str - task_name: str - use_large_distro: bool = False - require_multiversion: Optional[bool] - - @classmethod - def from_yaml_file(cls, path: str) -> "EvgExpansions": - """Read the generation configuration from the given file.""" - return cls(**read_yaml_file(path)) - - @property - def task(self) -> str: - """Get the task being generated.""" - return remove_gen_suffix(self.task_name) - - def is_asan_build(self) -> bool: - """Determine if this task is an ASAN build.""" - san_options = self.san_options - if san_options: - return ASAN_SIGNATURE in san_options - return False - - def get_suite_split_config(self, start_date: datetime, end_date: datetime) -> SuiteSplitConfig: - """ - Get the configuration for splitting suites based on Evergreen expansions. - - :param start_date: Start date for historic stats lookup. - :param end_date: End date for historic stats lookup. - :return: Configuration to use for splitting suites. - """ - return SuiteSplitConfig( - evg_project=self.project, - target_resmoke_time=self.target_resmoke_time, - max_sub_suites=self.max_sub_suites, - max_tests_per_suite=self.max_tests_per_suite, - start_date=start_date, - end_date=end_date, - ) - - def get_evg_config_gen_options(self, generated_config_dir: str) -> GenTaskOptions: - """ - Get the configuration for generating tasks from Evergreen expansions. - - :param generated_config_dir: Directory to write generated configuration. - :return: Configuration to use for splitting suites. - """ - return GenTaskOptions( - create_misc_suite=True, - is_patch=self.is_patch, - generated_config_dir=generated_config_dir, - use_default_timeouts=False, - ) - - def get_suite_split_params(self) -> SuiteSplitParameters: - """Get the parameters to use for splitting suites.""" - task = remove_gen_suffix(self.task_name) - return SuiteSplitParameters( - build_variant=self.build_variant, - task_name=task, - suite_name=self.suite or task, - filename=self.suite or task, - test_file_filter=None, - is_asan=self.is_asan_build(), - ) - - def get_gen_params(self) -> "ResmokeGenTaskParams": - """Get the parameters to use for generating tasks.""" - return ResmokeGenTaskParams( - use_large_distro=self.use_large_distro, large_distro_name=self.large_distro_name, - require_multiversion=self.require_multiversion, - repeat_suites=self.resmoke_repeat_suites, resmoke_args=self.resmoke_args, - resmoke_jobs_max=self.resmoke_jobs_max, config_location= - f"{self.build_variant}/{self.revision}/generate_tasks/{self.task}_gen-{self.build_id}.tgz" - ) - - -class EvgGenResmokeTaskOrchestrator: - """Orchestrator for generating an resmoke tasks.""" - - @inject.autoparams() - def __init__(self, gen_task_validation: GenTaskValidationService, - gen_task_options: GenTaskOptions) -> None: - """ - Initialize the orchestrator. - - :param gen_task_validation: Generate tasks validation service. - :param gen_task_options: Options for how tasks are generated. - """ - self.gen_task_validation = gen_task_validation - self.gen_task_options = gen_task_options - - def generate_task(self, task_id: str, split_params: SuiteSplitParameters, - gen_params: ResmokeGenTaskParams) -> None: - """ - Generate the specified resmoke task. - - :param task_id: Task ID of generating task. - :param split_params: Parameters describing how the task should be split. - :param gen_params: Parameters describing how the task should be generated. - """ - LOGGER.debug("config options", split_params=split_params, gen_params=gen_params) - if not self.gen_task_validation.should_task_be_generated(task_id): - LOGGER.info("Not generating configuration due to previous successful generation.") - return - - builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter - - builder.generate_suite(split_params, gen_params) - builder.add_display_task(GEN_PARENT_TASK, {f"{split_params.task_name}{GEN_SUFFIX}"}, - split_params.build_variant) - generated_config = builder.build(split_params.task_name + ".json") - generated_config.write_all_to_dir(self.gen_task_options.generated_config_dir) - - -@click.command() -@click.option("--expansion-file", type=str, required=True, - help="Location of expansions file generated by evergreen.") -@click.option("--evergreen-config", type=str, default=EVG_CONFIG_FILE, - help="Location of evergreen configuration file.") -@click.option("--verbose", is_flag=True, default=False, help="Enable verbose logging.") -def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None: - """ - Create a configuration for generate tasks to create sub suites for the specified resmoke suite. - - The `--expansion-file` should contain all the configuration needed to generate the tasks. - \f - :param expansion_file: Configuration file. - :param evergreen_config: Evergreen configuration file. - :param verbose: Use verbose logging. - """ - enable_logging(verbose) - - end_date = datetime.utcnow().replace(microsecond=0) - start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS) - - evg_expansions = EvgExpansions.from_yaml_file(expansion_file) - - def dependencies(binder: inject.Binder) -> None: - binder.bind(SuiteSplitConfig, evg_expansions.get_suite_split_config(start_date, end_date)) - binder.bind(SplitStrategy, greedy_division) - binder.bind(FallbackStrategy, round_robin_fallback) - binder.bind(GenTaskOptions, evg_expansions.get_evg_config_gen_options(GENERATED_CONFIG_DIR)) - binder.bind(EvergreenApi, RetryingEvergreenApi.get_api(config_file=evergreen_config)) - binder.bind(GenerationConfiguration, - GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE)) - binder.bind(ResmokeProxyConfig, - ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) - - inject.configure(dependencies) - - gen_task_orchestrator = EvgGenResmokeTaskOrchestrator() # pylint: disable=no-value-for-parameter - gen_task_orchestrator.generate_task(evg_expansions.task_id, - evg_expansions.get_suite_split_params(), - evg_expansions.get_gen_params()) - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py index bbb6e4fbbfd..a535705cb98 100644 --- a/buildscripts/selected_tests.py +++ b/buildscripts/selected_tests.py @@ -43,15 +43,12 @@ from buildscripts.ciconfig.evergreen import ( parse_evergreen_file, Variant, ) -from buildscripts.evergreen_generate_resmoke_tasks import ( - DEFAULT_TEST_SUITE_DIR, - GENERATE_CONFIG_FILE, -) from buildscripts.patch_builds.selected_tests.selected_tests_service import SelectedTestsService structlog.configure(logger_factory=LoggerFactory()) LOGGER = structlog.getLogger(__name__) +DEFAULT_TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites") TASK_ID_EXPANSION = "task_id" EVERGREEN_FILE = "etc/evergreen.yml" EVG_CONFIG_FILE = ".evergreen.yml" @@ -505,8 +502,7 @@ def main( binder.bind(SplitStrategy, greedy_division) binder.bind(FallbackStrategy, round_robin_fallback) binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options()) - binder.bind(GenerationConfiguration, - GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE)) + binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) diff --git a/buildscripts/task_generation/evg_config_builder.py b/buildscripts/task_generation/evg_config_builder.py index 09ba4dfab7d..0f40418bef4 100644 --- a/buildscripts/task_generation/evg_config_builder.py +++ b/buildscripts/task_generation/evg_config_builder.py @@ -56,7 +56,7 @@ class EvgConfigBuilder: :return: BuildVariant object being created. """ if build_variant not in self.build_variants: - self.build_variants[build_variant] = BuildVariant(build_variant) + self.build_variants[build_variant] = BuildVariant(build_variant, activate=False) return self.build_variants[build_variant] def _generate_suites_config(self, generated_suite: GeneratedSuite) -> List[GeneratedFile]: diff --git a/buildscripts/task_generation/gen_config.py b/buildscripts/task_generation/gen_config.py index dba51a44ce1..94834de089b 100644 --- a/buildscripts/task_generation/gen_config.py +++ b/buildscripts/task_generation/gen_config.py @@ -5,6 +5,8 @@ from pydantic import BaseModel from buildscripts.util.fileops import read_yaml_file +GENERATE_CONFIG_FILE = "etc/generate_subtasks_config.yml" + class GenerationConfiguration(BaseModel): """Configuration for generating sub-tasks.""" @@ -12,7 +14,7 @@ class GenerationConfiguration(BaseModel): build_variant_large_distro_exceptions: Set[str] @classmethod - def from_yaml_file(cls, path: str) -> "GenerationConfiguration": + def from_yaml_file(cls, path: str = GENERATE_CONFIG_FILE) -> "GenerationConfiguration": """Read the generation configuration from the given file.""" return cls(**read_yaml_file(path)) diff --git a/buildscripts/task_generation/gen_task_service.py b/buildscripts/task_generation/gen_task_service.py index 1a8eb463237..6492df50995 100644 --- a/buildscripts/task_generation/gen_task_service.py +++ b/buildscripts/task_generation/gen_task_service.py @@ -82,9 +82,9 @@ class GenTaskService: params.large_distro_name) if params.add_to_display_task: build_variant.display_task(fuzzer_task.task_name, fuzzer_task.sub_tasks, - distros=distros) + distros=distros, activate=False) else: - build_variant.add_tasks(fuzzer_task.sub_tasks, distros=distros) + build_variant.add_tasks(fuzzer_task.sub_tasks, distros=distros, activate=False) return fuzzer_task def generate_task(self, generated_suite: GeneratedSuite, build_variant: BuildVariant, @@ -102,7 +102,7 @@ class GenTaskService: distros = self._get_distro(build_variant.name, gen_params.use_large_distro, gen_params.large_distro_name) build_variant.display_task(generated_suite.display_task_name(), - execution_tasks=execution_tasks, distros=distros) + execution_tasks=execution_tasks, distros=distros, activate=False) def generate_multiversion_task(self, generated_suite: GeneratedSuite, build_variant: BuildVariant, @@ -121,7 +121,7 @@ class GenTaskService: distros = self._get_distro(build_variant.name, gen_params.use_large_distro, gen_params.large_distro_name) build_variant.display_task(generated_suite.display_task_name(), - execution_tasks=execution_tasks, distros=distros) + execution_tasks=execution_tasks, distros=distros, activate=False) def generate_multiversion_burnin_task(self, generated_suite: GeneratedSuite, gen_params: MultiversionGenTaskParams, diff --git a/buildscripts/task_generation/task_types/fuzzer_tasks.py b/buildscripts/task_generation/task_types/fuzzer_tasks.py index 883b8ca814c..9616dfbfaf5 100644 --- a/buildscripts/task_generation/task_types/fuzzer_tasks.py +++ b/buildscripts/task_generation/task_types/fuzzer_tasks.py @@ -1,11 +1,18 @@ """Task generation for fuzzer tasks.""" -from typing import NamedTuple, Set, Optional, Dict +from typing import NamedTuple, Set, Optional, Dict, List from shrub.v2 import Task, FunctionCall, TaskDependency from buildscripts.util import taskname +def get_multiversion_resmoke_args(is_sharded: bool) -> str: + """Return resmoke args used to configure a cluster for multiversion testing.""" + if is_sharded: + return "--numShards=2 --numReplSetNodes=2 " + return "--numReplSetNodes=3 --linearChain=on " + + class FuzzerTask(NamedTuple): """ Evergreen configuration for a generated fuzzer command. @@ -56,6 +63,8 @@ class FuzzerGenTaskParams(NamedTuple): large_distro_name: Optional[str] config_location: str add_to_display_task: bool = True + version_config: Optional[List[str]] = None + is_sharded: Optional[bool] = None def jstestfuzz_params(self) -> Dict[str, str]: """Build a dictionary of parameters to pass to jstestfuzz.""" @@ -64,6 +73,19 @@ class FuzzerGenTaskParams(NamedTuple): "npm_command": self.npm_command, } + def get_task_name(self, version: str) -> str: + """Get the name to use for generated tasks.""" + if version: + return f"{self.suite}_multiversion_{version}" + return self.task_name + + def get_resmoke_args(self) -> str: + """Get the resmoke arguments to use for generated tasks.""" + if self.is_sharded is not None: + mv_args = get_multiversion_resmoke_args(self.is_sharded) + return f"{self.resmoke_args or ''} --mixedBinVersions={self.version_config} {mv_args}" + return self.resmoke_args + class FuzzerGenTaskService: """A service for generating fuzzer tasks.""" @@ -75,30 +97,41 @@ class FuzzerGenTaskService: :param params: Parameters for how task should be generated. :return: Set of shrub tasks. """ - sub_tasks = {self.build_fuzzer_sub_task(index, params) for index in range(params.num_tasks)} - return FuzzerTask(task_name=params.task_name, sub_tasks=sub_tasks) + version_list = params.version_config + if version_list is None: + version_list = [""] + + sub_tasks = set() + for version in version_list: + sub_tasks = sub_tasks.union({ + self.build_fuzzer_sub_task(index, params, version) + for index in range(params.num_tasks) + }) + + return FuzzerTask(task_name=params.get_task_name(""), sub_tasks=sub_tasks) @staticmethod - def build_fuzzer_sub_task(task_index: int, params: FuzzerGenTaskParams) -> Task: + def build_fuzzer_sub_task(task_index: int, params: FuzzerGenTaskParams, version: str) -> Task: """ Build a shrub task to run the fuzzer. :param task_index: Index of sub task being generated. :param params: Parameters describing how tasks should be generated. + :param version: Multiversion version to generate against. :return: Shrub task to run the fuzzer. """ - sub_task_name = taskname.name_generated_task(params.task_name, task_index, params.num_tasks, - params.variant) + sub_task_name = taskname.name_generated_task( + params.get_task_name(version), task_index, params.num_tasks, params.variant) suite_arg = f"--suites={params.suite}" run_tests_vars = { "continue_on_failure": params.continue_on_failure, - "resmoke_args": f"{suite_arg} {params.resmoke_args}", + "resmoke_args": f"{suite_arg} {params.get_resmoke_args()}", "resmoke_jobs_max": params.resmoke_jobs_max, "should_shuffle": params.should_shuffle, "require_multiversion": params.require_multiversion, "timeout_secs": params.timeout_secs, - "task": params.task_name, + "task": params.get_task_name(version), "gen_task_config_location": params.config_location, } # yapf: disable diff --git a/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py b/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py index 10f556e9311..246525d9415 100644 --- a/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py +++ b/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py @@ -7,7 +7,8 @@ import buildscripts.task_generation.task_types.fuzzer_tasks as under_test # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access -def build_mock_fuzzer_params(multi_version=None, jstestfuzz_vars="vars for jstestfuzz"): +def build_mock_fuzzer_params(multi_version=None, jstestfuzz_vars="vars for jstestfuzz", + version_config=None, is_sharded=None): return under_test.FuzzerGenTaskParams( task_name="task name", variant="build variant", @@ -26,6 +27,8 @@ def build_mock_fuzzer_params(multi_version=None, jstestfuzz_vars="vars for jstes add_to_display_task=True, large_distro_name="large distro", config_location="config_location", + version_config=version_config, + is_sharded=is_sharded, ) @@ -44,6 +47,28 @@ class TestFuzzerGenTaskParams(unittest.TestCase): self.assertNotIn("None", params.jstestfuzz_params()["jstestfuzz_vars"]) + def test_get_task_name_with_no_version(self): + params = build_mock_fuzzer_params() + + self.assertEqual(params.task_name, params.get_task_name("")) + + def test_get_task_name_with_version(self): + params = build_mock_fuzzer_params() + expected_name = f"{params.suite}_multiversion_my_version" + + self.assertEqual(expected_name, params.get_task_name("my_version")) + + def test_get_non_sharded_resmoke_args(self): + params = build_mock_fuzzer_params() + + self.assertEqual(params.resmoke_args, params.get_resmoke_args()) + + def test_get_sharded_resmoke_args(self): + params = build_mock_fuzzer_params(is_sharded=True) + expected_args = f"{params.resmoke_args} --mixedBinVersions=None --numShards=2 --numReplSetNodes=2" + + self.assertEqual(expected_args, params.get_resmoke_args().strip()) + class TestGenerateTasks(unittest.TestCase): def test_fuzzer_tasks_are_generated(self): @@ -55,13 +80,23 @@ class TestGenerateTasks(unittest.TestCase): self.assertEqual(fuzzer_task.task_name, mock_params.task_name) self.assertEqual(len(fuzzer_task.sub_tasks), mock_params.num_tasks) + def test_fuzzers_for_multiversion_are_generated(self): + version_list = ["version0", "version1"] + mock_params = build_mock_fuzzer_params(version_config=version_list) + fuzzer_service = under_test.FuzzerGenTaskService() + + fuzzer_task = fuzzer_service.generate_tasks(mock_params) + + self.assertEqual(fuzzer_task.task_name, mock_params.task_name) + self.assertEqual(len(fuzzer_task.sub_tasks), mock_params.num_tasks * len(version_list)) + class TestBuildFuzzerSubTask(unittest.TestCase): def test_sub_task_should_be_built_correct(self): mock_params = build_mock_fuzzer_params() fuzzer_service = under_test.FuzzerGenTaskService() - sub_task = fuzzer_service.build_fuzzer_sub_task(3, mock_params) + sub_task = fuzzer_service.build_fuzzer_sub_task(3, mock_params, "") self.assertEqual(sub_task.name, f"{mock_params.task_name}_3_{mock_params.variant}") self.assertEqual(len(sub_task.commands), 4) @@ -70,7 +105,7 @@ class TestBuildFuzzerSubTask(unittest.TestCase): mock_params = build_mock_fuzzer_params(multi_version="multiversion value") fuzzer_service = under_test.FuzzerGenTaskService() - sub_task = fuzzer_service.build_fuzzer_sub_task(3, mock_params) + sub_task = fuzzer_service.build_fuzzer_sub_task(3, mock_params, "") self.assertEqual(sub_task.name, f"{mock_params.task_name}_3_{mock_params.variant}") self.assertEqual(len(sub_task.commands), 6) diff --git a/buildscripts/tests/test_burn_in_tests_multiversion.py b/buildscripts/tests/test_burn_in_tests_multiversion.py index 5d63fe76015..56aed5fc434 100644 --- a/buildscripts/tests/test_burn_in_tests_multiversion.py +++ b/buildscripts/tests/test_burn_in_tests_multiversion.py @@ -10,17 +10,16 @@ import unittest from mock import MagicMock, patch import inject -from shrub.v2 import BuildVariant, ShrubProject from evergreen import EvergreenApi import buildscripts.burn_in_tests_multiversion as under_test from buildscripts.burn_in_tests import TaskInfo from buildscripts.ciconfig.evergreen import parse_evergreen_file, EvergreenProjectConfig import buildscripts.resmokelib.parser as _parser -import buildscripts.evergreen_gen_multiversion_tests as gen_multiversion -from buildscripts.evergreen_burn_in_tests import GenerateBurnInExecutor, EvergreenFileChangeDetector -from buildscripts.evergreen_generate_resmoke_tasks import GENERATE_CONFIG_FILE +from buildscripts.evergreen_burn_in_tests import EvergreenFileChangeDetector from buildscripts.task_generation.gen_config import GenerationConfiguration +from buildscripts.task_generation.multiversion_util import REPL_MIXED_VERSION_CONFIGS, \ + SHARDED_MIXED_VERSION_CONFIGS from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig from buildscripts.task_generation.suite_split import SuiteSplitConfig from buildscripts.task_generation.suite_split_strategies import greedy_division, SplitStrategy, \ @@ -84,8 +83,8 @@ def create_multiversion_tests_by_task_mock(n_tasks, n_tests): _DATE = datetime(2018, 7, 15) BURN_IN_TESTS = "buildscripts.burn_in_tests" -NUM_REPL_MIXED_VERSION_CONFIGS = len(gen_multiversion.REPL_MIXED_VERSION_CONFIGS) -NUM_SHARDED_MIXED_VERSION_CONFIGS = len(gen_multiversion.SHARDED_MIXED_VERSION_CONFIGS) +NUM_REPL_MIXED_VERSION_CONFIGS = len(REPL_MIXED_VERSION_CONFIGS) +NUM_SHARDED_MIXED_VERSION_CONFIGS = len(SHARDED_MIXED_VERSION_CONFIGS) NS = "buildscripts.burn_in_tests_multiversion" @@ -165,8 +164,7 @@ def configure_dependencies(evg_api, split_config): binder.bind(FallbackStrategy, round_robin_fallback) binder.bind(GenTaskOptions, gen_task_options) binder.bind(EvergreenApi, evg_api) - binder.bind(GenerationConfiguration, - GenerationConfiguration.from_yaml_file(GENERATE_CONFIG_FILE)) + binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=under_test.DEFAULT_TEST_SUITE_DIR)) binder.bind(EvergreenFileChangeDetector, None) diff --git a/buildscripts/tests/test_evergreen_activate_gen_tasks.py b/buildscripts/tests/test_evergreen_activate_gen_tasks.py new file mode 100644 index 00000000000..dbb64380838 --- /dev/null +++ b/buildscripts/tests/test_evergreen_activate_gen_tasks.py @@ -0,0 +1,42 @@ +"""Unit tests for the generate_resmoke_suite script.""" +import unittest + +from mock import MagicMock + +from buildscripts import evergreen_activate_gen_tasks as under_test + +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access +# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter + + +def build_mock_task(name, task_id): + mock_task = MagicMock(display_name=name, task_id=task_id) + return mock_task + + +def build_mock_evg_api(mock_task_list): + mock_build = MagicMock() + mock_build.get_tasks.return_value = mock_task_list + mock_evg_api = MagicMock() + mock_evg_api.build_by_id.return_value = mock_build + return mock_evg_api + + +class TestActivateTask(unittest.TestCase): + def test_task_with_display_name_is_activated(self): + n_tasks = 5 + mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)] + mock_evg_api = build_mock_evg_api(mock_task_list) + + under_test.activate_task("build_id", "task_3", mock_evg_api) + + mock_evg_api.configure_task.assert_called_with("id_3", activated=True) + + def test_task_with_no_matching_name(self): + n_tasks = 5 + mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)] + mock_evg_api = build_mock_evg_api(mock_task_list) + + under_test.activate_task("build_id", "not_an_existing_task", mock_evg_api) + + mock_evg_api.configure_task.assert_not_called() diff --git a/buildscripts/tests/test_evergreen_gen_build_variant.py b/buildscripts/tests/test_evergreen_gen_build_variant.py new file mode 100644 index 00000000000..3204c63db83 --- /dev/null +++ b/buildscripts/tests/test_evergreen_gen_build_variant.py @@ -0,0 +1,317 @@ +"""Unit tests for the generate_resmoke_suite script.""" +from datetime import datetime, timedelta +import json +import os +from tempfile import TemporaryDirectory +import sys +import unittest + +import inject +import requests +import yaml +from mock import patch, MagicMock +from evergreen import EvergreenApi + +from buildscripts import evergreen_gen_build_variant as under_test +from buildscripts.ciconfig.evergreen import Variant, Task, EvergreenProjectConfig +from buildscripts.evergreen_gen_build_variant import EvgExpansions +from buildscripts.task_generation.gen_config import GenerationConfiguration +from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig +from buildscripts.task_generation.suite_split import SuiteSplitConfig +from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \ + greedy_division, round_robin_fallback +from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions + +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access +# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter + + +def build_mock_build_variant(expansions=None, task_list=None): + task_spec_list = [{"name": task.name} for task in task_list] if task_list else [] + config = { + "tasks": task_spec_list, + } + if expansions: + config["expansions"] = expansions + + if task_list is None: + task_list = [] + task_map = {task.name: task for task in task_list} + + return Variant(config, task_map, {}) + + +def build_mock_task(name, run_cmd, run_vars=None): + config = { + "name": + name, "commands": [ + {"func": "do setup"}, + { + "func": run_cmd, + "vars": run_vars if run_vars else {}, + }, + ] + } + return Task(config) + + +def build_mock_project_config(variant=None, task_defs=None): + mock_project = MagicMock() + if variant: + mock_project.get_variant.return_value = variant + + if task_defs: + mock_project.get_task.side_effect = task_defs + + return mock_project + + +def build_mock_expansions(): + mock_expansions = MagicMock() + mock_expansions.config_location.return_value = "/path/to/config" + return mock_expansions + + +def build_mock_evg_api(build_task_list): + mock_evg_api = MagicMock() + mock_evg_api.build_by_id.return_value.get_tasks.return_value = build_task_list + return mock_evg_api + + +def build_mock_orchestrator(build_expansions=None, task_def_list=None, build_task_list=None): + if build_expansions is None: + build_expansions = {} + if task_def_list is None: + task_def_list = [] + if build_task_list is None: + build_task_list = [] + + mock_build_variant = build_mock_build_variant(build_expansions, task_def_list) + mock_project = build_mock_project_config(mock_build_variant, task_def_list) + mock_evg_expansions = build_mock_expansions() + mock_evg_api = build_mock_evg_api(build_task_list) + + return under_test.GenerateBuildVariantOrchestrator( + gen_task_validation=MagicMock(), + gen_task_options=MagicMock(), + evg_project_config=mock_project, + evg_expansions=mock_evg_expansions, + multiversion_util=MagicMock(), + evg_api=mock_evg_api, + ) + + +class TestTranslateRunVar(unittest.TestCase): + def test_normal_value_should_be_returned(self): + run_var = "some value" + mock_build_variant = build_mock_build_variant() + self.assertEqual(run_var, under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_should_be_returned_from_build_variant(self): + run_var = "${my_expansion}" + value = "my value" + mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) + self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_not_found_should_return_none(self): + run_var = "${my_expansion}" + mock_build_variant = build_mock_build_variant(expansions={}) + self.assertIsNone(under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_not_found_should_return_default(self): + run_var = "${my_expansion|default}" + mock_build_variant = build_mock_build_variant(expansions={}) + self.assertEqual("default", under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_should_be_returned_from_build_variant_even_with_default(self): + run_var = "${my_expansion|default}" + value = "my value" + mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) + self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) + + +class TestTaskDefToSplitParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "resmoke_args": "run tests", + } + mock_task_def = build_mock_task("my_task", "generate resmoke tasks", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") + + self.assertEqual("build_variant", split_param.build_variant) + self.assertEqual("my_task", split_param.task_name) + self.assertEqual("my_task", split_param.suite_name) + self.assertEqual("my_task", split_param.filename) + + def test_params_should_allow_suite_to_be_overridden(self): + run_vars = { + "resmoke_args": "run tests", + "suite": "the suite", + } + mock_task_def = build_mock_task("my_task", "generate resmoke tasks", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") + + self.assertEqual("build_variant", split_param.build_variant) + self.assertEqual("my_task", split_param.task_name) + self.assertEqual("the suite", split_param.suite_name) + self.assertEqual("the suite", split_param.filename) + + +class TestTaskDefToGenParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "resmoke_args": "run tests", + } + mock_task_def = build_mock_task("my_task", "generate resmoke tasks", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") + + self.assertIsNone(gen_params.require_multiversion) + self.assertEqual("run tests", gen_params.resmoke_args) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + gen_params.config_location) + self.assertIsNone(gen_params.large_distro_name) + self.assertFalse(gen_params.use_large_distro) + + def test_params_should_be_overwritable(self): + run_vars = { + "resmoke_args": "run tests", + "use_large_distro": "true", + "require_multiversion": True, + } + mock_task_def = build_mock_task("my_task", "generate resmoke tasks", run_vars) + build_expansions = {"large_distro_name": "my large distro"} + mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, + task_def_list=[mock_task_def]) + gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") + + self.assertTrue(gen_params.require_multiversion) + self.assertEqual("run tests", gen_params.resmoke_args) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + gen_params.config_location) + self.assertEqual("my large distro", gen_params.large_distro_name) + self.assertTrue(gen_params.use_large_distro) + + +class TestTaskDefToFuzzerParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "name": "my_fuzzer", + "num_files": "5", + "num_tasks": "3", + } + mock_task_def = build_mock_task("my_task", "generate fuzzer tasks", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") + + self.assertEqual("my_fuzzer", fuzzer_params.task_name) + self.assertEqual(5, fuzzer_params.num_files) + self.assertEqual(3, fuzzer_params.num_tasks) + self.assertEqual("jstestfuzz", fuzzer_params.npm_command) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + fuzzer_params.config_location) + self.assertIsNone(fuzzer_params.large_distro_name) + self.assertFalse(fuzzer_params.use_large_distro) + + def test_params_should_be_overwritable(self): + run_vars = { + "name": "my_fuzzer", + "num_files": "${file_count|8}", + "num_tasks": "3", + "use_large_distro": "true", + "npm_command": "aggfuzzer", + } + mock_task_def = build_mock_task("my_task", "generate fuzzer tasks", run_vars) + build_expansions = {"large_distro_name": "my large distro"} + mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, + task_def_list=[mock_task_def]) + + fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") + + self.assertEqual("my_fuzzer", fuzzer_params.task_name) + self.assertEqual(8, fuzzer_params.num_files) + self.assertEqual(3, fuzzer_params.num_tasks) + self.assertEqual("aggfuzzer", fuzzer_params.npm_command) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + fuzzer_params.config_location) + self.assertEqual("my large distro", fuzzer_params.large_distro_name) + self.assertTrue(fuzzer_params.use_large_distro) + + +class TestGenerateBuildVariant(unittest.TestCase): + def test_a_whole_build_variant(self): + gen_run_vars = { + "resmoke_args": "run tests", + } + mv_gen_run_vars = {"resmoke_args": "run tests", "suite": "multiversion suite"} + fuzz_run_vars = { + "name": "my_fuzzer", + "num_files": "5", + "num_tasks": "3", + } + mv_fuzz_run_vars = { + "name": "my_fuzzer", + "num_files": "5", + "num_tasks": "3", + "is_jstestfuzz": "True", + "suite": "aggfuzzer", + } + mock_task_defs = [ + build_mock_task("my_gen_task", "generate resmoke tasks", gen_run_vars), + build_mock_task("my_fuzzer_task", "generate fuzzer tasks", fuzz_run_vars), + build_mock_task("my_mv_fuzzer_task", "generate implicit multiversion tasks", + mv_fuzz_run_vars), + build_mock_task("my_mv_gen_task", "generate implicit multiversion tasks", + mv_gen_run_vars), + ] + mock_orchestrator = build_mock_orchestrator(task_def_list=mock_task_defs) + builder = MagicMock() + + builder = mock_orchestrator.generate_build_variant(builder, "build variant") + + self.assertEqual(builder.generate_suite.call_count, 1) + self.assertEqual(builder.generate_fuzzer.call_count, 2) + self.assertEqual(builder.add_multiversion_suite.call_count, 1) + + +class TestAdjustTaskPriority(unittest.TestCase): + def test_task_is_updates(self): + starting_priority = 42 + task_id = "task 314" + mock_task = MagicMock(task_id=task_id, priority=starting_priority) + mock_orchestrator = build_mock_orchestrator() + + mock_orchestrator.adjust_task_priority(mock_task) + + mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, + priority=starting_priority + 1) + + def test_task_should_only_reach_99(self): + starting_priority = 99 + task_id = "task 314" + mock_task = MagicMock(task_id=task_id, priority=starting_priority) + mock_orchestrator = build_mock_orchestrator() + + mock_orchestrator.adjust_task_priority(mock_task) + + mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, + priority=starting_priority) + + +class TestAdjustGenTasksPriority(unittest.TestCase): + def test_gen_tasks_in_task_list_are_adjusted(self): + gen_tasks = {"task_3", "task_8", "task_13"} + n_build_tasks = 25 + mock_task_list = [ + MagicMock(display_name=f"task_{i}", priority=0) for i in range(n_build_tasks) + ] + mock_orchestrator = build_mock_orchestrator(build_task_list=mock_task_list) + + mock_orchestrator.adjust_gen_tasks_priority(gen_tasks) + + self.assertEqual(len(gen_tasks), mock_orchestrator.evg_api.configure_task.call_count) diff --git a/buildscripts/tests/test_evergreen_gen_multiversion_tests.py b/buildscripts/tests/test_evergreen_gen_multiversion_tests.py index 621d13978dd..9b7c666f623 100644 --- a/buildscripts/tests/test_evergreen_gen_multiversion_tests.py +++ b/buildscripts/tests/test_evergreen_gen_multiversion_tests.py @@ -2,13 +2,12 @@ import os import unittest -from tempfile import TemporaryDirectory, NamedTemporaryFile +from tempfile import TemporaryDirectory from mock import patch, MagicMock from click.testing import CliRunner from buildscripts import evergreen_gen_multiversion_tests as under_test -import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke from buildscripts.util.fileops import read_yaml_file # pylint: disable=missing-docstring, no-self-use diff --git a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py deleted file mode 100644 index 14e0bfd2f1f..00000000000 --- a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py +++ /dev/null @@ -1,231 +0,0 @@ -"""Unit tests for the generate_resmoke_suite script.""" -from datetime import datetime, timedelta -import json -import os -from tempfile import TemporaryDirectory -import sys -import unittest - -import inject -import requests -import yaml -from mock import patch, MagicMock -from evergreen import EvergreenApi - -from buildscripts import evergreen_generate_resmoke_tasks as under_test -from buildscripts.task_generation.gen_config import GenerationConfiguration -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig -from buildscripts.task_generation.suite_split import SuiteSplitConfig -from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \ - greedy_division, round_robin_fallback -from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions - -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access -# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter - - -def tst_stat_mock(file, duration, pass_count): - return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count) - - -def mock_test_stats_unavailable(evg_api_mock): - response = MagicMock(status_code=requests.codes.SERVICE_UNAVAILABLE) - evg_api_mock.test_stats_by_project.side_effect = requests.HTTPError(response=response) - return evg_api_mock - - -def mock_resmoke_config_file(test_list, filename): - config = { - "test_kind": "js_test", - "selector": { - "roots": test_list, - }, - "executor": { - "config": { - "shell_options": { - "global_vars": { - "TestData": { - "roleGraphInvalidationIsFatal": True, - } - } - } - } - } - } # yapf: disable - - with open(filename, "w") as fileh: - fileh.write(yaml.safe_dump(config)) - - -def configure_dependencies(evg_api, evg_expansions, config_dir, - test_suites_dir=under_test.DEFAULT_TEST_SUITE_DIR): - start_date = datetime.utcnow() - end_date = start_date - timedelta(weeks=2) - - def dependencies(binder: inject.Binder) -> None: - binder.bind(SuiteSplitConfig, evg_expansions.get_suite_split_config(start_date, end_date)) - binder.bind(SplitStrategy, greedy_division) - binder.bind(FallbackStrategy, round_robin_fallback) - binder.bind(GenTaskOptions, evg_expansions.get_evg_config_gen_options(config_dir)) - binder.bind(EvergreenApi, evg_api) - binder.bind(GenerationConfiguration, - GenerationConfiguration.from_yaml_file(under_test.GENERATE_CONFIG_FILE)) - binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=test_suites_dir)) - - inject.clear_and_configure(dependencies) - - -def build_mock_evg_expansions(target_resmoke_time=under_test.DEFAULT_TARGET_RESMOKE_TIME): - return under_test.EvgExpansions( - build_variant="build_variant", - max_sub_suites=100, - project="mongodb-mongo-master", - task_id="task314", - task_name="some_task_gen", - target_resmoke_time=target_resmoke_time, - build_id="build_id", - revision="abc123", - ) - - -class TestAcceptance(unittest.TestCase): - """A suite of Acceptance tests for evergreen_generate_resmoke_tasks.""" - - @staticmethod - def _mock_evg_api(successful_task=False): - evg_api_mock = MagicMock() - task_mock = evg_api_mock.task_by_id.return_value - task_mock.execution = 0 - if successful_task: - task_mock.execution = 1 - task_mock.get_execution.return_value.is_success.return_value = True - - return evg_api_mock - - @staticmethod - def _prep_dirs(tmpdir): - target_directory = os.path.join(tmpdir, "output") - source_directory = os.path.join(tmpdir, "input") - os.makedirs(source_directory) - - return target_directory, source_directory - - @staticmethod - def _mock_test_files(directory, n_tests, runtime, evg_api_mock, suites_config_mock): - test_list = [os.path.join(directory, f"test_name_{i}.js") for i in range(n_tests)] - mock_test_stats = [tst_stat_mock(file, runtime, 5) for file in test_list] - evg_api_mock.test_stats_by_project.return_value = mock_test_stats - suites_config_mock.return_value.tests = test_list - for test in test_list: - open(test, "w").close() - - return test_list - - def test_when_task_has_already_run_successfully(self): - """ - Given evergreen_generate_resmoke_tasks has already been run successfully by this task, - When it attempts to run again, - It does not generate any files. - """ - mock_evg_api = self._mock_evg_api(successful_task=True) - mock_evg_expansions = build_mock_evg_expansions() - - with TemporaryDirectory() as tmpdir: - configure_dependencies(mock_evg_api, mock_evg_expansions, tmpdir) - - orchestrator = under_test.EvgGenResmokeTaskOrchestrator() - orchestrator.generate_task(mock_evg_expansions.task_id, - mock_evg_expansions.get_suite_split_params(), - mock_evg_expansions.get_gen_params()) - - self.assertEqual(0, len(os.listdir(tmpdir))) - - @patch("buildscripts.resmokelib.suitesconfig.get_suite") - def test_when_evg_test_stats_is_down(self, suites_config_mock): - """ - Given Evergreen historic test stats endpoint is disabled, - When evergreen_generate_resmoke_tasks attempts to generate suites, - It generates suites based on "fallback_num_sub_suites". - """ - n_tests = 100 - mock_evg_api = mock_test_stats_unavailable(self._mock_evg_api()) - mock_evg_expansions = build_mock_evg_expansions() - task = mock_evg_expansions.task_name[:-4] - - with TemporaryDirectory() as tmpdir: - target_directory, source_directory = self._prep_dirs(tmpdir) - configure_dependencies(mock_evg_api, mock_evg_expansions, target_directory, - source_directory) - - suite_path = os.path.join(source_directory, task) - test_list = self._mock_test_files(source_directory, n_tests, 5, mock_evg_api, - suites_config_mock) - mock_resmoke_config_file(test_list, suite_path + ".yml") - - orchestrator = under_test.EvgGenResmokeTaskOrchestrator() - orchestrator.generate_task(mock_evg_expansions.task_id, - mock_evg_expansions.get_suite_split_params(), - mock_evg_expansions.get_gen_params()) - - # Were all the config files created? There should be one for each suite as well as - # the evergreen json config. - generated_files = os.listdir(target_directory) - # The expected suite count is the number of fallback suites + the _misc suite. - expected_suite_count = mock_evg_expansions.max_sub_suites + 1 - # We expect files for all the suites + the evergreen json config. - self.assertEqual(expected_suite_count + 1, len(generated_files)) - - # Taking a closer look at the evergreen json config. - expected_shrub_file = f"{task}.json" - self.assertIn(expected_shrub_file, generated_files) - with open(os.path.join(target_directory, expected_shrub_file)) as fileh: - shrub_config = json.load(fileh) - - # Is there a task in the config for all the suites we created? - self.assertEqual(expected_suite_count, len(shrub_config["tasks"])) - - @unittest.skipIf( - sys.platform.startswith("win"), "Since this test is messing with directories, " - "windows does not handle test generation correctly") - @patch("buildscripts.resmokelib.suitesconfig.get_suite") - def test_with_each_test_in_own_task(self, suites_config_mock): - """ - Given a task with all tests having a historic runtime over the target, - When evergreen_generate_resmoke_tasks attempts to generate suites, - It generates a suite for each test. - """ - n_tests = 4 - mock_evg_api = self._mock_evg_api() - mock_evg_expansions = build_mock_evg_expansions(target_resmoke_time=10) - task = mock_evg_expansions.task_name[:-4] - - with TemporaryDirectory() as tmpdir: - target_directory, source_directory = self._prep_dirs(tmpdir) - configure_dependencies(mock_evg_api, mock_evg_expansions, target_directory, - source_directory) - suite_path = os.path.join(source_directory, task) - test_list = self._mock_test_files(source_directory, n_tests, 15 * 60, mock_evg_api, - suites_config_mock) - mock_resmoke_config_file(test_list, suite_path + ".yml") - - orchestrator = under_test.EvgGenResmokeTaskOrchestrator() - orchestrator.generate_task(mock_evg_expansions.task_id, - mock_evg_expansions.get_suite_split_params(), - mock_evg_expansions.get_gen_params()) - - # Were all the config files created? There should be one for each suite as well as - # the evergreen json config. - generated_files = os.listdir(target_directory) - # The expected suite count is the number of tests + the _misc suite. - expected_suite_count = n_tests + 1 - # We expect files for all the suites + the evergreen json config. - self.assertEqual(expected_suite_count + 1, len(generated_files)) - - # Taking a closer look at the evergreen json config. - expected_shrub_file = f"{task}.json" - self.assertIn(expected_shrub_file, generated_files) - with open(os.path.join(target_directory, expected_shrub_file)) as fileh: - shrub_config = json.load(fileh) - - # Is there a task in the config for all the suites we created? - self.assertEqual(expected_suite_count, len(shrub_config["tasks"])) diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py index 9d8efdeb4aa..cd774444d34 100644 --- a/buildscripts/tests/test_selected_tests.py +++ b/buildscripts/tests/test_selected_tests.py @@ -58,8 +58,7 @@ def configure_dependencies(evg_api, evg_expansions, evg_project_config, selected binder.bind(FallbackStrategy, round_robin_fallback) binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options()) binder.bind(EvergreenApi, evg_api) - binder.bind(GenerationConfiguration, - GenerationConfiguration.from_yaml_file(under_test.GENERATE_CONFIG_FILE)) + binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=test_suites_dir)) binder.bind(SelectedTestsClient, selected_test_client) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 4a5a1d7d8b8..85db72c88d5 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -97,6 +97,16 @@ variables: resmoke_args: --help resmoke_jobs_max: 0 # No cap on number of jobs. +- &gen_task_template + name: gen_task_template + depends_on: + - name: build_variant_gen + - name: archive_dist_test + commands: + - func: "generate resmoke tasks" + vars: + resmoke_args: --help + - &benchmark_template name: benchmark_template depends_on: @@ -150,7 +160,9 @@ variables: - &jstestfuzz_template name: jstestfuzz_template exec_timeout_secs: 14400 # Time out the task if it runs for more than 4 hours. - depends_on: [] + depends_on: + - build_variant_gen + - archive_dist_test commands: - func: "generate fuzzer tasks" @@ -263,6 +275,7 @@ variables: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large + - name: build_variant_gen - name: burn_in_tests_gen - name: .aggfuzzer .common - name: audit @@ -1017,7 +1030,7 @@ functions: files: - src/generated_resmoke_config/*.json - "generate burn in tags": + "generate build variant": - *f_expansions_write - *configure_evergreen_api_credentials - command: subprocess.exec @@ -1025,89 +1038,62 @@ functions: params: binary: bash args: - - "./src/evergreen/burn_in_tests_generate.sh" + - "./src/evergreen/generate_build_variant.sh" - command: archive.targz_pack params: - target: burn_in_tags_gen.tgz - source_dir: src/generated_burn_in_tags_config + target: generate_tasks_config.tgz + source_dir: src/generated_resmoke_config include: - "*" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: burn_in_tags_gen.tgz - remote_file: ${project}/${build_variant}/${revision}/burn_in_tags_gen/burn_in_tags_gen-${build_id}.tgz + local_file: generate_tasks_config.tgz + remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task_name}-${build_id}.tgz bucket: mciuploads permissions: public-read - content_type: application/gzip - display_name: Burn_in_tags Task Config - Execution ${execution} + content_type: ${content_type|application/gzip} + display_name: Generated Task Config - Execution ${execution} + optional: true - command: generate.tasks params: + optional: true files: - - src/generated_burn_in_tags_config/burn_in_tags_gen.json + - src/generated_resmoke_config/*.json - "generate randomized multiversion tasks": - - command: manifest.load - - *git_get_project + "generate burn in tags": - *f_expansions_write - - *add_git_tag - - *kill_processes - - *cleanup_environment - - *set_up_venv - - *upload_pip_requirements - *configure_evergreen_api_credentials - - *f_expansions_write - - command: subprocess.exec type: test params: binary: bash args: - - "./src/evergreen/randomized_multiversion_tasks_generate.sh" - - - *do_multiversion_setup - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/randomized_multiversion_tasks_generate_exclude_tags.sh" + - "./src/evergreen/burn_in_tests_generate.sh" - command: archive.targz_pack params: - target: generate_tasks_config.tgz - source_dir: src/generated_resmoke_config - optional: true + target: burn_in_tags_gen.tgz + source_dir: src/generated_burn_in_tags_config include: - "*" - - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: generate_tasks_config.tgz - remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task_name}-${build_id}.tgz + local_file: burn_in_tags_gen.tgz + remote_file: ${project}/${build_variant}/${revision}/burn_in_tags_gen/burn_in_tags_gen-${build_id}.tgz bucket: mciuploads permissions: public-read content_type: application/gzip - display_name: Generated Task Config - Execution ${execution} - optional: true - - - command: timeout.update - params: - exec_timeout_secs: 10800 # 3 hours - timeout_secs: 10800 # 3 hours - - - *f_expansions_write - + display_name: Burn_in_tags Task Config - Execution ${execution} - command: generate.tasks params: - optional: true files: - - src/generated_resmoke_config/*.json + - src/generated_burn_in_tags_config/burn_in_tags_gen.json - "generate resmoke tasks": - - command: manifest.load - - *git_get_project + "generate randomized multiversion tasks": + - *fetch_artifacts - *f_expansions_write - *add_git_tag - *kill_processes @@ -1122,39 +1108,25 @@ functions: params: binary: bash args: - - "./src/evergreen/resmoke_tasks_generate.sh" - - - command: archive.targz_pack - params: - target: generate_tasks_config.tgz - source_dir: src/generated_resmoke_config - include: - - "*" - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: generate_tasks_config.tgz - remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task_name}-${build_id}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: Generated Task Config - Execution ${execution} - optional: true - - - command: timeout.update - params: - exec_timeout_secs: 10800 # 3 hours - timeout_secs: 10800 # 3 hours + - "./src/evergreen/gen_tasks_activate.sh" + "generate resmoke tasks": + - *fetch_artifacts - *f_expansions_write + - *add_git_tag + - *kill_processes + - *cleanup_environment + - *set_up_venv + - *upload_pip_requirements + - *f_expansions_write + - *configure_evergreen_api_credentials - - command: generate.tasks + - command: subprocess.exec + type: test params: - optional: true - files: - - src/generated_resmoke_config/*.json + binary: bash + args: + - "./src/evergreen/gen_tasks_activate.sh" # Used by generator "run generated tests": @@ -1177,6 +1149,11 @@ functions: - *determine_resmoke_jobs - *update_resmoke_jobs_expansions - *f_expansions_write + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/implicit_multiversions_tasks_generate.sh" - *execute_resmoke_tests # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't # directly actionable. We use type=setup rather than type=system or type=test for this command @@ -1305,8 +1282,7 @@ functions: "generate explicit multiversion tasks": - - command: manifest.load - - *git_get_project + - *fetch_artifacts - *f_expansions_write - *add_git_tag - *kill_processes @@ -1315,148 +1291,49 @@ functions: - *upload_pip_requirements - *f_expansions_write - *configure_evergreen_api_credentials - - *do_multiversion_setup - command: subprocess.exec type: test params: binary: bash args: - - "./src/evergreen/explicit_multiversion_tasks_generate.sh" - - - command: archive.targz_pack - params: - target: generate_tasks_config.tgz - source_dir: src/generated_resmoke_config - include: - - "*" - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: generate_tasks_config.tgz - remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task_name}-${build_id}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: Generated Task Config - Execution ${execution} - optional: true - - - command: timeout.update - params: - exec_timeout_secs: 10800 # 3 hours - timeout_secs: 10800 # 3 hours - - - *f_expansions_write - - - command: generate.tasks - params: - optional: true - files: - - src/generated_resmoke_config/*.json + - "./src/evergreen/gen_tasks_activate.sh" "generate implicit multiversion tasks": - - command: manifest.load - - *git_get_project + - *fetch_artifacts - *f_expansions_write - *add_git_tag - *kill_processes - *cleanup_environment - *set_up_venv - - *configure_evergreen_api_credentials + - *upload_pip_requirements - *f_expansions_write - - *do_multiversion_setup + - *configure_evergreen_api_credentials - - *f_expansions_write - command: subprocess.exec + type: test params: binary: bash args: - - "./src/evergreen/implicit_multiversions_tasks_generate.sh" - - - command: archive.targz_pack - params: - target: generate_tasks_config.tgz - source_dir: src/generated_resmoke_config - include: - - "*" - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: generate_tasks_config.tgz - remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task_name}-${build_id}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: Generated Task Config - Execution ${execution} - optional: true - - - command: timeout.update - params: - exec_timeout_secs: 10800 # 3 hours - timeout_secs: 10800 # 3 hours - - - *f_expansions_write - - - command: generate.tasks - params: - # Optional b/c reruns of this task will produce no JSON files to avoid - # generate.tasks failing - optional: true - files: - - src/generated_resmoke_config/*.json + - "./src/evergreen/gen_tasks_activate.sh" "generate fuzzer tasks": - - command: manifest.load - - *git_get_project + - *fetch_artifacts - *f_expansions_write - *add_git_tag - *kill_processes - *cleanup_environment - *set_up_venv - *upload_pip_requirements - - - *configure_evergreen_api_credentials - *f_expansions_write + - *configure_evergreen_api_credentials + - command: subprocess.exec + type: test params: binary: bash args: - - "./src/evergreen/fuzzer_tasks_generate.sh" - - - command: archive.targz_pack - params: - target: generate_tasks_config.tgz - source_dir: src/generated_resmoke_config - include: - - "*" - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: generate_tasks_config.tgz - remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${name}_gen-${build_id}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: Generated Task Config - Execution ${execution} - optional: true - - - command: timeout.update - params: - exec_timeout_secs: 10800 # 3 hours - timeout_secs: 10800 # 3 hours - - *f_expansions_write - - - command: generate.tasks - params: - optional: true - files: - - src/generated_resmoke_config/${name}.json + - "./src/evergreen/gen_tasks_activate.sh" "setup jstestfuzz": - *f_expansions_write @@ -2176,6 +2053,7 @@ tasks: - "patch_test_tags.tgz" - "./build/**.gcno" - "./etc/*san.suppressions" + - "./etc/backports_required_for_multiversion_tests.yml" - "./etc/expansions.default.yml" - "./etc/pip/**" - "./etc/repo_config.yaml" @@ -3895,7 +3773,8 @@ tasks: name: resharding_fuzzer_idempotency ## Tests that the multiversion test generation logic is not broken. -- name: multiversion_sanity_check_passthrough_gen +- <<: *gen_task_template + name: multiversion_sanity_check_passthrough_gen tags: [] commands: - func: "generate implicit multiversion tasks" @@ -3904,7 +3783,8 @@ tasks: resmoke_args: --storageEngine=wiredTiger --includeWithAnyTags=multiversion_sanity_check require_multiversion: true -- name: replica_sets_jscore_multiversion_passthrough_gen +- <<: *gen_task_template + name: replica_sets_jscore_multiversion_passthrough_gen tags: ["multiversion_passthrough"] commands: - func: "generate implicit multiversion tasks" @@ -4041,10 +3921,9 @@ tasks: resmoke_args: --suites=aggregation_read_concern_majority_passthrough --storageEngine=wiredTiger resmoke_jobs_factor: 0.5 -- name: aggregation_secondary_reads_gen +- <<: *gen_task_template + name: aggregation_secondary_reads_gen tags: ["aggregation", "secondary_reads"] - depends_on: - - name: aggregation commands: - func: "generate resmoke tasks" vars: @@ -4083,7 +3962,8 @@ tasks: vars: resmoke_args: --suites=audit --storageEngine=wiredTiger -- name: auth_gen +- <<: *gen_task_template + name: auth_gen tags: ["auth"] commands: - func: "generate resmoke tasks" @@ -4111,6 +3991,17 @@ tasks: repeat_tests_min: 2 repeat_tests_max: 1000 +- name: build_variant_gen + commands: + - command: manifest.load + - *git_get_project + - *f_expansions_write + - *kill_processes + - *cleanup_environment + - func: "set up venv" + - func: "upload pip requirements" + - func: "generate build variant" + - name: selected_tests_gen tags: [] commands: @@ -4123,7 +4014,8 @@ tasks: - func: "upload pip requirements" - func: "generate selected tests" -- name: auth_audit_gen +- <<: *gen_task_template + name: auth_audit_gen tags: ["auth", "audit"] commands: - func: "generate resmoke tasks" @@ -4139,7 +4031,7 @@ tasks: vars: resmoke_args: --suites=change_streams --storageEngine=wiredTiger -- <<: *task_template +- <<: *gen_task_template name: change_streams_multiversion_passthrough_gen tags: ["multiversion_passthrough"] commands: @@ -4202,11 +4094,9 @@ tasks: vars: resmoke_args: --suites=change_streams_sharded_collections_passthrough --storageEngine=wiredTiger -- <<: *task_template +- <<: *gen_task_template name: change_streams_sharded_collections_multiversion_passthrough_gen tags: ["multiversion_passthrough"] - depends_on: - - name: change_streams_multiversion_passthrough_gen commands: - func: "generate implicit multiversion tasks" vars: @@ -4668,14 +4558,16 @@ tasks: vars: resmoke_args: --suites=causally_consistent_jscore_txns_passthrough --storageEngine=wiredTiger -- name: sharded_causally_consistent_jscore_txns_passthrough_gen +- <<: *gen_task_template + name: sharded_causally_consistent_jscore_txns_passthrough_gen tags: ["sharding", "jscore", "causally_consistent", "txns"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: sharded_causally_consistent_jscore_txns_passthrough_without_snapshot_gen +- <<: *gen_task_template + name: sharded_causally_consistent_jscore_txns_passthrough_without_snapshot_gen tags: ["sharding", "wo_snapshot", "causally_consistent", "jscore"] commands: - func: "generate resmoke tasks" @@ -4683,7 +4575,8 @@ tasks: suite: sharded_causally_consistent_jscore_txns_passthrough resmoke_args: --storageEngine=wiredTiger --excludeWithAnyTags=uses_snapshot_read_concern -- name: causally_consistent_hedged_reads_jscore_passthrough_gen +- <<: *gen_task_template + name: causally_consistent_hedged_reads_jscore_passthrough_gen tags: ["causally_consistent", "sharding", "jscore"] commands: - func: "generate resmoke tasks" @@ -4708,7 +4601,8 @@ tasks: vars: resmoke_args: --suites=replica_sets_jscore_passthrough --storageEngine=wiredTiger -- name: replica_sets_reconfig_jscore_passthrough_gen +- <<: *gen_task_template + name: replica_sets_reconfig_jscore_passthrough_gen tags: [] commands: - func: "generate resmoke tasks" @@ -4716,7 +4610,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_reconfig_jscore_stepdown_passthrough_gen +- <<: *gen_task_template + name: replica_sets_reconfig_jscore_stepdown_passthrough_gen tags: [] commands: - func: "generate resmoke tasks" @@ -4732,7 +4627,8 @@ tasks: vars: resmoke_args: --suites=replica_sets_reconfig_kill_primary_jscore_passthrough --storageEngine=wiredTiger -- name: replica_sets_api_version_jscore_passthrough_gen +- <<: *gen_task_template + name: replica_sets_api_version_jscore_passthrough_gen tags: [] commands: - func: "generate resmoke tasks" @@ -4740,7 +4636,8 @@ tasks: resmoke_args: --storageEngine=wiredTiger fallback_num_sub_suites: 5 -- name: replica_sets_jscore_passthrough_gen +- <<: *gen_task_template + name: replica_sets_jscore_passthrough_gen tags: [] commands: - func: "generate resmoke tasks" @@ -4766,7 +4663,8 @@ tasks: vars: resmoke_args: --suites=replica_sets_multi_stmt_txn_jscore_passthrough --storageEngine=wiredTiger -- name: replica_sets_multi_stmt_txn_stepdown_jscore_passthrough_gen +- <<: *gen_task_template + name: replica_sets_multi_stmt_txn_stepdown_jscore_passthrough_gen tags: ["replica_sets", "non_maj_read"] commands: - func: "generate resmoke tasks" @@ -4792,14 +4690,16 @@ tasks: vars: resmoke_args: --suites=replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough --storageEngine=wiredTiger -- name: replica_sets_update_v1_oplog_jscore_passthrough_gen +- <<: *gen_task_template + name: replica_sets_update_v1_oplog_jscore_passthrough_gen tags: ["replica_sets", "non_maj_read"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_initsync_jscore_passthrough_gen +- <<: *gen_task_template + name: replica_sets_initsync_jscore_passthrough_gen tags: ["replica_sets", "san", "large"] commands: - func: "generate resmoke tasks" @@ -4851,7 +4751,8 @@ tasks: vars: resmoke_args: --suites=mongos_test -- name: multiversion_auth_gen +- <<: *gen_task_template + name: multiversion_auth_gen tags: ["auth", "multiversion"] commands: - func: "generate resmoke tasks" @@ -4859,7 +4760,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" require_multiversion: true -- name: multiversion_gen +- <<: *gen_task_template + name: multiversion_gen tags: [] commands: - func: "generate resmoke tasks" @@ -4870,7 +4772,8 @@ tasks: # Tests the runFeatureFlagMultiversionTest helper. # This requires the 'featureFlagToaster' and 'featureFlagSpoon' parameters to be set to true on # build variants that enable this task. -- name: feature_flag_multiversion_gen +- <<: *gen_task_template + name: feature_flag_multiversion_gen tags: [] commands: - func: "generate resmoke tasks" @@ -4878,7 +4781,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" require_multiversion: true -- name: unittest_shell_hang_analyzer_gen +- <<: *gen_task_template + name: unittest_shell_hang_analyzer_gen tags: [] commands: - func: "generate resmoke tasks" @@ -4886,7 +4790,8 @@ tasks: suite: unittest_shell_hang_analyzer resmoke_args: --storageEngine=wiredTiger -- name: noPassthrough_gen +- <<: *gen_task_template + name: noPassthrough_gen tags: ["misc_js"] commands: - func: "generate resmoke tasks" @@ -4896,7 +4801,8 @@ tasks: use_large_distro: "true" # Only run hot_backups tests for hot_backups variant. -- name: noPassthroughHotBackups_gen +- <<: *gen_task_template + name: noPassthroughHotBackups_gen tags: [] commands: - func: "generate resmoke tasks" @@ -4905,7 +4811,8 @@ tasks: resmoke_args: --storageEngine=wiredTiger src/mongo/db/modules/*/jstests/hot_backups/*.js use_large_distro: "true" -- name: noPassthroughWithMongod_gen +- <<: *gen_task_template + name: noPassthroughWithMongod_gen tags: ["misc_js"] commands: - func: "generate resmoke tasks" @@ -4914,7 +4821,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: slow1_gen +- <<: *gen_task_template + name: slow1_gen tags: ["misc_js", "non_win_dbg"] commands: - func: "generate resmoke tasks" @@ -4941,7 +4849,7 @@ tasks: vars: resmoke_args: --suites=sharded_collections_jscore_passthrough --storageEngine=wiredTiger -- <<: *task_template +- <<: *gen_task_template name: sharded_collections_jscore_multiversion_passthrough_gen tags: ["multiversion_passthrough"] commands: @@ -4960,7 +4868,7 @@ tasks: vars: resmoke_args: --suites=sharding_jscore_passthrough --storageEngine=wiredTiger -- <<: *task_template +- <<: *gen_task_template name: sharding_jscore_multiversion_passthrough_gen tags: ["multiversion_passthrough"] commands: @@ -4970,7 +4878,8 @@ tasks: resmoke_args: --storageEngine=wiredTiger require_multiversion: true -- name: sharding_api_version_jscore_passthrough_gen +- <<: *gen_task_template + name: sharding_api_version_jscore_passthrough_gen tags: ["sharding", "jscore"] commands: - func: "generate resmoke tasks" @@ -4996,7 +4905,8 @@ tasks: vars: resmoke_args: --suites=sharded_multi_stmt_txn_jscore_passthrough --storageEngine=wiredTiger -- name: multi_shard_multi_stmt_txn_jscore_passthrough_gen +- <<: *gen_task_template + name: multi_shard_multi_stmt_txn_jscore_passthrough_gen tags: ["multi_shard", "multi_stmt", "common"] commands: - func: "generate resmoke tasks" @@ -5005,14 +4915,16 @@ tasks: resmoke_args: --storageEngine=wiredTiger resmoke_jobs_max: 0 # No cap on number of jobs. -- name: multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough_gen +- <<: *gen_task_template + name: multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough_gen tags: ["multi_shard", "common"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: multi_stmt_txn_jscore_passthrough_with_migration_gen +- <<: *gen_task_template + name: multi_stmt_txn_jscore_passthrough_with_migration_gen tags: ["multi_stmt"] commands: - func: "generate resmoke tasks" @@ -5020,7 +4932,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough_gen +- <<: *gen_task_template + name: multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough_gen tags: ["multi_shard"] commands: - func: "generate resmoke tasks" @@ -5028,7 +4941,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough_gen +- <<: *gen_task_template + name: multi_shard_multi_stmt_txn_stepdown_primary_jscore_passthrough_gen tags: ["multi_shard"] commands: - func: "generate resmoke tasks" @@ -5036,7 +4950,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: tenant_migration_jscore_passthrough_gen +- <<: *gen_task_template + name: tenant_migration_jscore_passthrough_gen tags: ["tenant_migration"] commands: - func: "generate resmoke tasks" @@ -5044,7 +4959,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: tenant_migration_causally_consistent_jscore_passthrough_gen +- <<: *gen_task_template + name: tenant_migration_causally_consistent_jscore_passthrough_gen tags: ["tenant_migration"] commands: - func: "generate resmoke tasks" @@ -5052,7 +4968,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: tenant_migration_multi_stmt_txn_jscore_passthrough_gen +- <<: *gen_task_template + name: tenant_migration_multi_stmt_txn_jscore_passthrough_gen tags: ["tenant_migration", "txn"] commands: - func: "generate resmoke tasks" @@ -5060,7 +4977,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: tenant_migration_stepdown_jscore_passthrough_gen +- <<: *gen_task_template + name: tenant_migration_stepdown_jscore_passthrough_gen tags: ["tenant_migration"] commands: - func: "generate resmoke tasks" @@ -5068,7 +4986,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: tenant_migration_terminate_primary_jscore_passthrough_gen +- <<: *gen_task_template + name: tenant_migration_terminate_primary_jscore_passthrough_gen tags: ["tenant_migration"] commands: - func: "generate resmoke tasks" @@ -5077,7 +4996,8 @@ tasks: resmoke_args: --storageEngine=wiredTiger fallback_num_sub_suites: 10 -- name: tenant_migration_kill_primary_jscore_passthrough_gen +- <<: *gen_task_template + name: tenant_migration_kill_primary_jscore_passthrough_gen tags: ["tenant_migration"] commands: - func: "generate resmoke tasks" @@ -5086,7 +5006,8 @@ tasks: resmoke_args: --storageEngine=wiredTiger fallback_num_sub_suites: 10 -- name: parallel_gen +- <<: *gen_task_template + name: parallel_gen tags: ["misc_js"] commands: - func: "generate resmoke tasks" @@ -5124,7 +5045,8 @@ tasks: resmoke_args: --suites=concurrency_replication_metrics --storageEngine=wiredTiger resmoke_jobs_max: 1 -- name: concurrency_replication_gen +- <<: *gen_task_template + name: concurrency_replication_gen tags: ["concurrency", "common", "repl"] commands: - func: "generate resmoke tasks" @@ -5132,7 +5054,7 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- <<: *task_template +- <<: *gen_task_template name: concurrency_replication_multiversion_passthrough_gen tags: [multiversion_passthrough] commands: @@ -5142,7 +5064,8 @@ tasks: resmoke_args: --storageEngine=wiredTiger require_multiversion: true -- name: concurrency_replication_causal_consistency_gen +- <<: *gen_task_template + name: concurrency_replication_causal_consistency_gen tags: ["concurrency", "repl", "large", "non_live_record"] commands: - func: "generate resmoke tasks" @@ -5191,7 +5114,8 @@ tasks: resmoke_args: --suites=concurrency_replication_multi_stmt_txn_ubsan --storageEngine=wiredTiger resmoke_jobs_max: 1 -- name: concurrency_replication_wiredtiger_cursor_sweeps_gen +- <<: *gen_task_template + name: concurrency_replication_wiredtiger_cursor_sweeps_gen tags: ["concurrency", "repl"] commands: - func: "generate resmoke tasks" @@ -5199,7 +5123,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_replication_wiredtiger_eviction_debug_gen +- <<: *gen_task_template + name: concurrency_replication_wiredtiger_eviction_debug_gen tags: ["concurrency", "repl", "debug_only"] commands: - func: "generate resmoke tasks" @@ -5207,7 +5132,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_sharded_replication_gen +- <<: *gen_task_template + name: concurrency_sharded_replication_gen tags: ["concurrency", "common", "read_concern_maj", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5216,7 +5142,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_replication_multiversion_passthrough_gen +- <<: *gen_task_template + name: concurrency_sharded_replication_multiversion_passthrough_gen tags: ["multiversion_passthrough", "sharded"] commands: - func: "generate implicit multiversion tasks" @@ -5225,7 +5152,8 @@ tasks: resmoke_args: --storageEngine=wiredTiger require_multiversion: true -- name: concurrency_sharded_replication_with_balancer_gen +- <<: *gen_task_template + name: concurrency_sharded_replication_with_balancer_gen tags: ["concurrency", "common", "read_concern_maj", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5234,7 +5162,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_replication_no_txns_gen +- <<: *gen_task_template + name: concurrency_sharded_replication_no_txns_gen tags: ["concurrency", "no_txns", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5244,7 +5173,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_replication_no_txns_with_balancer_gen +- <<: *gen_task_template + name: concurrency_sharded_replication_no_txns_with_balancer_gen tags: ["concurrency", "no_txns", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5254,7 +5184,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_clusterwide_ops_add_remove_shards_gen +- <<: *gen_task_template + name: concurrency_sharded_clusterwide_ops_add_remove_shards_gen tags: ["concurrency", "common", "read_concern_maj", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5263,7 +5194,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_causal_consistency_gen +- <<: *gen_task_template + name: concurrency_sharded_causal_consistency_gen tags: ["concurrency", "non_live_record", "sharded"] commands: - func: "generate resmoke tasks" @@ -5272,7 +5204,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_causal_consistency_and_balancer_gen +- <<: *gen_task_template + name: concurrency_sharded_causal_consistency_and_balancer_gen tags: ["concurrency", "large", "non_live_record", "sharded"] commands: - func: "generate resmoke tasks" @@ -5281,7 +5214,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_with_stepdowns_gen +- <<: *gen_task_template + name: concurrency_sharded_with_stepdowns_gen tags: ["concurrency", "stepdowns", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5290,7 +5224,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_with_stepdowns_and_balancer_gen +- <<: *gen_task_template + name: concurrency_sharded_with_stepdowns_and_balancer_gen tags: ["concurrency", "stepdowns", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5299,7 +5234,8 @@ tasks: use_large_distro: "true" resmoke_jobs_max: 1 -- name: concurrency_sharded_terminate_primary_with_balancer_gen +- <<: *gen_task_template + name: concurrency_sharded_terminate_primary_with_balancer_gen tags: ["concurrency", "stepdowns", "kill_terminate", "sharded"] commands: - func: "generate resmoke tasks" @@ -5308,7 +5244,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_sharded_kill_primary_with_balancer_gen +- <<: *gen_task_template + name: concurrency_sharded_kill_primary_with_balancer_gen tags: ["concurrency", "stepdowns", "kill_terminate", "sharded"] commands: - func: "generate resmoke tasks" @@ -5317,7 +5254,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_sharded_multi_stmt_txn_gen +- <<: *gen_task_template + name: concurrency_sharded_multi_stmt_txn_gen tags: ["concurrency", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5326,7 +5264,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_sharded_multi_stmt_txn_with_balancer_gen +- <<: *gen_task_template + name: concurrency_sharded_multi_stmt_txn_with_balancer_gen tags: ["concurrency", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5335,7 +5274,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_sharded_local_read_write_multi_stmt_txn_gen +- <<: *gen_task_template + name: concurrency_sharded_local_read_write_multi_stmt_txn_gen tags: ["concurrency", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5344,7 +5284,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer_gen +- <<: *gen_task_template + name: concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer_gen tags: ["concurrency", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5353,7 +5294,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_sharded_multi_stmt_txn_with_stepdowns_gen +- <<: *gen_task_template + name: concurrency_sharded_multi_stmt_txn_with_stepdowns_gen tags: ["concurrency", "stepdowns", "large", "sharded"] commands: - func: "generate resmoke tasks" @@ -5362,7 +5304,7 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- <<: *task_template +- <<: *gen_task_template name: concurrency_sharded_multi_stmt_txn_terminate_primary_gen tags: ["concurrency", "stepdowns", "kill_terminate", "sharded"] commands: @@ -5372,7 +5314,7 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- <<: *task_template +- <<: *gen_task_template name: concurrency_sharded_multi_stmt_txn_kill_primary_gen tags: ["concurrency", "stepdowns", "kill_terminate", "sharded"] commands: @@ -5382,7 +5324,8 @@ tasks: resmoke_args: "--storageEngine=wiredTiger" resmoke_jobs_max: 1 -- name: concurrency_simultaneous_gen +- <<: *gen_task_template + name: concurrency_simultaneous_gen tags: ["concurrency", "common"] commands: - func: "generate resmoke tasks" @@ -5429,7 +5372,8 @@ tasks: vars: resmoke_args: --suites=read_concern_linearizable_passthrough --storageEngine=wiredTiger -- name: read_concern_majority_passthrough_gen +- <<: *gen_task_template + name: read_concern_majority_passthrough_gen tags: ["read_write_concern"] commands: - func: "generate resmoke tasks" @@ -5455,7 +5399,8 @@ tasks: vars: resmoke_args: --suites=cwrwc_passthrough --storageEngine=wiredTiger -- name: cwrwc_rc_majority_passthrough_gen +- <<: *gen_task_template + name: cwrwc_rc_majority_passthrough_gen tags: ["read_write_concern"] commands: - func: "generate resmoke tasks" @@ -5472,7 +5417,8 @@ tasks: vars: resmoke_args: --suites=cwrwc_wc_majority_passthrough --storageEngine=wiredTiger -- name: secondary_reads_passthrough_gen +- <<: *gen_task_template + name: secondary_reads_passthrough_gen tags: [] commands: - func: "generate resmoke tasks" @@ -5480,14 +5426,16 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_gen +- <<: *gen_task_template + name: replica_sets_gen tags: ["replica_sets", "san", "large"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_ese_gen +- <<: *gen_task_template + name: replica_sets_ese_gen tags: ["replica_sets", "encrypt", "san"] commands: - func: "generate resmoke tasks" @@ -5495,7 +5443,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_ese_gcm_gen +- <<: *gen_task_template + name: replica_sets_ese_gcm_gen tags: ["replica_sets", "encrypt", "san", "gcm"] commands: - func: "generate resmoke tasks" @@ -5503,7 +5452,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_auth_gen +- <<: *gen_task_template + name: replica_sets_auth_gen tags: ["replica_sets", "common", "san", "auth"] commands: - func: "generate resmoke tasks" @@ -5511,28 +5461,31 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_large_txns_format_gen +- <<: *gen_task_template + name: replica_sets_large_txns_format_gen tags: ["replica_sets", "multi_oplog", "san"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_max_mirroring_gen +- <<: *gen_task_template + name: replica_sets_max_mirroring_gen tags: ["replica_sets", "san"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: replica_sets_update_v1_oplog_gen +- <<: *gen_task_template + name: replica_sets_update_v1_oplog_gen tags: ["replica_sets", "san"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- <<: *task_template +- <<: *gen_task_template name: replica_sets_multiversion_gen tags: ["random_multiversion_ds"] commands: @@ -5551,7 +5504,8 @@ tasks: vars: resmoke_args: --suites=sasl --storageEngine=wiredTiger -- name: sharding_gen +- <<: *gen_task_template + name: sharding_gen tags: ["sharding", "common"] commands: - func: "generate resmoke tasks" @@ -5559,7 +5513,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: sharding_multiversion_gen +- <<: *gen_task_template + name: sharding_multiversion_gen tags: ["random_multiversion_ds"] commands: - func: "generate randomized multiversion tasks" @@ -5569,7 +5524,8 @@ tasks: require_multiversion: true suite: sharding_multiversion -- name: sharding_max_mirroring_gen +- <<: *gen_task_template + name: sharding_max_mirroring_gen tags: ["sharding", "common"] commands: - func: "generate resmoke tasks" @@ -5577,7 +5533,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: sharding_csrs_continuous_config_stepdown_gen +- <<: *gen_task_template + name: sharding_csrs_continuous_config_stepdown_gen tags: ["sharding", "common", "csrs", "non_live_record"] commands: - func: "generate resmoke tasks" @@ -5586,7 +5543,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: sharding_ese_gen +- <<: *gen_task_template + name: sharding_ese_gen tags: ["sharding", "encrypt"] commands: - func: "generate resmoke tasks" @@ -5594,7 +5552,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: sharding_ese_gcm_gen +- <<: *gen_task_template + name: sharding_ese_gcm_gen tags: ["sharding", "encrypt", "gcm"] commands: - func: "generate resmoke tasks" @@ -5602,7 +5561,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: sharding_auth_gen +- <<: *gen_task_template + name: sharding_auth_gen tags: ["sharding", "auth"] commands: - func: "generate resmoke tasks" @@ -5610,7 +5570,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: sharding_auth_audit_gen +- <<: *gen_task_template + name: sharding_auth_audit_gen tags: ["auth", "audit", "non_live_record"] commands: - func: "generate resmoke tasks" @@ -5618,7 +5579,8 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: sharding_last_lts_mongos_and_mixed_shards_gen +- <<: *gen_task_template + name: sharding_last_lts_mongos_and_mixed_shards_gen tags: ["sharding", "common", "multiversion"] commands: - func: "generate explicit multiversion tasks" @@ -5627,7 +5589,8 @@ tasks: require_multiversion: true resmoke_args: --tagFile=generated_resmoke_config/multiversion_exclude_tags.yml -- name: sharding_update_v1_oplog_gen +- <<: *gen_task_template + name: sharding_update_v1_oplog_gen tags: ["sharding", "common"] commands: - func: "generate resmoke tasks" @@ -5647,14 +5610,16 @@ tasks: snmp_config_path: SNMPCONFPATH=snmpconf resmoke_args: --suites=snmp --storageEngine=wiredTiger -- name: ssl_gen +- <<: *gen_task_template + name: ssl_gen tags: ["encrypt", "ssl"] commands: - func: "generate resmoke tasks" vars: resmoke_args: "--storageEngine=wiredTiger --mongodSetParameters='{logComponentVerbosity: {network: 2, replication: {heartbeats: 2}}}'" -- name: sslSpecial_gen +- <<: *gen_task_template + name: sslSpecial_gen tags: ["encrypt", "ssl"] commands: - func: "generate resmoke tasks" @@ -5662,7 +5627,8 @@ tasks: suite: ssl_special resmoke_args: --storageEngine=wiredTiger -- name: ssl_x509_gen +- <<: *gen_task_template + name: ssl_x509_gen tags: ["encrypt", "ssl"] commands: - func: "generate resmoke tasks" @@ -5706,42 +5672,48 @@ tasks: vars: resmoke_args: --suites=session_jscore_passthrough --storageEngine=wiredTiger -- name: causally_consistent_jscore_passthrough_gen +- <<: *gen_task_template + name: causally_consistent_jscore_passthrough_gen tags: ["causally_consistent"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: causally_consistent_jscore_passthrough_auth_gen +- <<: *gen_task_template + name: causally_consistent_jscore_passthrough_auth_gen tags: ["causally_consistent"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: causally_consistent_read_concern_snapshot_passthrough_gen +- <<: *gen_task_template + name: causally_consistent_read_concern_snapshot_passthrough_gen tags: ["causally_consistent", "read_write_concern", "durable_history"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: sharded_causally_consistent_read_concern_snapshot_passthrough_gen +- <<: *gen_task_template + name: sharded_causally_consistent_read_concern_snapshot_passthrough_gen tags: ["causally_consistent", "read_write_concern", "durable_history"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: sharded_causally_consistent_jscore_passthrough_gen +- <<: *gen_task_template + name: sharded_causally_consistent_jscore_passthrough_gen tags: ["causally_consistent"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: retryable_writes_jscore_passthrough_gen +- <<: *gen_task_template + name: retryable_writes_jscore_passthrough_gen tags: ["retry"] commands: - func: "generate resmoke tasks" @@ -5749,91 +5721,104 @@ tasks: use_large_distro: "true" resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_replication_default_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_replication_default_refresh_jscore_passthrough_gen tags: ["logical_session_cache", "repl"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_replication_100ms_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_replication_100ms_refresh_jscore_passthrough_gen tags: ["logical_session_cache", "repl"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_replication_1sec_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_replication_1sec_refresh_jscore_passthrough_gen tags: ["logical_session_cache", "one_sec", "repl"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_replication_10sec_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_replication_10sec_refresh_jscore_passthrough_gen tags: ["logical_session_cache", "repl"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_sharding_default_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_sharding_default_refresh_jscore_passthrough_gen tags: ["logical_session_cache"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough_gen tags: ["logical_session_cache"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_sharding_100ms_refresh_jscore_txns_passthrough_gen tags: ["logical_session_cache"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough_gen tags: ["logical_session_cache", "one_sec"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough_gen tags: ["logical_session_cache"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_standalone_default_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_standalone_default_refresh_jscore_passthrough_gen tags: ["logical_session_cache"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough_gen tags: ["logical_session_cache"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough_gen tags: ["logical_session_cache", "one_sec"] commands: - func: "generate resmoke tasks" vars: resmoke_args: --storageEngine=wiredTiger -- name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough_gen +- <<: *gen_task_template + name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough_gen tags: ["logical_session_cache"] commands: - func: "generate resmoke tasks" @@ -7307,6 +7292,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-large + - name: build_variant_gen - name: .integration !.audit distros: - rhel80-medium @@ -7345,6 +7331,7 @@ buildvariants: - name: compile_test_and_package_parallel_dbtest_stream_TG distros: - rhel80-xlarge + - name: build_variant_gen - name: .aggregation !.encrypt - name: .auth !.audit !.multiversion - name: .causally_consistent !.wo_snapshot @@ -7410,6 +7397,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-xlarge + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: aggregation_auth @@ -7446,6 +7434,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: .auth !.audit !.multiversion @@ -7518,6 +7507,7 @@ buildvariants: - name: compile_ninja_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: .aggfuzzer .common - name: audit - name: causally_consistent_jscore_txns_passthrough @@ -7587,6 +7577,7 @@ buildvariants: scons_cache_scope: shared tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: aggregation - name: aggregation_wildcard_fuzzer_gen - name: .auth !.audit !.multiversion !.jscore @@ -7636,6 +7627,7 @@ buildvariants: scons_cache_scope: shared tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: free_monitoring - name: jsCore - name: replica_sets_jscore_passthrough @@ -7669,6 +7661,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu2004-large + - name: build_variant_gen - name: .aggfuzzer .common !.multiversion - name: aggregation - name: .auth !.audit !.multiversion @@ -7723,6 +7716,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu2004-large + - name: build_variant_gen - name: .aggfuzzer .common !.multiversion - name: audit - name: causally_consistent_jscore_txns_passthrough @@ -7773,6 +7767,7 @@ buildvariants: scons_cache_scope: shared tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: aggregation - name: aggregation_wildcard_fuzzer_gen - name: .auth !.audit !.multiversion !.jscore @@ -7821,6 +7816,7 @@ buildvariants: scons_cache_scope: shared tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: free_monitoring - name: jsCore - name: replica_sets_jscore_passthrough @@ -7858,6 +7854,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - amazon1-2018-build + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: .auth !.multiversion @@ -7911,6 +7908,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - amazon1-2018-build + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: .auth !.audit !.multiversion @@ -7966,6 +7964,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - amazon2-build + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: audit @@ -8018,6 +8017,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - amazon2-build + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: .auth !.audit !.multiversion @@ -8070,6 +8070,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - amazon2-arm64-large + - name: build_variant_gen - name: .aggfuzzer !.multiversion - name: audit - name: auth_audit_gen @@ -8122,6 +8123,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - amazon2-arm64-large + - name: build_variant_gen - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough @@ -8210,6 +8212,7 @@ buildvariants: - name: compile_build_tools_next_TG distros: - windows-vsCurrent-xlarge + - name: build_variant_gen - name: .aggregation !.auth !.encrypt - name: aggregation_expression_multiversion_fuzzer_gen - name: aggregation_expression_optimization_fuzzer_gen @@ -8288,6 +8291,7 @@ buildvariants: - name: compile_build_tools_next_TG distros: - windows-vsCurrent-xlarge + - name: build_variant_gen - name: burn_in_tests_gen - name: buildscripts_test - name: noPassthrough_gen @@ -8330,6 +8334,7 @@ buildvariants: - name: compile_build_tools_next_TG distros: - windows-vsCurrent-xlarge + - name: build_variant_gen - name: burn_in_tests_gen - name: noPassthrough_gen # Disabling as the following tests are not aware of feature flags. @@ -8373,6 +8378,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-xlarge + - name: build_variant_gen - name: audit - name: auth_audit_gen - name: buildscripts_test @@ -8434,6 +8440,7 @@ buildvariants: - name: compile_and_archive_dist_test_then_package_TG distros: - windows-vsCurrent-xlarge + - name: build_variant_gen - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough @@ -8511,6 +8518,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large + - name: build_variant_gen - name: .aggfuzzer .common - name: audit - name: auth_audit_gen @@ -8561,6 +8569,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large + - name: build_variant_gen - name: .aggfuzzer - name: .aggregation !.auth !.encrypt !.unwind - name: auth_gen @@ -8624,6 +8633,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - windows-vsCurrent-large + - name: build_variant_gen - name: .aggfuzzer - name: .aggregation !.auth !.encrypt !.unwind - name: auth_gen @@ -8708,6 +8718,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG - name: compile_build_tools_next_TG + - name: build_variant_gen - name: .aggregation !.auth !.encrypt !.unwind - name: auth_gen - name: .causally_consistent !.sharding @@ -8757,6 +8768,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG - name: compile_build_tools_next_TG + - name: build_variant_gen - name: aggregation - name: auth_gen - name: causally_consistent_jscore_txns_passthrough @@ -8792,6 +8804,7 @@ buildvariants: - name: compile_ninja_next_TG - name: compile_build_tools_next_TG - name: libdeps_graph_linting_TG + - name: build_variant_gen - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough @@ -8821,6 +8834,7 @@ buildvariants: num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough @@ -8847,6 +8861,7 @@ buildvariants: num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: audit - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough @@ -8960,6 +8975,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-large + - name: build_variant_gen - name: .aggfuzzer - name: .aggregation - name: audit @@ -9073,6 +9089,7 @@ buildvariants: - name: compile_test_and_package_parallel_dbtest_stream_TG distros: - rhel80-xlarge + - name: build_variant_gen - name: lint_pylinters - name: lint_clang_format - name: lint_eslint @@ -9205,6 +9222,7 @@ buildvariants: - name: libdeps_graph_linting_TG distros: - rhel80-large + - name: build_variant_gen - name: burn_in_tests_gen - name: .aggfuzzer - name: .aggregation @@ -9320,6 +9338,7 @@ buildvariants: # - name: compile_test_and_package_parallel_dbtest_stream_TG # distros: # - rhel80-medium + - name: build_variant_gen - name: .aggfuzzer - name: .aggregation !.no_async !.secondary_reads !.sharded !.unwind !aggregation_disabled_optimization - name: .change_streams @@ -9392,6 +9411,7 @@ buildvariants: - name: compile_and_archive_dist_test_then_package_TG distros: - rhel80-large + - name: build_variant_gen - name: auth_gen - name: auth_audit_gen - name: causally_consistent_jscore_txns_passthrough @@ -9464,6 +9484,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-xlarge + - name: build_variant_gen - name: .multiversion_fuzzer - name: .multiversion_passthrough - name: .random_multiversion_ds @@ -9517,6 +9538,7 @@ buildvariants: use_scons_cache: false tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: .aggregation !.unwind - name: audit - name: .auth @@ -9571,6 +9593,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-medium + - name: build_variant_gen - name: .aggregation - name: .auth - name: unittest_shell_hang_analyzer_gen @@ -9626,6 +9649,7 @@ buildvariants: - name: compile_build_tools_next_TG distros: - rhel70 + - name: build_variant_gen - name: .aggfuzzer - name: audit - name: auth_audit_gen @@ -9683,6 +9707,7 @@ buildvariants: - name: compile_and_archive_dist_test_then_package_TG distros: - rhel80-large + - name: build_variant_gen - name: jsCore - name: sharding_gen - name: replica_sets_gen @@ -9715,6 +9740,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-xlarge + - name: build_variant_gen - name: .aggfuzzer - name: audit - name: auth_audit_gen @@ -9769,6 +9795,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel82-arm64-large + - name: build_variant_gen - name: .aggfuzzer !.multiversion - name: audit - name: auth_audit_gen @@ -9826,6 +9853,7 @@ buildvariants: has_packages: false tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: jsCore - name: noPassthroughHotBackups_gen @@ -9845,6 +9873,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel70 + - name: build_variant_gen - name: .aggfuzzer - name: audit - name: auth_audit_gen @@ -9894,6 +9923,7 @@ buildvariants: - name: compile_build_tools_next_TG distros: - ubuntu1804-xlarge + - name: build_variant_gen - name: jsCore - name: .read_write_concern !.write !.aggregation - name: replica_sets_jscore_passthrough @@ -9920,6 +9950,7 @@ buildvariants: - name: compile_and_archive_dist_test_then_package_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: jsCore - name: sharding_gen - name: replica_sets_gen @@ -9947,6 +9978,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel70 + - name: build_variant_gen - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough @@ -9997,6 +10029,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-build + - name: build_variant_gen - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough @@ -10045,6 +10078,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel82-arm64-large + - name: build_variant_gen - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough @@ -10082,6 +10116,7 @@ buildvariants: - name: compile_and_archive_dist_test_then_package_TG distros: - rhel70 + - name: build_variant_gen - name: .ssl - name: jsCore - name: external_auth @@ -10115,6 +10150,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel81-power8-large + - name: build_variant_gen - name: .aggregation .common - name: audit - name: .auth !.multiversion !.jscore @@ -10195,6 +10231,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel72-zseries-build + - name: build_variant_gen - name: .aggregation .common - name: audit - name: .auth !.multiversion !.jscore @@ -10245,6 +10282,7 @@ buildvariants: - name: compile_and_archive_dist_test_then_package_TG distros: - rhel72-zseries-build + - name: build_variant_gen - name: jsCore - name: replica_sets_jscore_passthrough @@ -10268,6 +10306,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel72-zseries-build + - name: build_variant_gen - name: jsCore - name: replica_sets_jscore_passthrough - name: ssl_gen @@ -10330,6 +10369,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - suse12-build + - name: build_variant_gen - name: .aggfuzzer .common - name: audit - name: causally_consistent_jscore_txns_passthrough @@ -10373,6 +10413,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - suse12-build + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: .auth !.audit !.multiversion @@ -10422,6 +10463,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - suse15-build + - name: build_variant_gen - name: .aggfuzzer .common !.multiversion - name: audit - name: causally_consistent_jscore_txns_passthrough @@ -10462,6 +10504,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - suse15-build + - name: build_variant_gen - name: .aggfuzzer .common !.multiversion - name: aggregation - name: .auth !.audit !.multiversion @@ -10515,6 +10558,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - debian92-build + - name: build_variant_gen - name: .aggfuzzer .common - name: audit - name: causally_consistent_jscore_txns_passthrough @@ -10558,6 +10602,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - debian92-build + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: aggregation_auth @@ -10611,6 +10656,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - debian10-build + - name: build_variant_gen - name: .aggfuzzer .common - name: audit - name: causally_consistent_jscore_txns_passthrough @@ -10654,6 +10700,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - debian10-build + - name: build_variant_gen - name: .aggfuzzer .common - name: aggregation - name: aggregation_auth @@ -10715,6 +10762,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel80-xlarge + - name: build_variant_gen - name: .aggfuzzer .common - name: .aggregation !.unwind !.encrypt - name: audit @@ -10779,6 +10827,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG # TODO (SERVER-58125): Re-enable the timeseries fuzzer for EFT + - name: build_variant_gen - name: .aggfuzzer .common !.timeseries - name: aggregation - name: .auth !.multiversion !.audit !.sharding @@ -10818,6 +10867,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - rhel72-zseries-build + - name: build_variant_gen - name: .aggregation .common - name: .auth !.multiversion - name: audit @@ -10866,6 +10916,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG - name: compile_benchmarks + - name: build_variant_gen - name: .aggregation - name: .auth - name: audit @@ -10942,6 +10993,7 @@ buildvariants: tasks: - name: compile_and_archive_dist_test_then_package_TG - name: compile_benchmarks + - name: build_variant_gen - name: .aggregation # - name: .auth - name: audit @@ -11012,6 +11064,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG - name: compile_benchmarks + - name: build_variant_gen - name: .aggregation - name: .auth - name: audit @@ -11073,6 +11126,7 @@ buildvariants: test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling tasks: - name: compile_test_and_package_serial_TG + - name: build_variant_gen - name: .aggfuzzer .common - name: free_monitoring - name: .jstestfuzz !.initsync @@ -11100,6 +11154,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG - name: compile_benchmarks + - name: build_variant_gen - name: .aggregation - name: .auth - name: audit @@ -11171,6 +11226,7 @@ buildvariants: tasks: - name: compile_and_archive_dist_test_then_package_TG - name: compile_benchmarks + - name: build_variant_gen - name: .aggregation - name: .auth - name: audit @@ -11237,6 +11293,7 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG - name: compile_benchmarks + - name: build_variant_gen - name: .aggregation - name: .auth - name: audit @@ -11310,6 +11367,7 @@ buildvariants: - name: compile_test_and_package_parallel_dbtest_stream_TG distros: - ubuntu1804-xlarge + - name: build_variant_gen - name: jsCore - name: jsCore_txns - name: unittest_shell_hang_analyzer_gen @@ -11340,6 +11398,7 @@ buildvariants: - name: compile_test_and_package_parallel_core_stream_TG distros: - ubuntu1804-xlarge + - name: build_variant_gen - name: jsCore - name: jsCore_txns # Disabling these tests as they are not aware of feature flags. @@ -11451,6 +11510,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: .aggregation !.no_async - name: .sharding .auth - name: .sharding .causally_consistent !.wo_snapshot @@ -11484,6 +11544,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: .aggregation !.no_async - name: .sharding .auth - name: .sharding .causally_consistent !.wo_snapshot @@ -11513,6 +11574,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: .aggregation !.no_async - name: .sharding .auth - name: .sharding .causally_consistent !.wo_snapshot @@ -11543,6 +11605,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: .aggregation !.no_async - name: .sharding .auth - name: .sharding .causally_consistent !.wo_snapshot @@ -11573,6 +11636,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: .aggregation !.no_async - name: .sharding .auth - name: .sharding .causally_consistent !.wo_snapshot @@ -11603,6 +11667,7 @@ buildvariants: - name: compile_test_and_package_serial_TG distros: - ubuntu1804-build + - name: build_variant_gen - name: .aggregation !.no_async - name: .sharding .auth - name: .sharding .causally_consistent !.wo_snapshot @@ -11681,6 +11746,7 @@ buildvariants: test_flags: --excludeWithAnyTags=requires_fast_memory,live_record_incompatible tasks: - name: compile_and_archive_dist_test_TG + - name: build_variant_gen - name: .aggfuzzer - name: .aggregation # - name: audit diff --git a/etc/pip/components/evergreen.req b/etc/pip/components/evergreen.req index e4dda2bd320..d8d602a16f1 100644 --- a/etc/pip/components/evergreen.req +++ b/etc/pip/components/evergreen.req @@ -3,5 +3,5 @@ dataclasses; python_version < "3.7" inject ~= 4.3.1 GitPython ~= 3.1.7 psutil -pydantic ~= 1.7.3 +pydantic ~= 1.8.2 structlog ~= 19.2.0 diff --git a/etc/pip/components/resmoke.req b/etc/pip/components/resmoke.req index d775049d0f4..5187552d080 100644 --- a/etc/pip/components/resmoke.req +++ b/etc/pip/components/resmoke.req @@ -1,10 +1,10 @@ curatorbin == 1.2.1 PyKMIP == 0.4.0 # It's now 0.8.0. We're far enough back to have API conflicts. -evergreen.py == 2.1.0 +evergreen.py == 3.1.0 jinja2 MarkupSafe == 1.1.0 # See SERVER-57036, this is a transitive dependency of jinja2 mock -shrub.py == 1.1.0 +shrub.py == 1.1.4 ocspresponder == 0.5.0 flask == 1.1.1 ocspbuilder == 0.10.2 diff --git a/evergreen/fuzzer_tasks_generate.sh b/evergreen/gen_tasks_activate.sh index 97f1b7024bd..4dee68caee3 100644 --- a/evergreen/fuzzer_tasks_generate.sh +++ b/evergreen/gen_tasks_activate.sh @@ -4,7 +4,6 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" cd src set -o errexit -set -o verbose activate_venv -$python buildscripts/evergreen_gen_fuzzer_tests.py --expansion-file ../expansions.yml +$python buildscripts/evergreen_activate_gen_tasks.py --expansion-file ../expansions.yml --verbose diff --git a/evergreen/generate_build_variant.sh b/evergreen/generate_build_variant.sh new file mode 100644 index 00000000000..3c11c8c1a8c --- /dev/null +++ b/evergreen/generate_build_variant.sh @@ -0,0 +1,13 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit + +activate_venv +PATH=$PATH:$HOME $python buildscripts/evergreen_gen_build_variant.py \ + --expansion-file ../expansions.yml \ + --evg-api-config ./.evergreen.yml \ + --output-file ${build_variant}.json \ + --verbose diff --git a/evergreen/implicit_multiversions_tasks_generate.sh b/evergreen/implicit_multiversions_tasks_generate.sh index 11d3426f576..038951e82ac 100644 --- a/evergreen/implicit_multiversions_tasks_generate.sh +++ b/evergreen/implicit_multiversions_tasks_generate.sh @@ -7,5 +7,7 @@ set -o errexit activate_venv PATH="$PATH:/data/multiversion" -$python buildscripts/evergreen_gen_multiversion_tests.py run --expansion-file ../expansions.yml -$python buildscripts/evergreen_gen_multiversion_tests.py generate-exclude-tags + +if [ -n "${require_multiversion}" ]; then + $python buildscripts/evergreen_gen_multiversion_tests.py generate-exclude-tags +fi diff --git a/evergreen/resmoke_tasks_generate.sh b/evergreen/resmoke_tasks_generate.sh deleted file mode 100644 index c4c7f64b0ad..00000000000 --- a/evergreen/resmoke_tasks_generate.sh +++ /dev/null @@ -1,14 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -if [ -n "$GENERATE_BUILD_VARIANTS" ]; then - echo "Skipping generation since 'generate_build_variants' is set." - exit 0 -fi - -cd src - -set -o errexit - -activate_venv -$python buildscripts/evergreen_generate_resmoke_tasks.py --expansion-file ../expansions.yml --verbose |