From 579607dde38104436580a8c53aeda8b17a2acdbb Mon Sep 17 00:00:00 2001 From: David Bradford Date: Tue, 5 Apr 2022 13:07:10 +0000 Subject: SERVER-65179: Forward task definition dependencies to generated tasks --- buildscripts/evergreen_gen_build_variant.py | 37 ++++++++++++++-------- buildscripts/selected_tests.py | 2 ++ .../task_generation/task_types/fuzzer_tasks.py | 7 +++- .../task_generation/task_types/resmoke_tasks.py | 27 +++++++++------- .../task_types/test_fuzzer_tasks.py | 3 +- .../task_types/test_resmoke_tasks.py | 7 ++-- .../tests/task_generation/test_gen_task_service.py | 7 ++-- etc/evergreen_yml_components/definitions.yml | 4 +-- 8 files changed, 60 insertions(+), 34 deletions(-) diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py index 0ff8c7996da..ee140d73b9d 100644 --- a/buildscripts/evergreen_gen_build_variant.py +++ b/buildscripts/evergreen_gen_build_variant.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 """Generate configuration for a build variant.""" +import hashlib +import os from concurrent.futures import ThreadPoolExecutor as Executor from datetime import datetime, timedelta from time import perf_counter -from typing import Optional, Any, Set, Tuple -import hashlib -import os +from typing import Any, Optional, Set, Tuple import click import inject @@ -14,20 +14,25 @@ from pydantic import BaseModel from evergreen import EvergreenApi, RetryingEvergreenApi from evergreen import Task as EvgTask -from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, parse_evergreen_file, Task, \ - Variant -from buildscripts.task_generation.constants import MAX_WORKERS, LOOKBACK_DURATION_DAYS, MAX_TASK_PRIORITY, \ - GENERATED_CONFIG_DIR, GEN_PARENT_TASK, EXPANSION_RE +from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, Task, Variant, + parse_evergreen_file) +from buildscripts.task_generation.constants import (EXPANSION_RE, GEN_PARENT_TASK, + GENERATED_CONFIG_DIR, LOOKBACK_DURATION_DAYS, + MAX_TASK_PRIORITY, MAX_WORKERS) from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder from buildscripts.task_generation.gen_config import GenerationConfiguration -from buildscripts.task_generation.gen_task_validation import GenTaskValidationService +from buildscripts.task_generation.gen_task_validation import \ + GenTaskValidationService from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService -from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters -from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \ - greedy_division, round_robin_fallback -from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskParams -from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions -from buildscripts.task_generation.task_types.resmoke_tasks import ResmokeGenTaskParams +from buildscripts.task_generation.suite_split import (SuiteSplitConfig, SuiteSplitParameters) +from buildscripts.task_generation.suite_split_strategies import ( + FallbackStrategy, SplitStrategy, greedy_division, round_robin_fallback) +from buildscripts.task_generation.task_types.fuzzer_tasks import \ + FuzzerGenTaskParams +from buildscripts.task_generation.task_types.gentask_options import \ + GenTaskOptions +from buildscripts.task_generation.task_types.resmoke_tasks import \ + ResmokeGenTaskParams from buildscripts.util.cmdutils import enable_logging from buildscripts.util.fileops import read_yaml_file from buildscripts.util.taskname import remove_gen_suffix @@ -235,6 +240,8 @@ class GenerateBuildVariantOrchestrator: resmoke_jobs_max=run_vars.get("resmoke_jobs_max"), large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), config_location=self.evg_expansions.config_location(), + dependencies={depends_on["name"] + for depends_on in task_def.depends_on}, ) def task_def_to_fuzzer_params(self, task_def: Task, build_variant: str) -> FuzzerGenTaskParams: @@ -268,6 +275,8 @@ class GenerateBuildVariantOrchestrator: use_large_distro=run_vars.get("use_large_distro", False), large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), config_location=self.evg_expansions.config_location(), + dependencies={depends_on["name"] + for depends_on in task_def.depends_on}, ) def generate(self, task_id: str, build_variant_name: str, output_file: str) -> None: diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py index 5ffbe1284ab..f39666a0c37 100644 --- a/buildscripts/selected_tests.py +++ b/buildscripts/selected_tests.py @@ -417,6 +417,8 @@ class SelectedTestsOrchestrator: resmoke_args=task_config.get("resmoke_args", ""), resmoke_jobs_max=task_config.get("resmoke_jobs_max"), config_location=self.evg_expansions.get_config_location(), + dependencies={depends_on["name"] + for depends_on in task_def.depends_on}, ) builder.generate_suite(split_params, gen_params) diff --git a/buildscripts/task_generation/task_types/fuzzer_tasks.py b/buildscripts/task_generation/task_types/fuzzer_tasks.py index 039dee97d67..f6665fc13ca 100644 --- a/buildscripts/task_generation/task_types/fuzzer_tasks.py +++ b/buildscripts/task_generation/task_types/fuzzer_tasks.py @@ -41,6 +41,8 @@ class FuzzerGenTaskParams(NamedTuple): timeout_secs: Timeout before test execution is considered hung. require_multiversion_setup: Requires downloading Multiversion binaries. use_large_distro: Should tests be generated on a large distro. + config_location: S3 path to the generated config tarball. None if no generated config files. + dependencies: Set of dependencies generated tasks should depend on. """ task_name: str @@ -59,6 +61,7 @@ class FuzzerGenTaskParams(NamedTuple): use_large_distro: Optional[bool] large_distro_name: Optional[str] config_location: str + dependencies: Set[str] def jstestfuzz_params(self) -> Dict[str, str]: """Build a dictionary of parameters to pass to jstestfuzz.""" @@ -141,4 +144,6 @@ class FuzzerGenTaskService: FunctionCall(RUN_GENERATED_TESTS, run_tests_vars) ] - return Task(sub_task_name, commands, {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)}) + dependencies = {TaskDependency(dependency) for dependency in params.dependencies} + + return Task(sub_task_name, commands, dependencies) diff --git a/buildscripts/task_generation/task_types/resmoke_tasks.py b/buildscripts/task_generation/task_types/resmoke_tasks.py index 5ddd4f9a541..a864604712a 100644 --- a/buildscripts/task_generation/task_types/resmoke_tasks.py +++ b/buildscripts/task_generation/task_types/resmoke_tasks.py @@ -1,19 +1,21 @@ """Task generation for split resmoke tasks.""" -from typing import Set, Any, Dict, NamedTuple, Optional, List +from typing import Any, Dict, List, NamedTuple, Optional, Set import inject import structlog -from shrub.v2 import Task, TaskDependency, FunctionCall +from shrub.v2 import FunctionCall, Task, TaskDependency from buildscripts.ciconfig.evergreen import EvergreenProjectConfig from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG -from buildscripts.task_generation.constants import ARCHIVE_DIST_TEST_DEBUG_TASK, EXCLUDES_TAGS_FILE_PATH, \ - BACKPORT_REQUIRED_TAG, CONFIGURE_EVG_CREDENTIALS, RUN_GENERATED_TESTS +from buildscripts.task_generation.constants import ( + BACKPORT_REQUIRED_TAG, CONFIGURE_EVG_CREDENTIALS, EXCLUDES_TAGS_FILE_PATH, RUN_GENERATED_TESTS) from buildscripts.task_generation.suite_split import GeneratedSuite -from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions -from buildscripts.task_generation.task_types.models.resmoke_task_model import ResmokeTask -from buildscripts.task_generation.task_types.multiversion_decorator import MultiversionGenTaskDecorator, \ - MultiversionDecoratorParams +from buildscripts.task_generation.task_types.gentask_options import \ + GenTaskOptions +from buildscripts.task_generation.task_types.models.resmoke_task_model import \ + ResmokeTask +from buildscripts.task_generation.task_types.multiversion_decorator import ( + MultiversionDecoratorParams, MultiversionGenTaskDecorator) from buildscripts.timeouts.timeout import TimeoutEstimate LOGGER = structlog.getLogger(__name__) @@ -44,6 +46,7 @@ class ResmokeGenTaskParams(NamedTuple): resmoke_jobs_max: Max number of jobs that resmoke should execute in parallel. depends_on: List of tasks this task depends on. config_location: S3 path to the generated config tarball. None if no generated config files. + dependencies: Set of dependencies generated tasks should depend on. """ use_large_distro: bool @@ -54,6 +57,7 @@ class ResmokeGenTaskParams(NamedTuple): resmoke_args: str resmoke_jobs_max: Optional[int] config_location: str + dependencies: Set[str] class ResmokeGenTaskService: @@ -157,7 +161,8 @@ class ResmokeGenTaskService: FunctionCall(RUN_GENERATED_TESTS, run_tests_vars), ] - shrub_task = Task(sub_task_name, [cmd for cmd in commands if cmd], self._get_dependencies()) + shrub_task = Task(sub_task_name, [cmd for cmd in commands if cmd], + self._get_dependencies(params)) return ResmokeTask(shrub_task=shrub_task, resmoke_suite_name=suite.suite_name, execution_task_suite_yaml_path=sub_suite_file_path, execution_task_suite_yaml_name=sub_suite_file, test_list=sub_suite_roots, @@ -228,7 +233,7 @@ class ResmokeGenTaskService: return variables @staticmethod - def _get_dependencies() -> Set[TaskDependency]: + def _get_dependencies(params: ResmokeGenTaskParams) -> Set[TaskDependency]: """Get the set of dependency tasks for these suites.""" - dependencies = {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)} + dependencies = {TaskDependency(dependency) for dependency in params.dependencies} return dependencies diff --git a/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py b/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py index 7d1ce8bf119..0b81fbdc295 100644 --- a/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py +++ b/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py @@ -5,9 +5,9 @@ import unittest import inject import buildscripts.task_generation.task_types.fuzzer_tasks as under_test +from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService def build_mock_fuzzer_params(jstestfuzz_vars="vars for jstestfuzz", npm_command="jstestfuzz"): @@ -28,6 +28,7 @@ def build_mock_fuzzer_params(jstestfuzz_vars="vars for jstestfuzz", npm_command= use_large_distro=None, large_distro_name="large distro", config_location="config_location", + dependencies={"archive_dist_test_debug"}, ) diff --git a/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py b/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py index 39c9e85923a..564df7d9121 100644 --- a/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py +++ b/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py @@ -2,13 +2,13 @@ import unittest import inject -from mock import MagicMock import buildscripts.task_generation.task_types.resmoke_tasks as under_test -from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, Variant +from buildscripts.ciconfig.evergreen import EvergreenProjectConfig from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService from buildscripts.task_generation.suite_split import GeneratedSuite, SubSuite -from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions +from buildscripts.task_generation.task_types.gentask_options import \ + GenTaskOptions from buildscripts.util.teststats import TestRuntime # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access @@ -47,6 +47,7 @@ def build_mock_gen_params(repeat_suites=1, resmoke_args="resmoke args"): resmoke_jobs_max=None, large_distro_name=None, config_location="generated_config", + dependencies={"archive_dist_test_debug"}, ) diff --git a/buildscripts/tests/task_generation/test_gen_task_service.py b/buildscripts/tests/task_generation/test_gen_task_service.py index caf6027736e..831efed4d72 100644 --- a/buildscripts/tests/task_generation/test_gen_task_service.py +++ b/buildscripts/tests/task_generation/test_gen_task_service.py @@ -6,9 +6,11 @@ from unittest.mock import MagicMock import inject from shrub.v2 import BuildVariant -import buildscripts.task_generation.gen_task_service as under_test from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService -from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskService +from buildscripts.task_generation.task_types.fuzzer_tasks import \ + FuzzerGenTaskService + +import buildscripts.task_generation.gen_task_service as under_test # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access @@ -31,6 +33,7 @@ def build_mock_fuzzer_params(multi_version=False, use_large_distro=None, large_d use_large_distro=use_large_distro, large_distro_name=large_distro_name, config_location="config location", + dependencies={"archive_dist_test_debug"}, ) diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index c2e26d315d2..31f36651e0f 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -89,7 +89,7 @@ variables: name: gen_task_template depends_on: - name: build_variant_gen - - name: archive_dist_test + - name: archive_dist_test_debug commands: - func: "generate resmoke tasks" vars: @@ -154,7 +154,7 @@ variables: exec_timeout_secs: 14400 # Time out the task if it runs for more than 4 hours. depends_on: - build_variant_gen - - archive_dist_test + - archive_dist_test_debug commands: - func: "generate resmoke tasks" -- cgit v1.2.1