SERVER-65179: Forward task definition dependencies to generated tasks

8 files changed, 97 insertions, 34 deletions

diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py
index 0ff8c7996da..86f8c046c5b 100644
--- a/buildscripts/evergreen_gen_build_variant.py
+++ b/buildscripts/evergreen_gen_build_variant.py
@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 """Generate configuration for a build variant."""
+import hashlib
+import os
 from concurrent.futures import ThreadPoolExecutor as Executor
 from datetime import datetime, timedelta
 from time import perf_counter
-from typing import Optional, Any, Set, Tuple
-import hashlib
-import os
+from typing import Any, Optional, Set, Tuple
 
 import click
 import inject
@@ -14,20 +14,25 @@ from pydantic import BaseModel
 from evergreen import EvergreenApi, RetryingEvergreenApi
 from evergreen import Task as EvgTask
 
-from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, parse_evergreen_file, Task, \
-    Variant
-from buildscripts.task_generation.constants import MAX_WORKERS, LOOKBACK_DURATION_DAYS, MAX_TASK_PRIORITY, \
-    GENERATED_CONFIG_DIR, GEN_PARENT_TASK, EXPANSION_RE
+from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, Task, Variant,
+                                             parse_evergreen_file)
+from buildscripts.task_generation.constants import (EXPANSION_RE, GEN_PARENT_TASK,
+                                                    GENERATED_CONFIG_DIR, LOOKBACK_DURATION_DAYS,
+                                                    MAX_TASK_PRIORITY, MAX_WORKERS)
 from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder
 from buildscripts.task_generation.gen_config import GenerationConfiguration
-from buildscripts.task_generation.gen_task_validation import GenTaskValidationService
+from buildscripts.task_generation.gen_task_validation import \
+    GenTaskValidationService
 from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
-from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters
-from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \
-    greedy_division, round_robin_fallback
-from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskParams
-from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions
-from buildscripts.task_generation.task_types.resmoke_tasks import ResmokeGenTaskParams
+from buildscripts.task_generation.suite_split import (SuiteSplitConfig, SuiteSplitParameters)
+from buildscripts.task_generation.suite_split_strategies import (
+    FallbackStrategy, SplitStrategy, greedy_division, round_robin_fallback)
+from buildscripts.task_generation.task_types.fuzzer_tasks import \
+    FuzzerGenTaskParams
+from buildscripts.task_generation.task_types.gentask_options import \
+    GenTaskOptions
+from buildscripts.task_generation.task_types.resmoke_tasks import \
+    ResmokeGenTaskParams
 from buildscripts.util.cmdutils import enable_logging
 from buildscripts.util.fileops import read_yaml_file
 from buildscripts.util.taskname import remove_gen_suffix
@@ -235,8 +240,26 @@ class GenerateBuildVariantOrchestrator:
             resmoke_jobs_max=run_vars.get("resmoke_jobs_max"),
             large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"),
             config_location=self.evg_expansions.config_location(),
+            dependencies=self.determine_task_dependencies(task_def),
         )
 
+    def determine_task_dependencies(self, task_def: Task) -> Set[str]:
+        """
+        Determine the dependencies to use for tasks generated from the given task definition.
+
+        This should include all tasks that the task definition depends on except for the currently
+        running task.
+
+        :param task_def: Task definition to use.
+        :return: Set of dependencies to generate.
+        """
+        dependency_set = {
+            task["name"]
+            for task in task_def.depends_on if task["name"] != self.evg_expansions.task_name
+        }
+
+        return dependency_set
+
     def task_def_to_fuzzer_params(self, task_def: Task, build_variant: str) -> FuzzerGenTaskParams:
         """
         Build parameters for how a fuzzer task should be generated based on its task definition.
@@ -268,6 +291,7 @@ class GenerateBuildVariantOrchestrator:
             use_large_distro=run_vars.get("use_large_distro", False),
             large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"),
             config_location=self.evg_expansions.config_location(),
+            dependencies=self.determine_task_dependencies(task_def),
         )
 
     def generate(self, task_id: str, build_variant_name: str, output_file: str) -> None:
diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py
index 5ffbe1284ab..f39666a0c37 100644
--- a/buildscripts/selected_tests.py
+++ b/buildscripts/selected_tests.py
@@ -417,6 +417,8 @@ class SelectedTestsOrchestrator:
                 resmoke_args=task_config.get("resmoke_args", ""),
                 resmoke_jobs_max=task_config.get("resmoke_jobs_max"),
                 config_location=self.evg_expansions.get_config_location(),
+                dependencies={depends_on["name"]
+                              for depends_on in task_def.depends_on},
             )
             builder.generate_suite(split_params, gen_params)
 
diff --git a/buildscripts/task_generation/task_types/fuzzer_tasks.py b/buildscripts/task_generation/task_types/fuzzer_tasks.py
index 039dee97d67..d8d85dab408 100644
--- a/buildscripts/task_generation/task_types/fuzzer_tasks.py
+++ b/buildscripts/task_generation/task_types/fuzzer_tasks.py
@@ -4,7 +4,7 @@ from typing import NamedTuple, Set, Optional, Dict
 from shrub.v2 import Task, FunctionCall, TaskDependency
 
 from buildscripts.patch_builds.task_generation import TimeoutInfo
-from buildscripts.task_generation.constants import ARCHIVE_DIST_TEST_DEBUG_TASK, CONFIGURE_EVG_CREDENTIALS, \
+from buildscripts.task_generation.constants import CONFIGURE_EVG_CREDENTIALS, \
     RUN_GENERATED_TESTS
 from buildscripts.task_generation.task_types.multiversion_decorator import MultiversionGenTaskDecorator, \
     MultiversionDecoratorParams
@@ -41,6 +41,8 @@ class FuzzerGenTaskParams(NamedTuple):
     timeout_secs: Timeout before test execution is considered hung.
     require_multiversion_setup: Requires downloading Multiversion binaries.
     use_large_distro: Should tests be generated on a large distro.
+    config_location: S3 path to the generated config tarball. None if no generated config files.
+    dependencies: Set of dependencies generated tasks should depend on.
     """
 
     task_name: str
@@ -59,6 +61,7 @@ class FuzzerGenTaskParams(NamedTuple):
     use_large_distro: Optional[bool]
     large_distro_name: Optional[str]
     config_location: str
+    dependencies: Set[str]
 
     def jstestfuzz_params(self) -> Dict[str, str]:
         """Build a dictionary of parameters to pass to jstestfuzz."""
@@ -141,4 +144,6 @@ class FuzzerGenTaskService:
             FunctionCall(RUN_GENERATED_TESTS, run_tests_vars)
         ]
 
-        return Task(sub_task_name, commands, {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)})
+        dependencies = {TaskDependency(dependency) for dependency in params.dependencies}
+
+        return Task(sub_task_name, commands, dependencies)
diff --git a/buildscripts/task_generation/task_types/resmoke_tasks.py b/buildscripts/task_generation/task_types/resmoke_tasks.py
index 5ddd4f9a541..a864604712a 100644
--- a/buildscripts/task_generation/task_types/resmoke_tasks.py
+++ b/buildscripts/task_generation/task_types/resmoke_tasks.py
@@ -1,19 +1,21 @@
 """Task generation for split resmoke tasks."""
-from typing import Set, Any, Dict, NamedTuple, Optional, List
+from typing import Any, Dict, List, NamedTuple, Optional, Set
 
 import inject
 import structlog
-from shrub.v2 import Task, TaskDependency, FunctionCall
+from shrub.v2 import FunctionCall, Task, TaskDependency
 
 from buildscripts.ciconfig.evergreen import EvergreenProjectConfig
 from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG
-from buildscripts.task_generation.constants import ARCHIVE_DIST_TEST_DEBUG_TASK, EXCLUDES_TAGS_FILE_PATH, \
-    BACKPORT_REQUIRED_TAG, CONFIGURE_EVG_CREDENTIALS, RUN_GENERATED_TESTS
+from buildscripts.task_generation.constants import (
+    BACKPORT_REQUIRED_TAG, CONFIGURE_EVG_CREDENTIALS, EXCLUDES_TAGS_FILE_PATH, RUN_GENERATED_TESTS)
 from buildscripts.task_generation.suite_split import GeneratedSuite
-from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions
-from buildscripts.task_generation.task_types.models.resmoke_task_model import ResmokeTask
-from buildscripts.task_generation.task_types.multiversion_decorator import MultiversionGenTaskDecorator, \
-    MultiversionDecoratorParams
+from buildscripts.task_generation.task_types.gentask_options import \
+    GenTaskOptions
+from buildscripts.task_generation.task_types.models.resmoke_task_model import \
+    ResmokeTask
+from buildscripts.task_generation.task_types.multiversion_decorator import (
+    MultiversionDecoratorParams, MultiversionGenTaskDecorator)
 from buildscripts.timeouts.timeout import TimeoutEstimate
 
 LOGGER = structlog.getLogger(__name__)
@@ -44,6 +46,7 @@ class ResmokeGenTaskParams(NamedTuple):
     resmoke_jobs_max: Max number of jobs that resmoke should execute in parallel.
     depends_on: List of tasks this task depends on.
     config_location: S3 path to the generated config tarball. None if no generated config files.
+    dependencies: Set of dependencies generated tasks should depend on.
     """
 
     use_large_distro: bool
@@ -54,6 +57,7 @@ class ResmokeGenTaskParams(NamedTuple):
     resmoke_args: str
     resmoke_jobs_max: Optional[int]
     config_location: str
+    dependencies: Set[str]
 
 
 class ResmokeGenTaskService:
@@ -157,7 +161,8 @@ class ResmokeGenTaskService:
             FunctionCall(RUN_GENERATED_TESTS, run_tests_vars),
         ]
 
-        shrub_task = Task(sub_task_name, [cmd for cmd in commands if cmd], self._get_dependencies())
+        shrub_task = Task(sub_task_name, [cmd for cmd in commands if cmd],
+                          self._get_dependencies(params))
         return ResmokeTask(shrub_task=shrub_task, resmoke_suite_name=suite.suite_name,
                            execution_task_suite_yaml_path=sub_suite_file_path,
                            execution_task_suite_yaml_name=sub_suite_file, test_list=sub_suite_roots,
@@ -228,7 +233,7 @@ class ResmokeGenTaskService:
         return variables
 
     @staticmethod
-    def _get_dependencies() -> Set[TaskDependency]:
+    def _get_dependencies(params: ResmokeGenTaskParams) -> Set[TaskDependency]:
         """Get the set of dependency tasks for these suites."""
-        dependencies = {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)}
+        dependencies = {TaskDependency(dependency) for dependency in params.dependencies}
         return dependencies
diff --git a/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py b/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py
index 7d1ce8bf119..133bdd3ca15 100644
--- a/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py
+++ b/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py
@@ -5,9 +5,9 @@ import unittest
 import inject
 
 import buildscripts.task_generation.task_types.fuzzer_tasks as under_test
+from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
-from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
 
 
 def build_mock_fuzzer_params(jstestfuzz_vars="vars for jstestfuzz", npm_command="jstestfuzz"):
@@ -28,6 +28,7 @@ def build_mock_fuzzer_params(jstestfuzz_vars="vars for jstestfuzz", npm_command=
         use_large_distro=None,
         large_distro_name="large distro",
         config_location="config_location",
+        dependencies={"task_dependency"},
     )
 
 
diff --git a/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py b/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py
index 39c9e85923a..07f99ed0ccc 100644
--- a/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py
+++ b/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py
@@ -2,13 +2,13 @@
 import unittest
 
 import inject
-from mock import MagicMock
 
 import buildscripts.task_generation.task_types.resmoke_tasks as under_test
-from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, Variant
+from buildscripts.ciconfig.evergreen import EvergreenProjectConfig
 from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
 from buildscripts.task_generation.suite_split import GeneratedSuite, SubSuite
-from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions
+from buildscripts.task_generation.task_types.gentask_options import \
+    GenTaskOptions
 from buildscripts.util.teststats import TestRuntime
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
@@ -47,6 +47,7 @@ def build_mock_gen_params(repeat_suites=1, resmoke_args="resmoke args"):
         resmoke_jobs_max=None,
         large_distro_name=None,
         config_location="generated_config",
+        dependencies={"task_dependency"},
     )
 
 
diff --git a/buildscripts/tests/task_generation/test_gen_task_service.py b/buildscripts/tests/task_generation/test_gen_task_service.py
index caf6027736e..38250444986 100644
--- a/buildscripts/tests/task_generation/test_gen_task_service.py
+++ b/buildscripts/tests/task_generation/test_gen_task_service.py
@@ -6,9 +6,11 @@ from unittest.mock import MagicMock
 import inject
 from shrub.v2 import BuildVariant
 
-import buildscripts.task_generation.gen_task_service as under_test
 from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
-from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskService
+from buildscripts.task_generation.task_types.fuzzer_tasks import \
+    FuzzerGenTaskService
+
+import buildscripts.task_generation.gen_task_service as under_test
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
 
@@ -31,6 +33,7 @@ def build_mock_fuzzer_params(multi_version=False, use_large_distro=None, large_d
         use_large_distro=use_large_distro,
         large_distro_name=large_distro_name,
         config_location="config location",
+        dependencies={"task_dependency"},
     )
 
 
diff --git a/buildscripts/tests/test_evergreen_gen_build_variant.py b/buildscripts/tests/test_evergreen_gen_build_variant.py
index b5a6ca09998..7ece404ab36 100644
--- a/buildscripts/tests/test_evergreen_gen_build_variant.py
+++ b/buildscripts/tests/test_evergreen_gen_build_variant.py
@@ -25,7 +25,7 @@ def build_mock_build_variant(expansions=None, task_list=None):
     return Variant(config, task_map, {})
 
 
-def build_mock_task(name, run_vars=None):
+def build_mock_task(name, run_vars=None, depends_on=None):
     config = {
         "name":
             name, "commands": [
@@ -36,6 +36,9 @@ def build_mock_task(name, run_vars=None):
                 },
             ]
     }
+
+    if depends_on is not None:
+        config["depends_on"] = depends_on
     return Task(config)
 
 
@@ -54,6 +57,7 @@ def build_mock_expansions():
     mock_expansions = MagicMock()
     mock_expansions.config_location.return_value = "/path/to/config"
     mock_expansions.get_max_sub_suites.return_value = 998
+    mock_expansions.task_name = "generating_task"
     return mock_expansions
 
 
@@ -194,6 +198,24 @@ class TestTaskDefToSplitParams(unittest.TestCase):
         self.assertEqual("the suite", split_param.filename)
 
 
+class TestDetermineTaskDependencies(unittest.TestCase):
+    def test_running_task_should_not_be_included_in_depends_on(self):
+        run_vars = {
+            "resmoke_args": "run tests",
+        }
+        depends_on = [
+            {"name": "compile"},
+            {"name": "build_variant_gen"},
+        ]
+        mock_task_def = build_mock_task("my_task", run_vars, depends_on=depends_on)
+        mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def])
+
+        dependencies = mock_orchestrator.determine_task_dependencies(mock_task_def)
+
+        self.assertIn("compile", dependencies)
+        self.assertNotIn("generating_task", dependencies)
+
+
 class TestTaskDefToGenParams(unittest.TestCase):
     def test_params_should_be_generated(self):
         run_vars = {