SERVER-65179: Forward task definition dependencies to generated tasks

8 files changed, 60 insertions, 34 deletions

diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py
index 0ff8c7996da..ee140d73b9d 100644
--- a/buildscripts/evergreen_gen_build_variant.py
+++ b/buildscripts/evergreen_gen_build_variant.py
@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 """Generate configuration for a build variant."""
+import hashlib
+import os
 from concurrent.futures import ThreadPoolExecutor as Executor
 from datetime import datetime, timedelta
 from time import perf_counter
-from typing import Optional, Any, Set, Tuple
-import hashlib
-import os
+from typing import Any, Optional, Set, Tuple
 
 import click
 import inject
@@ -14,20 +14,25 @@ from pydantic import BaseModel
 from evergreen import EvergreenApi, RetryingEvergreenApi
 from evergreen import Task as EvgTask
 
-from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, parse_evergreen_file, Task, \
-    Variant
-from buildscripts.task_generation.constants import MAX_WORKERS, LOOKBACK_DURATION_DAYS, MAX_TASK_PRIORITY, \
-    GENERATED_CONFIG_DIR, GEN_PARENT_TASK, EXPANSION_RE
+from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, Task, Variant,
+                                             parse_evergreen_file)
+from buildscripts.task_generation.constants import (EXPANSION_RE, GEN_PARENT_TASK,
+                                                    GENERATED_CONFIG_DIR, LOOKBACK_DURATION_DAYS,
+                                                    MAX_TASK_PRIORITY, MAX_WORKERS)
 from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder
 from buildscripts.task_generation.gen_config import GenerationConfiguration
-from buildscripts.task_generation.gen_task_validation import GenTaskValidationService
+from buildscripts.task_generation.gen_task_validation import \
+    GenTaskValidationService
 from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
-from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters
-from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \
-    greedy_division, round_robin_fallback
-from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskParams
-from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions
-from buildscripts.task_generation.task_types.resmoke_tasks import ResmokeGenTaskParams
+from buildscripts.task_generation.suite_split import (SuiteSplitConfig, SuiteSplitParameters)
+from buildscripts.task_generation.suite_split_strategies import (
+    FallbackStrategy, SplitStrategy, greedy_division, round_robin_fallback)
+from buildscripts.task_generation.task_types.fuzzer_tasks import \
+    FuzzerGenTaskParams
+from buildscripts.task_generation.task_types.gentask_options import \
+    GenTaskOptions
+from buildscripts.task_generation.task_types.resmoke_tasks import \
+    ResmokeGenTaskParams
 from buildscripts.util.cmdutils import enable_logging
 from buildscripts.util.fileops import read_yaml_file
 from buildscripts.util.taskname import remove_gen_suffix
@@ -235,6 +240,8 @@ class GenerateBuildVariantOrchestrator:
             resmoke_jobs_max=run_vars.get("resmoke_jobs_max"),
             large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"),
             config_location=self.evg_expansions.config_location(),
+            dependencies={depends_on["name"]
+                          for depends_on in task_def.depends_on},
         )
 
     def task_def_to_fuzzer_params(self, task_def: Task, build_variant: str) -> FuzzerGenTaskParams:
@@ -268,6 +275,8 @@ class GenerateBuildVariantOrchestrator:
             use_large_distro=run_vars.get("use_large_distro", False),
             large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"),
             config_location=self.evg_expansions.config_location(),
+            dependencies={depends_on["name"]
+                          for depends_on in task_def.depends_on},
         )
 
     def generate(self, task_id: str, build_variant_name: str, output_file: str) -> None:
diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py
index 5ffbe1284ab..f39666a0c37 100644
--- a/buildscripts/selected_tests.py
+++ b/buildscripts/selected_tests.py
@@ -417,6 +417,8 @@ class SelectedTestsOrchestrator:
                 resmoke_args=task_config.get("resmoke_args", ""),
                 resmoke_jobs_max=task_config.get("resmoke_jobs_max"),
                 config_location=self.evg_expansions.get_config_location(),
+                dependencies={depends_on["name"]
+                              for depends_on in task_def.depends_on},
             )
             builder.generate_suite(split_params, gen_params)
 
diff --git a/buildscripts/task_generation/task_types/fuzzer_tasks.py b/buildscripts/task_generation/task_types/fuzzer_tasks.py
index 039dee97d67..f6665fc13ca 100644
--- a/buildscripts/task_generation/task_types/fuzzer_tasks.py
+++ b/buildscripts/task_generation/task_types/fuzzer_tasks.py
@@ -41,6 +41,8 @@ class FuzzerGenTaskParams(NamedTuple):
     timeout_secs: Timeout before test execution is considered hung.
     require_multiversion_setup: Requires downloading Multiversion binaries.
     use_large_distro: Should tests be generated on a large distro.
+    config_location: S3 path to the generated config tarball. None if no generated config files.
+    dependencies: Set of dependencies generated tasks should depend on.
     """
 
     task_name: str
@@ -59,6 +61,7 @@ class FuzzerGenTaskParams(NamedTuple):
     use_large_distro: Optional[bool]
     large_distro_name: Optional[str]
     config_location: str
+    dependencies: Set[str]
 
     def jstestfuzz_params(self) -> Dict[str, str]:
         """Build a dictionary of parameters to pass to jstestfuzz."""
@@ -141,4 +144,6 @@ class FuzzerGenTaskService:
             FunctionCall(RUN_GENERATED_TESTS, run_tests_vars)
         ]
 
-        return Task(sub_task_name, commands, {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)})
+        dependencies = {TaskDependency(dependency) for dependency in params.dependencies}
+
+        return Task(sub_task_name, commands, dependencies)
diff --git a/buildscripts/task_generation/task_types/resmoke_tasks.py b/buildscripts/task_generation/task_types/resmoke_tasks.py
index 5ddd4f9a541..a864604712a 100644
--- a/buildscripts/task_generation/task_types/resmoke_tasks.py
+++ b/buildscripts/task_generation/task_types/resmoke_tasks.py
@@ -1,19 +1,21 @@
 """Task generation for split resmoke tasks."""
-from typing import Set, Any, Dict, NamedTuple, Optional, List
+from typing import Any, Dict, List, NamedTuple, Optional, Set
 
 import inject
 import structlog
-from shrub.v2 import Task, TaskDependency, FunctionCall
+from shrub.v2 import FunctionCall, Task, TaskDependency
 
 from buildscripts.ciconfig.evergreen import EvergreenProjectConfig
 from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG
-from buildscripts.task_generation.constants import ARCHIVE_DIST_TEST_DEBUG_TASK, EXCLUDES_TAGS_FILE_PATH, \
-    BACKPORT_REQUIRED_TAG, CONFIGURE_EVG_CREDENTIALS, RUN_GENERATED_TESTS
+from buildscripts.task_generation.constants import (
+    BACKPORT_REQUIRED_TAG, CONFIGURE_EVG_CREDENTIALS, EXCLUDES_TAGS_FILE_PATH, RUN_GENERATED_TESTS)
 from buildscripts.task_generation.suite_split import GeneratedSuite
-from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions
-from buildscripts.task_generation.task_types.models.resmoke_task_model import ResmokeTask
-from buildscripts.task_generation.task_types.multiversion_decorator import MultiversionGenTaskDecorator, \
-    MultiversionDecoratorParams
+from buildscripts.task_generation.task_types.gentask_options import \
+    GenTaskOptions
+from buildscripts.task_generation.task_types.models.resmoke_task_model import \
+    ResmokeTask
+from buildscripts.task_generation.task_types.multiversion_decorator import (
+    MultiversionDecoratorParams, MultiversionGenTaskDecorator)
 from buildscripts.timeouts.timeout import TimeoutEstimate
 
 LOGGER = structlog.getLogger(__name__)
@@ -44,6 +46,7 @@ class ResmokeGenTaskParams(NamedTuple):
     resmoke_jobs_max: Max number of jobs that resmoke should execute in parallel.
     depends_on: List of tasks this task depends on.
     config_location: S3 path to the generated config tarball. None if no generated config files.
+    dependencies: Set of dependencies generated tasks should depend on.
     """
 
     use_large_distro: bool
@@ -54,6 +57,7 @@ class ResmokeGenTaskParams(NamedTuple):
     resmoke_args: str
     resmoke_jobs_max: Optional[int]
     config_location: str
+    dependencies: Set[str]
 
 
 class ResmokeGenTaskService:
@@ -157,7 +161,8 @@ class ResmokeGenTaskService:
             FunctionCall(RUN_GENERATED_TESTS, run_tests_vars),
         ]
 
-        shrub_task = Task(sub_task_name, [cmd for cmd in commands if cmd], self._get_dependencies())
+        shrub_task = Task(sub_task_name, [cmd for cmd in commands if cmd],
+                          self._get_dependencies(params))
         return ResmokeTask(shrub_task=shrub_task, resmoke_suite_name=suite.suite_name,
                            execution_task_suite_yaml_path=sub_suite_file_path,
                            execution_task_suite_yaml_name=sub_suite_file, test_list=sub_suite_roots,
@@ -228,7 +233,7 @@ class ResmokeGenTaskService:
         return variables
 
     @staticmethod
-    def _get_dependencies() -> Set[TaskDependency]:
+    def _get_dependencies(params: ResmokeGenTaskParams) -> Set[TaskDependency]:
         """Get the set of dependency tasks for these suites."""
-        dependencies = {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)}
+        dependencies = {TaskDependency(dependency) for dependency in params.dependencies}
         return dependencies
diff --git a/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py b/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py
index 7d1ce8bf119..0b81fbdc295 100644
--- a/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py
+++ b/buildscripts/tests/task_generation/task_types/test_fuzzer_tasks.py
@@ -5,9 +5,9 @@ import unittest
 import inject
 
 import buildscripts.task_generation.task_types.fuzzer_tasks as under_test
+from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
-from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
 
 
 def build_mock_fuzzer_params(jstestfuzz_vars="vars for jstestfuzz", npm_command="jstestfuzz"):
@@ -28,6 +28,7 @@ def build_mock_fuzzer_params(jstestfuzz_vars="vars for jstestfuzz", npm_command=
         use_large_distro=None,
         large_distro_name="large distro",
         config_location="config_location",
+        dependencies={"archive_dist_test_debug"},
     )
 
 
diff --git a/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py b/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py
index 39c9e85923a..564df7d9121 100644
--- a/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py
+++ b/buildscripts/tests/task_generation/task_types/test_resmoke_tasks.py
@@ -2,13 +2,13 @@
 import unittest
 
 import inject
-from mock import MagicMock
 
 import buildscripts.task_generation.task_types.resmoke_tasks as under_test
-from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, Variant
+from buildscripts.ciconfig.evergreen import EvergreenProjectConfig
 from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
 from buildscripts.task_generation.suite_split import GeneratedSuite, SubSuite
-from buildscripts.task_generation.task_types.gentask_options import GenTaskOptions
+from buildscripts.task_generation.task_types.gentask_options import \
+    GenTaskOptions
 from buildscripts.util.teststats import TestRuntime
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
@@ -47,6 +47,7 @@ def build_mock_gen_params(repeat_suites=1, resmoke_args="resmoke args"):
         resmoke_jobs_max=None,
         large_distro_name=None,
         config_location="generated_config",
+        dependencies={"archive_dist_test_debug"},
     )
 
 
diff --git a/buildscripts/tests/task_generation/test_gen_task_service.py b/buildscripts/tests/task_generation/test_gen_task_service.py
index caf6027736e..831efed4d72 100644
--- a/buildscripts/tests/task_generation/test_gen_task_service.py
+++ b/buildscripts/tests/task_generation/test_gen_task_service.py
@@ -6,9 +6,11 @@ from unittest.mock import MagicMock
 import inject
 from shrub.v2 import BuildVariant
 
-import buildscripts.task_generation.gen_task_service as under_test
 from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
-from buildscripts.task_generation.task_types.fuzzer_tasks import FuzzerGenTaskService
+from buildscripts.task_generation.task_types.fuzzer_tasks import \
+    FuzzerGenTaskService
+
+import buildscripts.task_generation.gen_task_service as under_test
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
 
@@ -31,6 +33,7 @@ def build_mock_fuzzer_params(multi_version=False, use_large_distro=None, large_d
         use_large_distro=use_large_distro,
         large_distro_name=large_distro_name,
         config_location="config location",
+        dependencies={"archive_dist_test_debug"},
     )
 
 
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml
index c2e26d315d2..31f36651e0f 100644
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@@ -89,7 +89,7 @@ variables:
   name: gen_task_template
   depends_on:
   - name: build_variant_gen
-  - name: archive_dist_test
+  - name: archive_dist_test_debug
   commands:
   - func: "generate resmoke tasks"
     vars:
@@ -154,7 +154,7 @@ variables:
   exec_timeout_secs: 14400 # Time out the task if it runs for more than 4 hours.
   depends_on:
   - build_variant_gen
-  - archive_dist_test
+  - archive_dist_test_debug
   commands:
   - func: "generate resmoke tasks"