SERVER-69353 Generate burn-in tasks separately

11 files changed, 168 insertions, 80 deletions

diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index 27da0996b63..84c7d98aab0 100755
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -13,6 +13,7 @@ from typing import Optional, Set, Tuple, List, Dict, NamedTuple
 import click
 import yaml
 from git import Repo
+from pydantic import BaseModel
 import structlog
 from structlog.stdlib import LoggerFactory
 
@@ -352,7 +353,8 @@ def create_task_list_for_tests(changed_tests: Set[str], build_variant: str,
 
 
 def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig,
-                         changed_tests: Set[str], install_dir: str) -> Dict[str, TaskInfo]:
+                         changed_tests: Set[str],
+                         install_dir: Optional[str]) -> Dict[str, TaskInfo]:
     """
     Create a list of tests by task.
 
@@ -367,7 +369,11 @@ def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig,
         exclude_tests.append(f"{ENTERPRISE_MODULE_PATH}/**/*")
     changed_tests = filter_tests(changed_tests, exclude_tests)
 
-    buildscripts.resmokelib.parser.set_run_options(f"--installDir={shlex.quote(install_dir)}")
+    run_options = ""
+    if install_dir is not None:
+        run_options = f"--installDir={shlex.quote(install_dir)}"
+    buildscripts.resmokelib.parser.set_run_options(run_options)
+
     if changed_tests:
         return create_task_list_for_tests(changed_tests, build_variant, evg_conf, exclude_suites,
                                           exclude_tasks)
@@ -408,7 +414,7 @@ def _configure_logging(verbose: bool):
     logging.basicConfig(
         format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s",
         level=level,
-        stream=sys.stdout,
+        stream=sys.stderr,
     )
     for log_name in EXTERNAL_LOGGERS:
         logging.getLogger(log_name).setLevel(logging.WARNING)
@@ -537,11 +543,45 @@ class LocalBurnInExecutor(BurnInExecutor):
         run_tests(tests_by_task, resmoke_cmd)
 
 
+class DiscoveredTask(BaseModel):
+    """
+    Model for a discovered task to run.
+
+    * task_name: Name of discovered task.
+    * test_list: List of tests to run under discovered task.
+    """
+
+    task_name: str
+    test_list: List[str]
+
+
+class DiscoveredTaskList(BaseModel):
+    """Model for a list of discovered tasks."""
+
+    discovered_tasks: List[DiscoveredTask]
+
+
+class YamlBurnInExecutor(BurnInExecutor):
+    """A burn-in executor that outputs discovered tasks as YAML."""
+
+    def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None:
+        """
+        Report the given tasks and their tests to stdout.
+
+        :param tests_by_task: Dictionary of tasks to run with tests to run in each.
+        """
+        discovered_tasks = DiscoveredTaskList(discovered_tasks=[
+            DiscoveredTask(task_name=task_name, test_list=task_info.tests)
+            for task_name, task_info in tests_by_task.items()
+        ])
+        print(yaml.safe_dump(discovered_tasks.dict()))
+
+
 class BurnInOrchestrator:
     """Orchestrate the execution of burn_in_tests."""
 
     def __init__(self, change_detector: FileChangeDetector, burn_in_executor: BurnInExecutor,
-                 evg_conf: EvergreenProjectConfig) -> None:
+                 evg_conf: EvergreenProjectConfig, install_dir: Optional[str]) -> None:
         """
         Create a new orchestrator.
 
@@ -552,8 +592,9 @@ class BurnInOrchestrator:
         self.change_detector = change_detector
         self.burn_in_executor = burn_in_executor
         self.evg_conf = evg_conf
+        self.install_dir = install_dir
 
-    def burn_in(self, repos: List[Repo], build_variant: str, install_dir: str) -> None:
+    def burn_in(self, repos: List[Repo], build_variant: str) -> None:
         """
         Execute burn in tests for the given git repositories.
 
@@ -564,7 +605,7 @@ class BurnInOrchestrator:
         LOGGER.info("Found changed tests", files=changed_tests)
 
         tests_by_task = create_tests_by_task(build_variant, self.evg_conf, changed_tests,
-                                             install_dir)
+                                             self.install_dir)
         LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
 
         self.burn_in_executor.execute(tests_by_task)
@@ -584,18 +625,20 @@ class BurnInOrchestrator:
               help="The maximum number of times to repeat tests if time option is specified.")
 @click.option("--repeat-tests-secs", "repeat_tests_secs", default=None, type=int, metavar="SECONDS",
               help="Repeat tests for the given time (in secs).")
+@click.option("--yaml", "use_yaml", is_flag=True, default=False,
+              help="Output discovered tasks in YAML. Tests will not be run.")
 @click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.")
 @click.option(
     "--origin-rev", "origin_rev", default=None,
     help="The revision in the mongo repo that changes will be compared against if specified.")
-@click.option("--install-dir", "install_dir", required=True, type=str,
+@click.option("--install-dir", "install_dir", type=str,
               help="Path to bin directory of a testable installation")
 @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED)
 # pylint: disable=too-many-arguments,too-many-locals
 def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int],
          repeat_tests_min: Optional[int], repeat_tests_max: Optional[int],
          repeat_tests_secs: Optional[int], resmoke_args: str, verbose: bool,
-         origin_rev: Optional[str], install_dir: str) -> None:
+         origin_rev: Optional[str], install_dir: Optional[str], use_yaml: bool) -> None:
     """
     Run new or changed tests in repeated mode to validate their stability.
 
@@ -639,11 +682,13 @@ def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int],
 
     change_detector = LocalFileChangeDetector(origin_rev)
     executor = LocalBurnInExecutor(resmoke_args, repeat_config)
-    if no_exec:
+    if use_yaml:
+        executor = YamlBurnInExecutor()
+    elif no_exec:
         executor = NopBurnInExecutor()
 
-    burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf)
-    burn_in_orchestrator.burn_in(repos, build_variant, install_dir)
+    burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir)
+    burn_in_orchestrator.burn_in(repos, build_variant)
 
 
 if __name__ == "__main__":
diff --git a/buildscripts/evergreen_burn_in_tests.py b/buildscripts/evergreen_burn_in_tests.py
index 37a0d2874ff..a628a9a5d20 100644
--- a/buildscripts/evergreen_burn_in_tests.py
+++ b/buildscripts/evergreen_burn_in_tests.py
@@ -436,8 +436,8 @@ def burn_in(task_id: str, build_variant: str, generate_config: GenerateConfig,
     change_detector = EvergreenFileChangeDetector(task_id, evg_api, os.environ)
     executor = GenerateBurnInExecutor(generate_config, repeat_config, evg_api, generate_tasks_file)
 
-    burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf)
-    burn_in_orchestrator.burn_in(repos, build_variant, install_dir)
+    burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir)
+    burn_in_orchestrator.burn_in(repos, build_variant)
 
 
 @click.command()
diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py
index c51f8c60ecf..369131eff60 100644
--- a/buildscripts/tests/test_burn_in_tests.py
+++ b/buildscripts/tests/test_burn_in_tests.py
@@ -4,12 +4,14 @@ from __future__ import absolute_import
 
 import collections
 import datetime
+from io import StringIO
 import os
 import sys
 import subprocess
 import unittest
 
 from mock import Mock, patch, MagicMock
+import yaml
 
 import buildscripts.burn_in_tests as under_test
 from buildscripts.ciconfig.evergreen import parse_evergreen_file, VariantTask
@@ -556,3 +558,19 @@ class TestLocalFileChangeDetector(unittest.TestCase):
         self.assertIn(file_list[2], found_tests)
         self.assertNotIn(file_list[1], found_tests)
         self.assertEqual(2, len(found_tests))
+
+
+class TestYamlBurnInExecutor(unittest.TestCase):
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_found_tasks_should_be_reported_as_yaml(self, stdout):
+        n_tasks = 5
+        n_tests = 3
+        tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
+
+        yaml_executor = under_test.YamlBurnInExecutor()
+        yaml_executor.execute(tests_by_task)
+
+        yaml_raw = stdout.getvalue()
+        results = yaml.safe_load(yaml_raw)
+        self.assertEqual(n_tasks, len(results["discovered_tasks"]))
+        self.assertEqual(n_tests, len(results["discovered_tasks"][0]["test_list"]))
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index ef5e180275a..c1571745fd1 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -643,7 +643,7 @@ buildvariants:
     target_resmoke_time: 10
     max_sub_suites: 5
     large_distro_name: rhel80-medium
-    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-dynamic-required
     num_scons_link_jobs_available: 0.99
     test_flags: >-
       --mongodSetParameters="{roundtripBsonColumnOnValidate: true}"
@@ -1306,7 +1306,9 @@ buildvariants:
     idle_timeout_factor: 1.5
     exec_timeout_factor: 1.5
     large_distro_name: rhel80-medium
-    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-dynamic-required
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
   tasks:
   - name: compile_test_and_package_parallel_core_stream_TG
@@ -1579,7 +1581,9 @@ buildvariants:
     target_resmoke_time: 10
     max_sub_suites: 5
     large_distro_name: rhel80-medium
-    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-dynamic-required
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
     test_flags: >-
       --mongodSetParameters="{internalQueryForceClassicEngine: true}"
diff --git a/etc/evergreen_nightly.yml b/etc/evergreen_nightly.yml
index 5e8a5058866..abd31a0e91b 100644
--- a/etc/evergreen_nightly.yml
+++ b/etc/evergreen_nightly.yml
@@ -184,7 +184,9 @@ buildvariants:
     idle_timeout_factor: 1.5
     exec_timeout_factor: 1.5
     large_distro_name: rhel80-medium
-    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-dynamic-required
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
   tasks:
   - name: compile_test_and_package_parallel_core_stream_TG
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml
index 6b27f051064..9176c6b804e 100644
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@@ -107,6 +107,19 @@ variables:
     vars:
       resmoke_args: --help
 
+- &gen_burn_in_task_template
+  name: gen_burn_in_task_template
+  depends_on:
+  - name: version_gen
+    variant: generate-tasks-for-version
+  - name: version_burn_in_gen
+    variant: generate-tasks-for-version
+  - name: archive_dist_test_debug
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      resmoke_args: --help
+
 - &benchmark_template
   name: benchmark_template
   depends_on:
@@ -1073,7 +1086,7 @@ functions:
       files:
       - src/generated_resmoke_config/*.json
 
-  "generate burn in tags":
+  "generate version burn in":
   - *f_expansions_write
   - *configure_evergreen_api_credentials
   - command: subprocess.exec
@@ -1081,27 +1094,29 @@ functions:
     params:
       binary: bash
       args:
-      - "./src/evergreen/burn_in_tests_generate.sh"
+      - "./src/evergreen/generate_version_burn_in.sh"
   - command: archive.targz_pack
     params:
-      target: burn_in_tags_gen.tgz
-      source_dir: src/generated_burn_in_tags_config
+      target: generate_tasks_config.tgz
+      source_dir: src/generated_resmoke_config
       include:
       - "*"
   - command: s3.put
     params:
       aws_key: ${aws_key}
       aws_secret: ${aws_secret}
-      local_file: burn_in_tags_gen.tgz
-      remote_file: ${project}/${build_variant}/${revision}/burn_in_tags_gen/burn_in_tags_gen-${build_id}.tgz
+      local_file: generate_tasks_config.tgz
+      remote_file: ${project}/${revision}/generate_tasks/generated-burn-in-config-${version_id}.tgz
       bucket: mciuploads
       permissions: public-read
-      content_type: application/gzip
-      display_name: Burn_in_tags Task Config - Execution ${execution}
+      content_type: ${content_type|application/gzip}
+      display_name: Generated Burn In Task Config - Execution ${execution}
+      optional: true
   - command: generate.tasks
     params:
+      optional: true
       files:
-      - src/generated_burn_in_tags_config/burn_in_tags_gen.json
+      - src/generated_resmoke_config/*.json
 
   "generate resmoke tasks":
   - *fetch_artifacts
@@ -3274,46 +3289,11 @@ tasks:
       args:
       - "src/evergreen/check_idl_compat.sh"
 
-- name: burn_in_tests_gen
+- <<: *gen_burn_in_task_template
+  name: burn_in_tests_gen
   tags: []
   commands:
-  - command: manifest.load
-  - func: "git get project and add git tag"
-  - *f_expansions_write
-  - *kill_processes
-  - *cleanup_environment
-  - func: "set up venv"
-  - func: "upload pip requirements"
-  - func: "configure evergreen api credentials"
-  - command: subprocess.exec
-    type: test
-    params:
-      binary: bash
-      args:
-      - "src/evergreen/burn_in_tests.sh"
-      env:
-        BURN_IN_TESTS: ${burn_in_tests}
-  - command: archive.targz_pack
-    params:
-      target: src/burn_in_tests_gen.tgz
-      source_dir: src
-      include:
-      - burn_in_tests_gen.json
-
-  - command: s3.put
-    params:
-      aws_key: ${aws_key}
-      aws_secret: ${aws_secret}
-      local_file: src/burn_in_tests_gen.tgz
-      remote_file: ${project}/${build_variant}/${revision}/burn_in_tests_gen/burn_in_tests_gen-${build_id}.tgz
-      bucket: mciuploads
-      permissions: public-read
-      content_type: application/gzip
-      display_name: Burn_in_tests Task Config - Execution ${execution}
-  - command: generate.tasks
-    params:
-      files:
-      - src/burn_in_tests_gen.json
+  - func: "generate resmoke tasks"
 
 - <<: *benchmark_template
   name: benchmarks_orphaned
@@ -4325,27 +4305,25 @@ tasks:
   commands:
   - func: "generate resmoke tasks"
 
-- name: burn_in_tags_gen
+- <<: *gen_burn_in_task_template
+  name: burn_in_tags_gen
   tags: []
-  depends_on:
-  - name: archive_dist_test
+  commands:
+  - func: "generate resmoke tasks"
+
+- name: version_gen
   commands:
   - command: manifest.load
-  - func: "git get project and add git tag"
+  - *git_get_project
   - *f_expansions_write
+  - *add_git_tag
   - *kill_processes
   - *cleanup_environment
   - func: "set up venv"
   - func: "upload pip requirements"
-  - func: "configure evergreen api credentials"
-  - func: "generate burn in tags"
-    vars:
-      max_revisions: 25
-      repeat_tests_secs: 600
-      repeat_tests_min: 2
-      repeat_tests_max: 1000
+  - func: "generate version"
 
-- name: version_gen
+- name: version_burn_in_gen
   commands:
   - command: manifest.load
   - *git_get_project
@@ -4355,7 +4333,7 @@ tasks:
   - *cleanup_environment
   - func: "set up venv"
   - func: "upload pip requirements"
-  - func: "generate version"
+  - func: "generate version burn in"
 
 - name: version_expansions_gen
   commands:
diff --git a/etc/evergreen_yml_components/variants/task_generation.yml b/etc/evergreen_yml_components/variants/task_generation.yml
index b45a68d1ff8..e5cdf11b1c7 100644
--- a/etc/evergreen_yml_components/variants/task_generation.yml
+++ b/etc/evergreen_yml_components/variants/task_generation.yml
@@ -9,4 +9,5 @@ buildvariants:
   - rhel80-medium
   tasks:
   - name: version_gen
+  - name: version_burn_in_gen
   - name: version_expansions_gen
diff --git a/evergreen/generate_version.sh b/evergreen/generate_version.sh
index b51cc809f6c..5a0ad5bb8d6 100644
--- a/evergreen/generate_version.sh
+++ b/evergreen/generate_version.sh
@@ -6,9 +6,7 @@ cd src
 set -o errexit
 set -o verbose
 
-curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.5.3/mongo-task-generator --output mongo-task-generator
-chmod +x mongo-task-generator
-
+setup_mongo_task_generator
 activate_venv
 PATH=$PATH:$HOME:/ ./mongo-task-generator \
   --expansion-file ../expansions.yml \
diff --git a/evergreen/generate_version_burn_in.sh b/evergreen/generate_version_burn_in.sh
new file mode 100644
index 00000000000..8c37474c86e
--- /dev/null
+++ b/evergreen/generate_version_burn_in.sh
@@ -0,0 +1,17 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+setup_mongo_task_generator
+activate_venv
+PATH=$PATH:$HOME:/ ./mongo-task-generator \
+  --expansion-file ../expansions.yml \
+  --evg-auth-file ./.evergreen.yml \
+  --evg-project-file ${evergreen_config_file_path} \
+  --generate-sub-tasks-config etc/generate_subtasks_config.yml \
+  --burn-in \
+  $@
diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh
index 3844afaead5..72de6ac13bb 100755
--- a/evergreen/prelude.sh
+++ b/evergreen/prelude.sh
@@ -11,6 +11,7 @@ evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$evergreen_dir/prelude_python.sh"
 . "$evergreen_dir/prelude_venv.sh"
 . "$evergreen_dir/prelude_db_contrib_tool.sh"
+. "$evergreen_dir/prelude_mongo_task_generator.sh"
 
 expansions_yaml="$evergreen_dir/../../expansions.yml"
 expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh
new file mode 100644
index 00000000000..511ba44c7cd
--- /dev/null
+++ b/evergreen/prelude_mongo_task_generator.sh
@@ -0,0 +1,24 @@
+function setup_mongo_task_generator {
+  if [ ! -f mongo-task-generator ]; then
+    curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.6.2/mongo-task-generator --output mongo-task-generator
+    chmod +x mongo-task-generator
+  fi
+}
+
+## Comment above and uncomment below to test unreleased mongo-task-generator changes that are
+## pushed to the `<branch-name>` of the `git@github.com:<user-name>/mongo-task-generator.git`
+## repo
+#function setup_mongo_task_generator {
+#  if [ ! -f mongo-task-generator ]; then
+#
+#    curl https://sh.rustup.rs -sSf | sh -s -- -y
+#    source "$HOME/.cargo/env"
+#    git clone git@github.com:<user-name>/mongo-task-generator.git unreleased-mongo-task-generator
+#    pushd unreleased-mongo-task-generator
+#    git checkout <branch-name>
+#    cargo build --release --locked
+#    generator_path="$(pwd)/target/release/mongo-task-generator"
+#    popd
+#    cp "$generator_path" mongo-task-generator
+#  fi
+#}