SERVER-69353 Generate burn-in tasks separately

(cherry picked from commit 9efc698c77962029620d62669047f23026766f61)

7 files changed, 107 insertions, 17 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 369f2800f94..06a0186ce69 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -1212,6 +1212,8 @@ buildvariants:
     exec_timeout_factor: 1.5
     large_distro_name: rhel80-medium
     burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
   tasks:
   - name: compile_test_and_package_parallel_core_stream_TG
@@ -1502,6 +1504,8 @@ buildvariants:
     max_sub_suites: 5
     large_distro_name: rhel80-medium
     burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem enterprise-rhel-80-64-bit-multiversion
+    burn_in_tag_compile_distro: rhel80-xlarge
+    burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG
     num_scons_link_jobs_available: 0.99
     test_flags: >-
       --mongodSetParameters="{internalQueryFrameworkControl: forceClassicEngine}"
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml
index 5019ac2225a..89ae290d97b 100644
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@@ -102,6 +102,19 @@ variables:
     vars:
       resmoke_args: --help
 
+- &gen_burn_in_task_template
+  name: gen_burn_in_task_template
+  depends_on:
+  - name: version_gen
+    variant: generate-tasks-for-version
+  - name: version_burn_in_gen
+    variant: generate-tasks-for-version
+  - name: archive_dist_test_debug
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      resmoke_args: --help
+
 - &benchmark_template
   name: benchmark_template
   depends_on:
@@ -1033,6 +1046,38 @@ functions:
       files:
       - src/generated_resmoke_config/*.json
 
+  "generate version burn in":
+  - *f_expansions_write
+  - *configure_evergreen_api_credentials
+  - command: subprocess.exec
+    type: test
+    params:
+      binary: bash
+      args:
+      - "./src/evergreen/generate_version_burn_in.sh"
+  - command: archive.targz_pack
+    params:
+      target: generate_tasks_config.tgz
+      source_dir: src/generated_resmoke_config
+      include:
+      - "*"
+  - command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: generate_tasks_config.tgz
+      remote_file: ${project}/${revision}/generate_tasks/generated-burn-in-config-${version_id}.tgz
+      bucket: mciuploads
+      permissions: public-read
+      content_type: ${content_type|application/gzip}
+      display_name: Generated Burn In Task Config - Execution ${execution}
+      optional: true
+  - command: generate.tasks
+    params:
+      optional: true
+      files:
+      - src/generated_resmoke_config/*.json
+
   "generate resmoke tasks":
   - *fetch_artifacts
   - *f_expansions_write
@@ -3248,7 +3293,7 @@ tasks:
       args:
       - "src/evergreen/check_idl_compat.sh"
 
-- <<: *gen_task_template
+- <<: *gen_burn_in_task_template
   name: burn_in_tests_gen
   tags: []
   commands:
@@ -4313,7 +4358,7 @@ tasks:
   commands:
   - func: "generate resmoke tasks"
 
-- <<: *gen_task_template
+- <<: *gen_burn_in_task_template
   name: burn_in_tags_gen
   tags: []
   commands:
@@ -4331,6 +4376,18 @@ tasks:
   - func: "upload pip requirements"
   - func: "generate version"
 
+- name: version_burn_in_gen
+  commands:
+  - command: manifest.load
+  - *git_get_project
+  - *f_expansions_write
+  - *add_git_tag
+  - *kill_processes
+  - *cleanup_environment
+  - func: "set up venv"
+  - func: "upload pip requirements"
+  - func: "generate version burn in"
+
 - name: version_expansions_gen
   commands:
   - command: manifest.load
diff --git a/etc/evergreen_yml_components/variants/task_generation.yml b/etc/evergreen_yml_components/variants/task_generation.yml
index 6a345919a67..ba97594f051 100644
--- a/etc/evergreen_yml_components/variants/task_generation.yml
+++ b/etc/evergreen_yml_components/variants/task_generation.yml
@@ -10,4 +10,5 @@ buildvariants:
   - rhel80-medium
   tasks:
   - name: version_gen
+  - name: version_burn_in_gen
   - name: version_expansions_gen
diff --git a/evergreen/generate_version.sh b/evergreen/generate_version.sh
index ee629451f41..5a0ad5bb8d6 100644
--- a/evergreen/generate_version.sh
+++ b/evergreen/generate_version.sh
@@ -6,25 +6,11 @@ cd src
 set -o errexit
 set -o verbose
 
-curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.5.3/mongo-task-generator --output mongo-task-generator
-chmod +x mongo-task-generator
-
-## Comment above and uncomment below to test unreleased mongo-task-generator changes pushed to <branch-name>
-#curl https://sh.rustup.rs -sSf | sh -s -- -y
-#source "$HOME/.cargo/env"
-#git clone git@github.com:mongodb/mongo-task-generator.git unreleased-mongo-task-generator
-#pushd unreleased-mongo-task-generator
-#git checkout <branch-name>
-#cargo build --release --locked
-#generator_path="$(pwd)/target/release/mongo-task-generator"
-#popd
-#cp "$generator_path" mongo-task-generator
-
+setup_mongo_task_generator
 activate_venv
 PATH=$PATH:$HOME:/ ./mongo-task-generator \
   --expansion-file ../expansions.yml \
   --evg-auth-file ./.evergreen.yml \
   --evg-project-file ${evergreen_config_file_path} \
   --generate-sub-tasks-config etc/generate_subtasks_config.yml \
-  --burn-in \
   $@
diff --git a/evergreen/generate_version_burn_in.sh b/evergreen/generate_version_burn_in.sh
new file mode 100644
index 00000000000..8c37474c86e
--- /dev/null
+++ b/evergreen/generate_version_burn_in.sh
@@ -0,0 +1,17 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+setup_mongo_task_generator
+activate_venv
+PATH=$PATH:$HOME:/ ./mongo-task-generator \
+  --expansion-file ../expansions.yml \
+  --evg-auth-file ./.evergreen.yml \
+  --evg-project-file ${evergreen_config_file_path} \
+  --generate-sub-tasks-config etc/generate_subtasks_config.yml \
+  --burn-in \
+  $@
diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh
index 3844afaead5..72de6ac13bb 100755
--- a/evergreen/prelude.sh
+++ b/evergreen/prelude.sh
@@ -11,6 +11,7 @@ evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
 . "$evergreen_dir/prelude_python.sh"
 . "$evergreen_dir/prelude_venv.sh"
 . "$evergreen_dir/prelude_db_contrib_tool.sh"
+. "$evergreen_dir/prelude_mongo_task_generator.sh"
 
 expansions_yaml="$evergreen_dir/../../expansions.yml"
 expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh
new file mode 100644
index 00000000000..511ba44c7cd
--- /dev/null
+++ b/evergreen/prelude_mongo_task_generator.sh
@@ -0,0 +1,24 @@
+function setup_mongo_task_generator {
+  if [ ! -f mongo-task-generator ]; then
+    curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.6.2/mongo-task-generator --output mongo-task-generator
+    chmod +x mongo-task-generator
+  fi
+}
+
+## Comment above and uncomment below to test unreleased mongo-task-generator changes that are
+## pushed to the `<branch-name>` of the `git@github.com:<user-name>/mongo-task-generator.git`
+## repo
+#function setup_mongo_task_generator {
+#  if [ ! -f mongo-task-generator ]; then
+#
+#    curl https://sh.rustup.rs -sSf | sh -s -- -y
+#    source "$HOME/.cargo/env"
+#    git clone git@github.com:<user-name>/mongo-task-generator.git unreleased-mongo-task-generator
+#    pushd unreleased-mongo-task-generator
+#    git checkout <branch-name>
+#    cargo build --release --locked
+#    generator_path="$(pwd)/target/release/mongo-task-generator"
+#    popd
+#    cp "$generator_path" mongo-task-generator
+#  fi
+#}