SERVER-41762 burn_in_tags should not need to generate a compile task

7 files changed, 162 insertions, 58 deletions

diff --git a/buildscripts/burn_in_tags.py b/buildscripts/burn_in_tags.py
index 9b595ecd624..c1d5bd0d6b2 100644
--- a/buildscripts/burn_in_tags.py
+++ b/buildscripts/burn_in_tags.py
@@ -76,22 +76,28 @@ def _create_evg_buildvariant_map(expansions_file_data):
     :param expansions_file_data: Config data file to use.
     :return: Map of base buildvariants to their generated buildvariants.
     """
-    buildvariant_map = {}
-    base_variants = expansions_file_data["base_variants"].split(
-        " ") if expansions_file_data["base_variants"] else []
-    for base_variant in base_variants:
-        new_variant_name = f"{base_variant}-required"
-        buildvariant_map[base_variant] = new_variant_name
-    return buildvariant_map
-
-
-def _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant):
+    burn_in_tags_gen_variant = expansions_file_data["build_variant"]
+    evergreen_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
+    burn_in_tags_gen_variant_config = evergreen_conf.get_variant(burn_in_tags_gen_variant)
+    burn_in_tag_buildvariants = burn_in_tags_gen_variant_config.expansions.get(
+        "burn_in_tag_buildvariants")
+    if burn_in_tag_buildvariants:
+        return {
+            base_variant: f"{base_variant}-required"
+            for base_variant in burn_in_tag_buildvariants.split(" ")
+        }
+    return {}
+
+
+def _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant,
+                               burn_in_tags_gen_variant):
     """
     Generate buildvariants for a given shrub config.
 
     :param shrub_config: Shrub config object that the generated buildvariant will be built upon.
     :param buildvariant: The base variant that the generated run_buildvariant will be based on.
     :param run_buildvariant: The generated buildvariant.
+    :param burn_in_tags_gen_variant: The buildvariant on which the burn_in_tags_gen task runs.
     """
     evergreen_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
     base_variant_config = evergreen_conf.get_variant(buildvariant)
@@ -100,9 +106,9 @@ def _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant):
     new_variant_run_on = base_variant_config.run_on[0]
 
     task_spec = TaskSpec("compile_TG")
-    task_spec.distro("rhel62-large")
 
-    new_variant = shrub_config.variant(run_buildvariant)
+    new_variant = shrub_config.variant(run_buildvariant).expansion("burn_in_bypass",
+                                                                   burn_in_tags_gen_variant)
     new_variant.display_name(new_variant_display_name)
     new_variant.run_on(new_variant_run_on)
     new_variant.task(task_spec)
@@ -127,7 +133,8 @@ def _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build
         config_options = _get_config_options(expansions_file_data, buildvariant, run_buildvariant)
         tests_by_task = create_tests_by_task(config_options, evergreen_api)
         if tests_by_task:
-            _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant)
+            _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant,
+                                       expansions_file_data["build_variant"])
             create_generate_tasks_config(evergreen_api, shrub_config, config_options, tests_by_task,
                                          False)
 
diff --git a/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py
new file mode 100644
index 00000000000..eb92ddd36c9
--- /dev/null
+++ b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Bypass compile and fetch binaries for burn_in_tags."""
+
+from urllib.parse import urlparse
+from buildscripts.bypass_compile_and_fetch_binaries import (
+    find_suitable_build_id, generate_bypass_expansions, parse_args, read_evg_config,
+    requests_get_json, write_out_bypass_compile_expansions)
+
+
+def main():  # pylint: disable=too-many-locals,too-many-statements
+    """Execute Main program."""
+
+    args = parse_args()
+    evg_config = read_evg_config()
+    if evg_config is None:
+        print("Could not find ~/.evergreen.yml config file. Default compile bypass to false.")
+        return
+
+    api_server = "{url.scheme}://{url.netloc}".format(
+        url=urlparse(evg_config.get("api_server_host")))
+    revision_url = f"{api_server}/rest/v1/projects/{args.project}/revisions/{args.revision}"
+    revisions = requests_get_json(revision_url)
+    build_id = find_suitable_build_id(revisions["builds"], args)
+    if not build_id:
+        print("Could not find build id for revision {args.revision} on project {args.project}."
+              " Default compile bypass to false.")
+        return
+
+    expansions = generate_bypass_expansions(args.project, args.buildVariant, args.revision,
+                                            build_id)
+    write_out_bypass_compile_expansions(args.outFile, **expansions)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py
index f1a659710b4..94e20710bb8 100755
--- a/buildscripts/bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/bypass_compile_and_fetch_binaries.py
@@ -302,6 +302,25 @@ def parse_args():
     return parser.parse_args()
 
 
+def find_suitable_build_id(builds, args):
+    """
+    Find a build_id that fits the given parameters.
+
+    :param builds: List of builds.
+    :param args: The parameters a build must meet, including project, buildVariant, and revision.
+    :return: Build_id that matches the parameters.
+    """
+    prefix = "{}_{}_{}_".format(args.project, args.buildVariant, args.revision)
+    # The "project" and "buildVariant" passed in may contain "-", but the "builds" listed from
+    # Evergreen only contain "_". Replace the hyphens before searching for the build.
+    prefix = prefix.replace("-", "_")
+    build_id_pattern = re.compile(prefix)
+    for build_id in builds:
+        if build_id_pattern.search(build_id):
+            return build_id
+    return None
+
+
 def main():  # pylint: disable=too-many-locals,too-many-statements
     """Execute Main entry.
 
@@ -329,19 +348,8 @@ def main():  # pylint: disable=too-many-locals,too-many-statements
         revision_url = "{}/rest/v1/projects/{}/revisions/{}".format(api_server, args.project,
                                                                     args.revision)
         revisions = requests_get_json(revision_url)
-
-        prefix = "{}_{}_{}_".format(args.project, args.buildVariant, args.revision)
-        # The "project" and "buildVariant" passed in may contain "-", but the "builds" listed from
-        # Evergreen only contain "_". Replace the hyphens before searching for the build.
-        prefix = prefix.replace("-", "_")
-        build_id_pattern = re.compile(prefix)
-        build_id = None
-        for build_id in revisions["builds"]:
-            # Find a suitable build_id
-            match = build_id_pattern.search(build_id)
-            if match:
-                break
-        else:
+        build_id = find_suitable_build_id(revisions["builds"], args)
+        if not build_id:
             print("Could not find build id for revision {} on project {}."
                   " Default compile bypass to false.".format(args.revision, args.project))
             return
diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py
index b036370fdf9..5a9c72ca602 100644
--- a/buildscripts/tests/test_burn_in_tags.py
+++ b/buildscripts/tests/test_burn_in_tags.py
@@ -44,11 +44,10 @@ def get_evergreen_config():
 
 
 class TestCreateEvgBuildVariantMap(unittest.TestCase):
-    def test_create_evg_buildvariant_map(self):
-        expansions_file_data = {
-            "base_variants":
-                "enterprise-rhel-62-64-bit-majority-read-concern-off enterprise-rhel-62-64-bit-inmem"
-        }
+    @mock.patch(ns("evergreen"))
+    def test_create_evg_buildvariant_map(self, evergreen_mock):
+        evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config()
+        expansions_file_data = {"build_variant": "enterprise-rhel-62-64-bit"}
         buildvariant_map = burn_in_tags._create_evg_buildvariant_map(expansions_file_data)
         expected_buildvariant_map = {
             "enterprise-rhel-62-64-bit-majority-read-concern-off":
@@ -58,8 +57,10 @@ class TestCreateEvgBuildVariantMap(unittest.TestCase):
         }
         self.assertEqual(buildvariant_map, expected_buildvariant_map)
 
-    def test_create_evg_buildvariant_map_no_base_variants(self):
-        expansions_file_data = {"base_variants": ""}
+    @mock.patch(ns("evergreen"))
+    def test_create_evg_buildvariant_map_no_base_variants(self, evergreen_mock):
+        evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config()
+        expansions_file_data = {"build_variant": "buildvariant-without-burn-in-tag-buildvariants"}
         buildvariant_map = burn_in_tags._create_evg_buildvariant_map(expansions_file_data)
         self.assertEqual(buildvariant_map, {})
 
@@ -69,17 +70,22 @@ class TestGenerateEvgBuildVariants(unittest.TestCase):
     def test_generate_evg_buildvariant_one_base_variant(self, evergreen_mock):
         evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config()
         base_variant = "enterprise-rhel-62-64-bit-inmem"
-        expected_variant_data = get_evergreen_config().get_variant(base_variant)
-
+        generated_variant = "enterprise-rhel-62-64-bit-inmem-required"
+        burn_in_tags_gen_variant = "enterprise-rhel-62-64-bit"
         shrub_config = Configuration()
-        burn_in_tags._generate_evg_buildvariant(shrub_config, base_variant,
-                                                "enterprise-rhel-62-64-bit-inmem-required")
+        burn_in_tags._generate_evg_buildvariant(shrub_config, base_variant, generated_variant,
+                                                burn_in_tags_gen_variant)
+
+        expected_variant_data = get_evergreen_config().get_variant(base_variant)
         generated_buildvariants = shrub_config.to_map()["buildvariants"]
         self.assertEqual(len(generated_buildvariants), 1)
         generated_build_variant = generated_buildvariants[0]
-        self.assertEqual(generated_build_variant["name"], f"{expected_variant_data}-required")
+        self.assertEqual(generated_build_variant["name"], generated_variant)
         self.assertEqual(generated_build_variant["modules"], expected_variant_data.modules)
-        self.assertEqual(generated_build_variant["expansions"], expected_variant_data.expansions)
+        generated_expansions = generated_build_variant["expansions"]
+        burn_in_bypass_expansion_value = generated_expansions.pop("burn_in_bypass")
+        self.assertEqual(burn_in_bypass_expansion_value, burn_in_tags_gen_variant)
+        self.assertEqual(generated_expansions, expected_variant_data.expansions)
 
 
 class TestGenerateEvgTasks(unittest.TestCase):
diff --git a/buildscripts/tests/test_burn_in_tags_evergreen.yml b/buildscripts/tests/test_burn_in_tags_evergreen.yml
index 165cb669345..815906704f9 100644
--- a/buildscripts/tests/test_burn_in_tags_evergreen.yml
+++ b/buildscripts/tests/test_burn_in_tags_evergreen.yml
@@ -15,14 +15,40 @@ tasks:
   depends_on: []
   commands:
     - func: "fetch source"
+- name: burn_in_tags_gen
+  depends_on: []
+  commands:
+    - func: "fake command"
 
 buildvariants:
+- name: enterprise-rhel-62-64-bit
+  display_name: "! Enterprise RHEL 6.2"
+  expansions:
+    multiversion_platform: rhel62
+    burn_in_tag_buildvariants: enterprise-rhel-62-64-bit-majority-read-concern-off enterprise-rhel-62-64-bit-inmem
+  tasks:
+  - name: burn_in_tags_gen
+- name: buildvariant-without-burn-in-tag-buildvariants
+  display_name: "Buildvariant without burn in tag buildvariants expansion"
+  expansions:
+    multiversion_platform: rhel62
+  tasks:
+  - name: burn_in_tags_gen
 - name: enterprise-rhel-62-64-bit-majority-read-concern-off
   display_name: "Enterprise RHEL 6.2 (majority read concern off)"
   modules: ["enterprise"]
   run_on:
   - rhel62-small
   expansions: &enterprise-rhel-62-64-bit-majority-read-concern-off-expansions
+    multiversion_edition: enterprise
+  tasks:
+  - name: compile
+- name: enterprise-rhel-62-64-bit-inmem
+  display_name: Enterprise RHEL 6.2 (inMemory)
+  modules: ["enterprise"]
+  run_on:
+  - rhel62-small
+  expansions: &enterprise-rhel-62-64-bit-inmem-expansions
     test_flags: >-
       --majorityReadConcern=off
       --excludeWithAnyTags=requires_majority_read_concern,uses_prepare_transaction,uses_multi_shard_transaction,uses_atclustertime
@@ -32,24 +58,6 @@ buildvariants:
       --release
       --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
       MONGO_DISTMOD=rhel62
-      multiversion_platform: rhel62
-    multiversion_edition: enterprise
-    repo_edition: enterprise
-    scons_cache_scope: shared
-    tooltags: "ssl sasl gssapi"
-    large_distro_name: rhel62-large
-  tasks:
-  - name: compile
-- name: enterprise-rhel-62-64-bit-inmem
-  display_name: Enterprise RHEL 6.2 (inMemory)
-  modules:
-  - enterprise
-  run_on:
-  - rhel62-small
-  batchtime: 1440 # 1 day
-  expansions: &enterprise-rhel-62-64-bit-inmem-expansions
-    test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling
-    compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: rhel62
     multiversion_edition: enterprise
     scons_cache_scope: shared
diff --git a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
index 1adde7bc239..b93c7440504 100644
--- a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
@@ -2,6 +2,7 @@
 
 import unittest
 
+from collections import namedtuple
 from mock import mock_open, patch, MagicMock
 
 import buildscripts.bypass_compile_and_fetch_binaries as under_test
@@ -138,3 +139,28 @@ pytests/test2.py
 
             open_mock.return_value.__enter__.return_value = git_changes.splitlines()
             self.assertFalse(under_test.should_bypass_compile(MagicMock()))
+
+
+def generate_args():
+    project = "project-master"
+    build_variant = "rhel-62-64-bit"
+    revision = "fakesha"
+    Args = namedtuple("Args", ["project", "buildVariant", "revision"])
+    return Args(project=project, buildVariant=build_variant, revision=revision)
+
+
+class TestFindSuitableBuildID(unittest.TestCase):
+    def test_builds(self):
+        expected_build_id = "project_master_rhel_62_64_bit_fakesha_19_07_10_15_48_53"
+        builds = ["build1", "build2", expected_build_id]
+        args = generate_args()
+        build_id = under_test.find_suitable_build_id(builds, args)
+
+        self.assertEqual(build_id, expected_build_id)
+
+    def test_no_builds(self):
+        builds = []
+        args = generate_args()
+        build_id = under_test.find_suitable_build_id(builds, args)
+
+        self.assertEqual(build_id, None)
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index fbb2c55f3cf..197c2ee5b0a 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -1153,6 +1153,18 @@ functions:
         set -o verbose
         set -o errexit
 
+        if [ -n "${burn_in_bypass}" ]; then
+          ${activate_virtualenv}
+          # Evergreen executable is in $HOME, so add that to the path.
+          PATH=$PATH:$HOME $python buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py            \
+            --project ${project}                                               \
+            --buildVariant ${burn_in_bypass}                                   \
+            --revision ${revision}                                             \
+            --patchFile patch_files.txt                                        \
+            --outFile bypass_compile_expansions.yml                            \
+            --jsonArtifact artifacts.json
+        fi
+
         # For patch builds determine if we can bypass compile.
         if [[ "${is_patch}" = "true" && "${task_name}" = "compile" ]]; then
           ${activate_virtualenv}
@@ -5714,6 +5726,8 @@ tasks:
       fallback_num_sub_suites: 4
 
 - name: burn_in_tags_gen
+  depends_on:
+    - name: compile
   commands:
   - command: manifest.load
   - func: "git get project"
@@ -5727,7 +5741,6 @@ tasks:
       repeat_tests_secs: 600
       repeat_tests_min: 2
       repeat_tests_max: 1000
-      base_variants: enterprise-rhel-62-64-bit-majority-read-concern-off enterprise-rhel-62-64-bit-inmem linux-64-duroff
 
 - name: auth_audit_gen
   tags: ["auth", "audit"]
@@ -10231,6 +10244,7 @@ buildvariants:
     jstestfuzz_concurrent_num_files: 10
     target_resmoke_time: 10
     large_distro_name: rhel62-large
+    burn_in_tag_buildvariants: enterprise-rhel-62-64-bit-majority-read-concern-off enterprise-rhel-62-64-bit-inmem linux-64-duroff
   display_tasks:
   - *dbtest
   - *unittests