SERVER-45748: Ensure the correct task is used in bypass-compile for burn_in_tags

 delete mode 100644 buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py

(cherry picked from commit 403d955f25f2018d7d05c405005b6043cace7d63)

 delete mode 100644 buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py

5 files changed, 65 insertions, 124 deletions

diff --git a/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py
index c03b1b73938..8a7e0678941 100644
--- a/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py
@@ -17,39 +17,6 @@ LOGGER = structlog.get_logger(__name__)
 EVG_CONFIG_FILE = ".evergreen.yml"
 
 
-def _retrieve_used_build_id(build):
-    """
-    Determine what build_id should be used for downloading artifacts.
-
-    If bypass_compile was used by the main compile task, then our expansions should use the
-    same references.
-
-    :param build: Evergreen build containing the compile task to use.
-    :return: build_id that should be used for expansions.
-    """
-    log = LOGGER.bind(build_id=build.id)
-    tasks = build.get_tasks()
-    possible_compile_tasks = {task for task in tasks if task.display_name == "compile"}
-    if len(possible_compile_tasks) != 1:
-        log.warning("Could not find 'compile' task")
-        raise ValueError(f"Compile task not found in {build.id}")
-
-    compile_task = possible_compile_tasks.pop()
-
-    # We will use the 'Binaries' artifact to see what build_id to use.
-    binary_artifacts = [
-        artifact for artifact in compile_task.artifacts if artifact.name == "Binaries"
-    ]
-    for artifact in binary_artifacts:
-        log.info("Checking artifact for build_id", url=artifact.url)
-        build_id = artifact.url.split("/")[-1].split(".")[0]
-        prefix = "mongo-"
-        return build_id[len(prefix):]
-
-    log.warning("Count not determine build_id")
-    raise ValueError(f"Could not determine build id for bypass compile in {build.id}")
-
-
 @click.command()
 @click.option("--project", required=True, help="The evergreen project.")
 @click.option("--build-variant", required=True, help="Build variant where compile is running.")
@@ -82,10 +49,10 @@ def main(  # pylint: disable=too-many-arguments,too-many-locals
     evg_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE)
 
     version = evg_api.version_by_id(version_id)
-    build_id = _retrieve_used_build_id(version.build_by_variant(build_variant))
+    build = version.build_by_variant(build_variant)
 
     target = TargetBuild(project=project, revision=revision, build_variant=build_variant)
-    gather_artifacts_and_update_expansions(build_id, target, json_artifact, out_file, evg_api)
+    gather_artifacts_and_update_expansions(build, target, json_artifact, out_file)
 
 
 if __name__ == "__main__":
diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py
index cd636585e22..028c4b37665 100755
--- a/buildscripts/bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/bypass_compile_and_fetch_binaries.py
@@ -14,7 +14,7 @@ import urllib.request
 
 import click
 
-from evergreen.api import RetryingEvergreenApi
+from evergreen.api import RetryingEvergreenApi, EvergreenApi, Build, Task
 from git.repo import Repo
 import requests
 import structlog
@@ -281,7 +281,7 @@ def should_bypass_compile(patch_file, build_variant):
     return True
 
 
-def find_build_for_previous_compile_task(evg_api, target):
+def find_build_for_previous_compile_task(evg_api: EvergreenApi, target: TargetBuild) -> Build:
     """
     Find build_id of the base revision.
 
@@ -292,35 +292,32 @@ def find_build_for_previous_compile_task(evg_api, target):
     project_prefix = target.project.replace("-", "_")
     version_of_base_revision = "{}_{}".format(project_prefix, target.revision)
     version = evg_api.version_by_id(version_of_base_revision)
-    build_id = version.build_by_variant(target.build_variant).id
-    return build_id
+    build = version.build_by_variant(target.build_variant)
+    return build
 
 
-def find_previous_compile_task(evg_api, build_id, revision):
+def find_previous_compile_task(build: Build) -> Task:
     """
-    Find compile task that should be used for skip compile..
+    Find compile task that should be used for skip compile.
 
-    :param evg_api: Evergreen.py object.
-    :param build_id: Build id of the desired compile task.
-    :param revision: The base revision being run against.
+    :param build: Build containing the desired compile task.
     :return: Evergreen.py object containing data about the desired compile task.
     """
-    index = build_id.find(revision)
-    compile_task_id = "{}compile_{}".format(build_id[:index], build_id[index:])
-    task = evg_api.task_by_id(compile_task_id)
-    return task
+    tasks = [task for task in build.get_tasks() if task.display_name == "compile"]
+    assert len(tasks) == 1
+    return tasks[0]
 
 
-def fetch_artifacts(evg_api, build_id, revision):
+def fetch_artifacts(build: Build, revision: str):
     """
     Fetch artifacts from a given revision.
 
-    :param evg_api: Evergreen.py object.
-    :param build_id: Build id of the desired artifacts.
+    :param build: Build id of the desired artifacts.
     :param revision: The revision being fetched from.
     :return: Artifacts from the revision.
     """
-    task = find_previous_compile_task(evg_api, build_id, revision)
+    LOGGER.info("Fetching artifacts", build_id=build.id, revision=revision)
+    task = find_previous_compile_task(build)
     if task is None or not task.is_success():
         LOGGER.warning(
             "Could not retrieve artifacts because the compile task for base commit"
@@ -330,7 +327,7 @@ def fetch_artifacts(evg_api, build_id, revision):
     artifacts = []
     for artifact in task.artifacts:
         filename = os.path.basename(artifact.url)
-        if filename.startswith(build_id):
+        if filename.startswith(build.id):
             LOGGER.info("Retrieving archive", filename=filename)
             # This is the artifacts.tgz as referenced in evergreen.yml.
             try:
@@ -395,24 +392,23 @@ def update_artifact_permissions(permission_dict):
         os.chmod(path, perm)
 
 
-def gather_artifacts_and_update_expansions(build_id, target, json_artifact_file, expansions_file,
-                                           evg_api):
+def gather_artifacts_and_update_expansions(build: Build, target: TargetBuild, json_artifact_file,
+                                           expansions_file):
     """
     Fetch the artifacts for this build and save them to be used by other tasks.
 
-    :param build_id: build_id of build with artifacts.
+    :param build: build containing artifacts.
     :param target: Target build being bypassed.
     :param json_artifact_file: File to write json artifacts to.
     :param expansions_file: Files to write expansions to.
-    :param evg_api: Evergreen Api.
     """
-    artifacts = fetch_artifacts(evg_api, build_id, target.revision)
+    artifacts = fetch_artifacts(build, target.revision)
     update_artifact_permissions(ARTIFACTS_NEEDING_PERMISSIONS)
     write_out_artifacts(json_artifact_file, artifacts)
 
-    LOGGER.info("Creating expansions files", target=target, build_id=build_id)
+    LOGGER.info("Creating expansions files", target=target, build_id=build.id)
 
-    expansions = generate_bypass_expansions(target, build_id)
+    expansions = generate_bypass_expansions(target, build.id)
     write_out_bypass_compile_expansions(expansions_file, **expansions)
 
 
@@ -452,13 +448,13 @@ def main(  # pylint: disable=too-many-arguments,too-many-locals,too-many-stateme
     # Determine if we should bypass compile based on modified patch files.
     if should_bypass_compile(patch_file, build_variant):
         evg_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE)
-        build_id = find_build_for_previous_compile_task(evg_api, target)
-        if not build_id:
+        build = find_build_for_previous_compile_task(evg_api, target)
+        if not build:
             LOGGER.warning("Could not find build id. Default compile bypass to false.",
                            revision=revision, project=project)
             return
 
-        gather_artifacts_and_update_expansions(build_id, target, json_artifact, out_file, evg_api)
+        gather_artifacts_and_update_expansions(build, target, json_artifact, out_file)
 
 
 if __name__ == "__main__":
diff --git a/buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py b/buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py
deleted file mode 100644
index a4e3225d7f8..00000000000
--- a/buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""Unit tests for burn_in_tags_bypass_compile_and_fetch_binaries."""
-
-import unittest
-from unittest.mock import MagicMock
-
-import buildscripts.burn_in_tags_bypass_compile_and_fetch_binaries as under_test
-
-# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
-
-
-class TestRetrieveUsedBuildId(unittest.TestCase):
-    def test_build_with_no_compile_throws_exception(self):
-        build_mock = MagicMock()
-
-        with self.assertRaises(ValueError):
-            under_test._retrieve_used_build_id(build_mock)
-
-    def test_compile_with_no_binaries_artifact_throws_exception(self):
-        build_mock = MagicMock()
-        compile_task = MagicMock(display_name="compile")
-        build_mock.get_tasks.return_value = [compile_task]
-
-        with self.assertRaises(ValueError):
-            under_test._retrieve_used_build_id(build_mock)
-
-    def test_build_id_from_compile_binaries_is_used(self):
-        build_id = "this_is_the_build_id"
-        url = f"http://s3.amazon.com/mciuploads/mongodb/build_var//binaries/mongo-{build_id}.tgz"
-        build_mock = MagicMock()
-        compile_task = MagicMock(display_name="compile")
-        build_mock.get_tasks.return_value = [MagicMock(), compile_task, MagicMock()]
-        artifact_mock = MagicMock(url=url)
-        artifact_mock.name = "Binaries"
-        compile_task.artifacts = [MagicMock(), artifact_mock, MagicMock()]
-
-        self.assertEqual(build_id, under_test._retrieve_used_build_id(build_mock))
diff --git a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
index 1a2979e6e64..e6ff1016dd1 100644
--- a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
@@ -149,32 +149,29 @@ pytests/test2.py
 class TestFindBuildForPreviousCompileTask(unittest.TestCase):
     def test_find_build(self):
         target = under_test.TargetBuild(project="project", revision="a22", build_variant="variant")
-        expected_build_id = "project_variant_patch_a22_date"
         evergreen_api = MagicMock()
-        version_response = MagicMock()
-        evergreen_api.version_by_id.return_value = version_response
-        version_response.build_by_variant.return_value = MagicMock(id=expected_build_id)
+        version_response = evergreen_api.version_by_id.return_value
 
-        build_id = under_test.find_build_for_previous_compile_task(evergreen_api, target)
-        self.assertEqual(build_id, expected_build_id)
+        build = under_test.find_build_for_previous_compile_task(evergreen_api, target)
+        self.assertEqual(build, version_response.build_by_variant.return_value)
 
 
 class TestFetchArtifactsForPreviousCompileTask(unittest.TestCase):
     def test_fetch_artifacts_with_task_with_artifact(self):
         revision = "a22"
         build_id = "project_variant_patch_a22_date"
+        build = MagicMock(id=build_id)
 
         artifact_mock = MagicMock()
         artifact_mock.name = "Binaries"
         artifact_mock.url = "http://s3.amazon.com/mciuploads/mongodb/build_var//binaries/mongo-test.tgz"
         artifacts_mock = [artifact_mock]
 
-        task_response = MagicMock(status="success")
+        task_response = MagicMock(status="success", display_name="compile")
         task_response.artifacts = artifacts_mock
-        evergreen_api = MagicMock()
-        evergreen_api.task_by_id.return_value = task_response
+        build.get_tasks.return_value = [task_response]
 
-        artifact_files = under_test.fetch_artifacts(evergreen_api, build_id, revision)
+        artifact_files = under_test.fetch_artifacts(build, revision)
         expected_file = {
             "name": artifact_mock.name,
             "link": artifact_mock.url,
@@ -185,41 +182,47 @@ class TestFetchArtifactsForPreviousCompileTask(unittest.TestCase):
     def test_fetch_artifacts_with_task_with_no_artifacts(self):
         revision = "a22"
         build_id = "project_variant_patch_a22_date"
+        build = MagicMock(id=build_id)
 
         artifacts_mock = []
 
-        task_response = MagicMock(status="success")
+        task_response = MagicMock(status="success", display_name="compile")
         task_response.artifacts = artifacts_mock
-        evergreen_api = MagicMock()
-        evergreen_api.task_by_id.return_value = task_response
+        build.get_tasks.return_value = [task_response]
 
-        artifact_files = under_test.fetch_artifacts(evergreen_api, build_id, revision)
+        artifact_files = under_test.fetch_artifacts(build, revision)
         self.assertEqual(len(artifact_files), 0)
 
     def test_fetch_artifacts_with_task_with_null_artifacts(self):
         revision = "a22"
         build_id = "project_variant_patch_a22_date"
+        build = MagicMock(id=build_id)
 
-        task_response = MagicMock(status="failure")
+        task_response = MagicMock(status="failure", display_name="compile")
         task_response.is_success.return_value = False
-        evergreen_api = MagicMock()
-        evergreen_api.task_by_id.return_value = task_response
+        build.get_tasks.return_value = [task_response]
 
         with self.assertRaises(ValueError):
-            under_test.fetch_artifacts(evergreen_api, build_id, revision)
+            under_test.fetch_artifacts(build, revision)
 
 
 class TestFindPreviousCompileTask(unittest.TestCase):
     def test_find_task(self):
-        revision = "a22"
-        build_id = "project_variant_patch_a22_date"
-        evergreen_api = MagicMock()
-        task_response = MagicMock(status="success")
-        evergreen_api.task_by_id.return_value = task_response
+        task_response = MagicMock(status="success", display_name="compile")
+        build = MagicMock()
+        build.get_tasks.return_value = [task_response]
 
-        task = under_test.find_previous_compile_task(evergreen_api, build_id, revision)
+        task = under_test.find_previous_compile_task(build)
         self.assertEqual(task, task_response)
 
+    def test_build_with_no_compile(self):
+        task_response = MagicMock(status="success", display_name="not_compile")
+        build = MagicMock()
+        build.get_tasks.return_value = [task_response]
+
+        with self.assertRaises(AssertionError):
+            under_test.find_previous_compile_task(build)
+
 
 class TestUpdateArtifactPermissions(unittest.TestCase):
     @patch(ns("os.chmod"))
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index de9971839c0..743ad8b8838 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -975,8 +975,19 @@ functions:
         set -o verbose
         set -o errexit
 
+        if [ -n "${burn_in_bypass}" ]; then
+          ${activate_virtualenv}
+          # Evergreen executable is in $HOME, so add that to the path.
+          PATH=$PATH:$HOME $python buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py            \
+            --project ${project}                                               \
+            --build-variant ${burn_in_bypass}                                  \
+            --revision ${revision}                                             \
+            --out-file bypass_compile_expansions.yml                           \
+            --version-id ${version_id}                                         \
+            --json-artifact artifacts.json
+
         # For patch builds determine if we can bypass compile.
-        if [[ "${is_patch}" = "true" && "${task_name}" = "compile" ]]; then
+        elif [[ "${is_patch}" = "true" && "${task_name}" = "compile" ]]; then
           ${activate_virtualenv}
           # Evergreen executable is in $HOME, so add that to the path.
           PATH=$PATH:$HOME $python buildscripts/bypass_compile_and_fetch_binaries.py            \