SERVER-42696 Executables are missing from artifacts on generated build variants in burn_in_tagst

4 files changed, 148 insertions, 79 deletions

diff --git a/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py
index 59005aadd83..e9fdda4e8b0 100644
--- a/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py
@@ -1,15 +1,16 @@
 #!/usr/bin/env python3
 """Bypass compile and fetch binaries for burn_in_tags."""
+
 import logging
 import sys
 
 import click
-from evergreen.api import RetryingEvergreenApi
 import structlog
 from structlog.stdlib import LoggerFactory
 
-from buildscripts.bypass_compile_and_fetch_binaries import generate_bypass_expansions, \
-    write_out_bypass_compile_expansions
+from evergreen.api import RetryingEvergreenApi
+from buildscripts.bypass_compile_and_fetch_binaries import fetch_artifacts, \
+    generate_bypass_expansions, write_out_bypass_compile_expansions, write_out_artifacts
 
 structlog.configure(logger_factory=LoggerFactory())
 LOGGER = structlog.get_logger(__name__)
@@ -56,7 +57,10 @@ def _retrieve_used_build_id(build):
 @click.option("--revision", required=True, help="Base revision of the build.")
 @click.option("--out-file", required=True, help="File to write macros expansions to.")
 @click.option("--version-id", required=True, help="Evergreen version id of the current build.")
-def main(project, build_variant, revision, out_file, version_id):
+@click.option("--json-artifact", required=True,
+              help="The JSON file to write out the metadata of files to attach to task.")
+def main(  # pylint: disable=too-many-arguments,too-many-locals
+        project, build_variant, revision, out_file, version_id, json_artifact):
     """
     Create a file with expansions that can be used to bypass compile.
 
@@ -80,6 +84,9 @@ def main(project, build_variant, revision, out_file, version_id):
 
     version = evg_api.version_by_id(version_id)
     build_id = _retrieve_used_build_id(version.build_by_variant(build_variant))
+    artifacts = fetch_artifacts(evg_api, build_id, revision)
+
+    write_out_artifacts(json_artifact, artifacts)
 
     LOGGER.info("Creating expansions files", project=project, build_variant=build_variant,
                 revision=revision, build_id=build_id)
diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py
index 7ba76f6457a..cc322a88a5b 100755
--- a/buildscripts/bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/bypass_compile_and_fetch_binaries.py
@@ -268,11 +268,11 @@ def should_bypass_compile(patch_file, build_variant):
     return True
 
 
-def find_build_for_previous_compile_task(evergreen_api, revision, project, build_variant):
+def find_build_for_previous_compile_task(evg_api, revision, project, build_variant):
     """
     Find build_id of the base revision.
 
-    :param evergreen_api: Evergreen.py object.
+    :param evg_api: Evergreen.py object.
     :param revision: The base revision being run against.
     :param project: The evergreen project.
     :param build_variant: The build variant whose artifacts we want to use.
@@ -280,26 +280,102 @@ def find_build_for_previous_compile_task(evergreen_api, revision, project, build
     """
     project_prefix = project.replace("-", "_")
     version_of_base_revision = "{}_{}".format(project_prefix, revision)
-    version = evergreen_api.version_by_id(version_of_base_revision)
+    version = evg_api.version_by_id(version_of_base_revision)
     build_id = version.build_by_variant(build_variant).id
     return build_id
 
 
-def find_previous_compile_task(evergreen_api, build_id, revision):
+def find_previous_compile_task(evg_api, build_id, revision):
     """
     Find compile task that should be used for skip compile..
 
-    :param evergreen_api: Evergreen.py object.
+    :param evg_api: Evergreen.py object.
     :param build_id: Build id of the desired compile task.
     :param revision: The base revision being run against.
     :return: Evergreen.py object containing data about the desired compile task.
     """
     index = build_id.find(revision)
     compile_task_id = "{}compile_{}".format(build_id[:index], build_id[index:])
-    task = evergreen_api.task_by_id(compile_task_id)
+    task = evg_api.task_by_id(compile_task_id)
     return task
 
 
+def fetch_artifacts(evg_api, build_id, revision):
+    """
+    Fetch artifacts from a given revision.
+
+    :param evg_api: Evergreen.py object.
+    :param build_id: Build id of the desired artifacts.
+    :param revision: The revision being fetched from.
+    :return: Artifacts from the revision.
+    """
+    task = find_previous_compile_task(evg_api, build_id, revision)
+    if task is None or not task.is_success():
+        LOGGER.warning(
+            "Could not retrieve artifacts because the compile task for base commit"
+            " was not available. Default compile bypass to false.", task_id=task.task_id)
+        raise ValueError("No artifacts were found for the current task")
+    LOGGER.info("Fetching pre-existing artifacts from compile task", task_id=task.task_id)
+    artifacts = []
+    for artifact in task.artifacts:
+        filename = os.path.basename(artifact.url)
+        if filename.startswith(build_id):
+            LOGGER.info("Retrieving archive", filename=filename)
+            # This is the artifacts.tgz as referenced in evergreen.yml.
+            try:
+                urllib.request.urlretrieve(artifact.url, filename)
+            except urllib.error.ContentTooShortError:
+                LOGGER.warning(
+                    "The artifact could not be completely downloaded. Default"
+                    " compile bypass to false.", filename=filename)
+                raise ValueError("No artifacts were found for the current task")
+            # Need to extract certain files from the pre-existing artifacts.tgz.
+            extract_files = [
+                executable_name("mongobridge"),
+                executable_name("mongoebench"),
+                executable_name("mongoed"),
+                executable_name("mongotmock"),
+                executable_name("wt"),
+            ]
+            with tarfile.open(filename, "r:gz") as tar:
+                # The repo/ directory contains files needed by the package task. May
+                # need to add other files that would otherwise be generated by SCons
+                # if we did not bypass compile.
+                subdir = [
+                    tarinfo for tarinfo in tar.getmembers()
+                    if tarinfo.name.startswith("repo/") or tarinfo.name in extract_files
+                ]
+                LOGGER.info("Extracting the files...", filename=filename,
+                            files="\n".join(tarinfo.name for tarinfo in subdir))
+                tar.extractall(members=subdir)
+        elif filename.startswith("mongo-src"):
+            LOGGER.info("Retrieving mongo source", filename=filename)
+            # This is the distsrc.[tgz|zip] as referenced in evergreen.yml.
+            try:
+                urllib.request.urlretrieve(artifact.url, filename)
+            except urllib.error.ContentTooShortError:
+                LOGGER.warn(
+                    "The artifact could not be completely downloaded. Default"
+                    " compile bypass to false.", filename=filename)
+                raise ValueError("No artifacts were found for the current task")
+            extension = os.path.splitext(filename)[1]
+            distsrc_filename = "distsrc{}".format(extension)
+            LOGGER.info("Renaming", filename=filename, rename=distsrc_filename)
+            os.rename(filename, distsrc_filename)
+        else:
+            LOGGER.info("Linking base artifact to this patch build", filename=filename)
+            # For other artifacts we just add their URLs to the JSON file to upload.
+            files = {
+                "name": artifact.name,
+                "link": artifact.url,
+                "visibility": "private",
+            }
+            # Check the link exists, else raise an exception. Compile bypass is disabled.
+            requests.head(artifact.url).raise_for_status()
+            artifacts.append(files)
+    return artifacts
+
+
 @click.command()
 @click.option("--project", required=True, help="The evergreen project.")
 @click.option("--build-variant", required=True,
@@ -333,78 +409,13 @@ def main(  # pylint: disable=too-many-arguments,too-many-locals,too-many-stateme
 
     # Determine if we should bypass compile based on modified patch files.
     if should_bypass_compile(patch_file, build_variant):
-        evergreen_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE)
-        build_id = find_build_for_previous_compile_task(evergreen_api, revision, project,
-                                                        build_variant)
+        evg_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE)
+        build_id = find_build_for_previous_compile_task(evg_api, revision, project, build_variant)
         if not build_id:
             LOGGER.warning("Could not find build id. Default compile bypass to false.",
                            revision=revision, project=project)
             return
-        task = find_previous_compile_task(evergreen_api, build_id, revision)
-        if task is None or not task.is_success():
-            LOGGER.warning(
-                "Could not retrieve artifacts because the compile task for base commit"
-                " was not available. Default compile bypass to false.", task_id=task.task_id)
-            return
-        LOGGER.info("Fetching pre-existing artifacts from compile task", task_id=task.task_id)
-        artifacts = []
-        for artifact in task.artifacts:
-            filename = os.path.basename(artifact.url)
-            if filename.startswith(build_id):
-                LOGGER.info("Retrieving archive", filename=filename)
-                # This is the artifacts.tgz as referenced in evergreen.yml.
-                try:
-                    urllib.request.urlretrieve(artifact.url, filename)
-                except urllib.error.ContentTooShortError:
-                    LOGGER.warning(
-                        "The artifact could not be completely downloaded. Default"
-                        " compile bypass to false.", filename=filename)
-                    return
-
-                # Need to extract certain files from the pre-existing artifacts.tgz.
-                extract_files = [
-                    executable_name("mongobridge"),
-                    executable_name("mongoebench"),
-                    executable_name("mongoed"),
-                    executable_name("mongotmock"),
-                    executable_name("wt"),
-                ]
-                with tarfile.open(filename, "r:gz") as tar:
-                    # The repo/ directory contains files needed by the package task. May
-                    # need to add other files that would otherwise be generated by SCons
-                    # if we did not bypass compile.
-                    subdir = [
-                        tarinfo for tarinfo in tar.getmembers()
-                        if tarinfo.name.startswith("repo/") or tarinfo.name in extract_files
-                    ]
-                    LOGGER.info("Extracting the files...", filename=filename,
-                                files="\n".join(tarinfo.name for tarinfo in subdir))
-                    tar.extractall(members=subdir)
-            elif filename.startswith("mongo-src"):
-                LOGGER.info("Retrieving mongo source", filename=filename)
-                # This is the distsrc.[tgz|zip] as referenced in evergreen.yml.
-                try:
-                    urllib.request.urlretrieve(artifact.url, filename)
-                except urllib.error.ContentTooShortError:
-                    LOGGER.warn(
-                        "The artifact could not be completely downloaded. Default"
-                        " compile bypass to false.", filename=filename)
-                    return
-                extension = os.path.splitext(filename)[1]
-                distsrc_filename = "distsrc{}".format(extension)
-                LOGGER.info("Renaming", filename=filename, rename=distsrc_filename)
-                os.rename(filename, distsrc_filename)
-            else:
-                LOGGER.info("Linking base artifact to this patch build", filename=filename)
-                # For other artifacts we just add their URLs to the JSON file to upload.
-                files = {
-                    "name": artifact.name,
-                    "link": artifact.url,
-                    "visibility": "private",
-                }
-                # Check the link exists, else raise an exception. Compile bypass is disabled.
-                requests.head(artifact.url).raise_for_status()
-                artifacts.append(files)
+        artifacts = fetch_artifacts(evg_api, build_id, revision)
 
         # SERVER-21492 related issue where without running scons the jstests/libs/key1
         # and key2 files are not chmod to 0600. Need to change permissions here since we
diff --git a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
index 70088171eae..d137ced12a2 100644
--- a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py
@@ -162,6 +162,56 @@ class TestFindBuildForPreviousCompileTask(unittest.TestCase):
         self.assertEqual(build_id, expected_build_id)
 
 
+class TestFetchArtifactsForPreviousCompileTask(unittest.TestCase):
+    def test_fetch_artifacts_with_task_with_artifact(self):
+        revision = "a22"
+        build_id = "project_variant_patch_a22_date"
+
+        artifact_mock = MagicMock()
+        artifact_mock.name = "Binaries"
+        artifact_mock.url = "http://s3.amazon.com/mciuploads/mongodb/build_var//binaries/mongo-test.tgz"
+        artifacts_mock = [artifact_mock]
+
+        task_response = MagicMock(status="success")
+        task_response.artifacts = artifacts_mock
+        evergreen_api = MagicMock()
+        evergreen_api.task_by_id.return_value = task_response
+
+        artifact_files = under_test.fetch_artifacts(evergreen_api, build_id, revision)
+        expected_file = {
+            "name": artifact_mock.name,
+            "link": artifact_mock.url,
+            "visibility": "private",
+        }
+        self.assertIn(expected_file, artifact_files)
+
+    def test_fetch_artifacts_with_task_with_no_artifacts(self):
+        revision = "a22"
+        build_id = "project_variant_patch_a22_date"
+
+        artifacts_mock = []
+
+        task_response = MagicMock(status="success")
+        task_response.artifacts = artifacts_mock
+        evergreen_api = MagicMock()
+        evergreen_api.task_by_id.return_value = task_response
+
+        artifact_files = under_test.fetch_artifacts(evergreen_api, build_id, revision)
+        self.assertEqual(len(artifact_files), 0)
+
+    def test_fetch_artifacts_with_task_with_null_artifacts(self):
+        revision = "a22"
+        build_id = "project_variant_patch_a22_date"
+
+        task_response = MagicMock(status="failure")
+        task_response.is_success.return_value = False
+        evergreen_api = MagicMock()
+        evergreen_api.task_by_id.return_value = task_response
+
+        with self.assertRaises(ValueError):
+            under_test.fetch_artifacts(evergreen_api, build_id, revision)
+
+
 class TestFindPreviousCompileTask(unittest.TestCase):
     def test_find_task(self):
         revision = "a22"
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 17306460cdc..5266b411c4d 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -1201,7 +1201,8 @@ functions:
             --build-variant ${burn_in_bypass}                                  \
             --revision ${revision}                                             \
             --out-file bypass_compile_expansions.yml                           \
-            --version-id ${version_id}
+            --version-id ${version_id}                                         \
+            --json-artifact artifacts.json
         fi
 
         # For patch builds determine if we can bypass compile.