diff options
author | David Bradford <david.bradford@mongodb.com> | 2020-01-30 14:23:24 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-01-30 23:31:39 +0000 |
commit | 953a48f5a423606b848c923d0afc4760cb33d11c (patch) | |
tree | 093a7616fbbf2642819390b94e7e21e4983db6d7 | |
parent | fb51c17e8987b3250ee8a5fb98da218c4ede8296 (diff) | |
download | mongo-953a48f5a423606b848c923d0afc4760cb33d11c.tar.gz |
SERVER-45748: Ensure the correct task is used in bypass-compile for burn_in_tags
delete mode 100644 buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py
(cherry picked from commit 403d955f25f2018d7d05c405005b6043cace7d63)
delete mode 100644 buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py
5 files changed, 65 insertions, 124 deletions
diff --git a/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py index c03b1b73938..8a7e0678941 100644 --- a/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py +++ b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py @@ -17,39 +17,6 @@ LOGGER = structlog.get_logger(__name__) EVG_CONFIG_FILE = ".evergreen.yml" -def _retrieve_used_build_id(build): - """ - Determine what build_id should be used for downloading artifacts. - - If bypass_compile was used by the main compile task, then our expansions should use the - same references. - - :param build: Evergreen build containing the compile task to use. - :return: build_id that should be used for expansions. - """ - log = LOGGER.bind(build_id=build.id) - tasks = build.get_tasks() - possible_compile_tasks = {task for task in tasks if task.display_name == "compile"} - if len(possible_compile_tasks) != 1: - log.warning("Could not find 'compile' task") - raise ValueError(f"Compile task not found in {build.id}") - - compile_task = possible_compile_tasks.pop() - - # We will use the 'Binaries' artifact to see what build_id to use. - binary_artifacts = [ - artifact for artifact in compile_task.artifacts if artifact.name == "Binaries" - ] - for artifact in binary_artifacts: - log.info("Checking artifact for build_id", url=artifact.url) - build_id = artifact.url.split("/")[-1].split(".")[0] - prefix = "mongo-" - return build_id[len(prefix):] - - log.warning("Count not determine build_id") - raise ValueError(f"Could not determine build id for bypass compile in {build.id}") - - @click.command() @click.option("--project", required=True, help="The evergreen project.") @click.option("--build-variant", required=True, help="Build variant where compile is running.") @@ -82,10 +49,10 @@ def main( # pylint: disable=too-many-arguments,too-many-locals evg_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE) version = evg_api.version_by_id(version_id) - build_id = _retrieve_used_build_id(version.build_by_variant(build_variant)) + build = version.build_by_variant(build_variant) target = TargetBuild(project=project, revision=revision, build_variant=build_variant) - gather_artifacts_and_update_expansions(build_id, target, json_artifact, out_file, evg_api) + gather_artifacts_and_update_expansions(build, target, json_artifact, out_file) if __name__ == "__main__": diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py index cd636585e22..028c4b37665 100755 --- a/buildscripts/bypass_compile_and_fetch_binaries.py +++ b/buildscripts/bypass_compile_and_fetch_binaries.py @@ -14,7 +14,7 @@ import urllib.request import click -from evergreen.api import RetryingEvergreenApi +from evergreen.api import RetryingEvergreenApi, EvergreenApi, Build, Task from git.repo import Repo import requests import structlog @@ -281,7 +281,7 @@ def should_bypass_compile(patch_file, build_variant): return True -def find_build_for_previous_compile_task(evg_api, target): +def find_build_for_previous_compile_task(evg_api: EvergreenApi, target: TargetBuild) -> Build: """ Find build_id of the base revision. @@ -292,35 +292,32 @@ def find_build_for_previous_compile_task(evg_api, target): project_prefix = target.project.replace("-", "_") version_of_base_revision = "{}_{}".format(project_prefix, target.revision) version = evg_api.version_by_id(version_of_base_revision) - build_id = version.build_by_variant(target.build_variant).id - return build_id + build = version.build_by_variant(target.build_variant) + return build -def find_previous_compile_task(evg_api, build_id, revision): +def find_previous_compile_task(build: Build) -> Task: """ - Find compile task that should be used for skip compile.. + Find compile task that should be used for skip compile. - :param evg_api: Evergreen.py object. - :param build_id: Build id of the desired compile task. - :param revision: The base revision being run against. + :param build: Build containing the desired compile task. :return: Evergreen.py object containing data about the desired compile task. """ - index = build_id.find(revision) - compile_task_id = "{}compile_{}".format(build_id[:index], build_id[index:]) - task = evg_api.task_by_id(compile_task_id) - return task + tasks = [task for task in build.get_tasks() if task.display_name == "compile"] + assert len(tasks) == 1 + return tasks[0] -def fetch_artifacts(evg_api, build_id, revision): +def fetch_artifacts(build: Build, revision: str): """ Fetch artifacts from a given revision. - :param evg_api: Evergreen.py object. - :param build_id: Build id of the desired artifacts. + :param build: Build id of the desired artifacts. :param revision: The revision being fetched from. :return: Artifacts from the revision. """ - task = find_previous_compile_task(evg_api, build_id, revision) + LOGGER.info("Fetching artifacts", build_id=build.id, revision=revision) + task = find_previous_compile_task(build) if task is None or not task.is_success(): LOGGER.warning( "Could not retrieve artifacts because the compile task for base commit" @@ -330,7 +327,7 @@ def fetch_artifacts(evg_api, build_id, revision): artifacts = [] for artifact in task.artifacts: filename = os.path.basename(artifact.url) - if filename.startswith(build_id): + if filename.startswith(build.id): LOGGER.info("Retrieving archive", filename=filename) # This is the artifacts.tgz as referenced in evergreen.yml. try: @@ -395,24 +392,23 @@ def update_artifact_permissions(permission_dict): os.chmod(path, perm) -def gather_artifacts_and_update_expansions(build_id, target, json_artifact_file, expansions_file, - evg_api): +def gather_artifacts_and_update_expansions(build: Build, target: TargetBuild, json_artifact_file, + expansions_file): """ Fetch the artifacts for this build and save them to be used by other tasks. - :param build_id: build_id of build with artifacts. + :param build: build containing artifacts. :param target: Target build being bypassed. :param json_artifact_file: File to write json artifacts to. :param expansions_file: Files to write expansions to. - :param evg_api: Evergreen Api. """ - artifacts = fetch_artifacts(evg_api, build_id, target.revision) + artifacts = fetch_artifacts(build, target.revision) update_artifact_permissions(ARTIFACTS_NEEDING_PERMISSIONS) write_out_artifacts(json_artifact_file, artifacts) - LOGGER.info("Creating expansions files", target=target, build_id=build_id) + LOGGER.info("Creating expansions files", target=target, build_id=build.id) - expansions = generate_bypass_expansions(target, build_id) + expansions = generate_bypass_expansions(target, build.id) write_out_bypass_compile_expansions(expansions_file, **expansions) @@ -452,13 +448,13 @@ def main( # pylint: disable=too-many-arguments,too-many-locals,too-many-stateme # Determine if we should bypass compile based on modified patch files. if should_bypass_compile(patch_file, build_variant): evg_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE) - build_id = find_build_for_previous_compile_task(evg_api, target) - if not build_id: + build = find_build_for_previous_compile_task(evg_api, target) + if not build: LOGGER.warning("Could not find build id. Default compile bypass to false.", revision=revision, project=project) return - gather_artifacts_and_update_expansions(build_id, target, json_artifact, out_file, evg_api) + gather_artifacts_and_update_expansions(build, target, json_artifact, out_file) if __name__ == "__main__": diff --git a/buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py b/buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py deleted file mode 100644 index a4e3225d7f8..00000000000 --- a/buildscripts/tests/test_burn_in_tags_bypass_compile_and_fetch_binaries.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Unit tests for burn_in_tags_bypass_compile_and_fetch_binaries.""" - -import unittest -from unittest.mock import MagicMock - -import buildscripts.burn_in_tags_bypass_compile_and_fetch_binaries as under_test - -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access - - -class TestRetrieveUsedBuildId(unittest.TestCase): - def test_build_with_no_compile_throws_exception(self): - build_mock = MagicMock() - - with self.assertRaises(ValueError): - under_test._retrieve_used_build_id(build_mock) - - def test_compile_with_no_binaries_artifact_throws_exception(self): - build_mock = MagicMock() - compile_task = MagicMock(display_name="compile") - build_mock.get_tasks.return_value = [compile_task] - - with self.assertRaises(ValueError): - under_test._retrieve_used_build_id(build_mock) - - def test_build_id_from_compile_binaries_is_used(self): - build_id = "this_is_the_build_id" - url = f"http://s3.amazon.com/mciuploads/mongodb/build_var//binaries/mongo-{build_id}.tgz" - build_mock = MagicMock() - compile_task = MagicMock(display_name="compile") - build_mock.get_tasks.return_value = [MagicMock(), compile_task, MagicMock()] - artifact_mock = MagicMock(url=url) - artifact_mock.name = "Binaries" - compile_task.artifacts = [MagicMock(), artifact_mock, MagicMock()] - - self.assertEqual(build_id, under_test._retrieve_used_build_id(build_mock)) diff --git a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py index 1a2979e6e64..e6ff1016dd1 100644 --- a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py +++ b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py @@ -149,32 +149,29 @@ pytests/test2.py class TestFindBuildForPreviousCompileTask(unittest.TestCase): def test_find_build(self): target = under_test.TargetBuild(project="project", revision="a22", build_variant="variant") - expected_build_id = "project_variant_patch_a22_date" evergreen_api = MagicMock() - version_response = MagicMock() - evergreen_api.version_by_id.return_value = version_response - version_response.build_by_variant.return_value = MagicMock(id=expected_build_id) + version_response = evergreen_api.version_by_id.return_value - build_id = under_test.find_build_for_previous_compile_task(evergreen_api, target) - self.assertEqual(build_id, expected_build_id) + build = under_test.find_build_for_previous_compile_task(evergreen_api, target) + self.assertEqual(build, version_response.build_by_variant.return_value) class TestFetchArtifactsForPreviousCompileTask(unittest.TestCase): def test_fetch_artifacts_with_task_with_artifact(self): revision = "a22" build_id = "project_variant_patch_a22_date" + build = MagicMock(id=build_id) artifact_mock = MagicMock() artifact_mock.name = "Binaries" artifact_mock.url = "http://s3.amazon.com/mciuploads/mongodb/build_var//binaries/mongo-test.tgz" artifacts_mock = [artifact_mock] - task_response = MagicMock(status="success") + task_response = MagicMock(status="success", display_name="compile") task_response.artifacts = artifacts_mock - evergreen_api = MagicMock() - evergreen_api.task_by_id.return_value = task_response + build.get_tasks.return_value = [task_response] - artifact_files = under_test.fetch_artifacts(evergreen_api, build_id, revision) + artifact_files = under_test.fetch_artifacts(build, revision) expected_file = { "name": artifact_mock.name, "link": artifact_mock.url, @@ -185,41 +182,47 @@ class TestFetchArtifactsForPreviousCompileTask(unittest.TestCase): def test_fetch_artifacts_with_task_with_no_artifacts(self): revision = "a22" build_id = "project_variant_patch_a22_date" + build = MagicMock(id=build_id) artifacts_mock = [] - task_response = MagicMock(status="success") + task_response = MagicMock(status="success", display_name="compile") task_response.artifacts = artifacts_mock - evergreen_api = MagicMock() - evergreen_api.task_by_id.return_value = task_response + build.get_tasks.return_value = [task_response] - artifact_files = under_test.fetch_artifacts(evergreen_api, build_id, revision) + artifact_files = under_test.fetch_artifacts(build, revision) self.assertEqual(len(artifact_files), 0) def test_fetch_artifacts_with_task_with_null_artifacts(self): revision = "a22" build_id = "project_variant_patch_a22_date" + build = MagicMock(id=build_id) - task_response = MagicMock(status="failure") + task_response = MagicMock(status="failure", display_name="compile") task_response.is_success.return_value = False - evergreen_api = MagicMock() - evergreen_api.task_by_id.return_value = task_response + build.get_tasks.return_value = [task_response] with self.assertRaises(ValueError): - under_test.fetch_artifacts(evergreen_api, build_id, revision) + under_test.fetch_artifacts(build, revision) class TestFindPreviousCompileTask(unittest.TestCase): def test_find_task(self): - revision = "a22" - build_id = "project_variant_patch_a22_date" - evergreen_api = MagicMock() - task_response = MagicMock(status="success") - evergreen_api.task_by_id.return_value = task_response + task_response = MagicMock(status="success", display_name="compile") + build = MagicMock() + build.get_tasks.return_value = [task_response] - task = under_test.find_previous_compile_task(evergreen_api, build_id, revision) + task = under_test.find_previous_compile_task(build) self.assertEqual(task, task_response) + def test_build_with_no_compile(self): + task_response = MagicMock(status="success", display_name="not_compile") + build = MagicMock() + build.get_tasks.return_value = [task_response] + + with self.assertRaises(AssertionError): + under_test.find_previous_compile_task(build) + class TestUpdateArtifactPermissions(unittest.TestCase): @patch(ns("os.chmod")) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index de9971839c0..743ad8b8838 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -975,8 +975,19 @@ functions: set -o verbose set -o errexit + if [ -n "${burn_in_bypass}" ]; then + ${activate_virtualenv} + # Evergreen executable is in $HOME, so add that to the path. + PATH=$PATH:$HOME $python buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py \ + --project ${project} \ + --build-variant ${burn_in_bypass} \ + --revision ${revision} \ + --out-file bypass_compile_expansions.yml \ + --version-id ${version_id} \ + --json-artifact artifacts.json + # For patch builds determine if we can bypass compile. - if [[ "${is_patch}" = "true" && "${task_name}" = "compile" ]]; then + elif [[ "${is_patch}" = "true" && "${task_name}" = "compile" ]]; then ${activate_virtualenv} # Evergreen executable is in $HOME, so add that to the path. PATH=$PATH:$HOME $python buildscripts/bypass_compile_and_fetch_binaries.py \ |