diff options
-rw-r--r-- | buildscripts/burn_in_tags.py | 33 | ||||
-rw-r--r-- | buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py | 35 | ||||
-rwxr-xr-x | buildscripts/bypass_compile_and_fetch_binaries.py | 34 | ||||
-rw-r--r-- | buildscripts/tests/test_burn_in_tags.py | 32 | ||||
-rw-r--r-- | buildscripts/tests/test_burn_in_tags_evergreen.yml | 44 | ||||
-rw-r--r-- | buildscripts/tests/test_bypass_compile_and_fetch_binaries.py | 26 | ||||
-rw-r--r-- | etc/evergreen.yml | 16 |
7 files changed, 162 insertions, 58 deletions
diff --git a/buildscripts/burn_in_tags.py b/buildscripts/burn_in_tags.py index 9b595ecd624..c1d5bd0d6b2 100644 --- a/buildscripts/burn_in_tags.py +++ b/buildscripts/burn_in_tags.py @@ -76,22 +76,28 @@ def _create_evg_buildvariant_map(expansions_file_data): :param expansions_file_data: Config data file to use. :return: Map of base buildvariants to their generated buildvariants. """ - buildvariant_map = {} - base_variants = expansions_file_data["base_variants"].split( - " ") if expansions_file_data["base_variants"] else [] - for base_variant in base_variants: - new_variant_name = f"{base_variant}-required" - buildvariant_map[base_variant] = new_variant_name - return buildvariant_map - - -def _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant): + burn_in_tags_gen_variant = expansions_file_data["build_variant"] + evergreen_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE) + burn_in_tags_gen_variant_config = evergreen_conf.get_variant(burn_in_tags_gen_variant) + burn_in_tag_buildvariants = burn_in_tags_gen_variant_config.expansions.get( + "burn_in_tag_buildvariants") + if burn_in_tag_buildvariants: + return { + base_variant: f"{base_variant}-required" + for base_variant in burn_in_tag_buildvariants.split(" ") + } + return {} + + +def _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant, + burn_in_tags_gen_variant): """ Generate buildvariants for a given shrub config. :param shrub_config: Shrub config object that the generated buildvariant will be built upon. :param buildvariant: The base variant that the generated run_buildvariant will be based on. :param run_buildvariant: The generated buildvariant. + :param burn_in_tags_gen_variant: The buildvariant on which the burn_in_tags_gen task runs. """ evergreen_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE) base_variant_config = evergreen_conf.get_variant(buildvariant) @@ -100,9 +106,9 @@ def _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant): new_variant_run_on = base_variant_config.run_on[0] task_spec = TaskSpec("compile_TG") - task_spec.distro("rhel62-large") - new_variant = shrub_config.variant(run_buildvariant) + new_variant = shrub_config.variant(run_buildvariant).expansion("burn_in_bypass", + burn_in_tags_gen_variant) new_variant.display_name(new_variant_display_name) new_variant.run_on(new_variant_run_on) new_variant.task(task_spec) @@ -127,7 +133,8 @@ def _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build config_options = _get_config_options(expansions_file_data, buildvariant, run_buildvariant) tests_by_task = create_tests_by_task(config_options, evergreen_api) if tests_by_task: - _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant) + _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant, + expansions_file_data["build_variant"]) create_generate_tasks_config(evergreen_api, shrub_config, config_options, tests_by_task, False) diff --git a/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py new file mode 100644 index 00000000000..eb92ddd36c9 --- /dev/null +++ b/buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Bypass compile and fetch binaries for burn_in_tags.""" + +from urllib.parse import urlparse +from buildscripts.bypass_compile_and_fetch_binaries import ( + find_suitable_build_id, generate_bypass_expansions, parse_args, read_evg_config, + requests_get_json, write_out_bypass_compile_expansions) + + +def main(): # pylint: disable=too-many-locals,too-many-statements + """Execute Main program.""" + + args = parse_args() + evg_config = read_evg_config() + if evg_config is None: + print("Could not find ~/.evergreen.yml config file. Default compile bypass to false.") + return + + api_server = "{url.scheme}://{url.netloc}".format( + url=urlparse(evg_config.get("api_server_host"))) + revision_url = f"{api_server}/rest/v1/projects/{args.project}/revisions/{args.revision}" + revisions = requests_get_json(revision_url) + build_id = find_suitable_build_id(revisions["builds"], args) + if not build_id: + print("Could not find build id for revision {args.revision} on project {args.project}." + " Default compile bypass to false.") + return + + expansions = generate_bypass_expansions(args.project, args.buildVariant, args.revision, + build_id) + write_out_bypass_compile_expansions(args.outFile, **expansions) + + +if __name__ == "__main__": + main() diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py index f1a659710b4..94e20710bb8 100755 --- a/buildscripts/bypass_compile_and_fetch_binaries.py +++ b/buildscripts/bypass_compile_and_fetch_binaries.py @@ -302,6 +302,25 @@ def parse_args(): return parser.parse_args() +def find_suitable_build_id(builds, args): + """ + Find a build_id that fits the given parameters. + + :param builds: List of builds. + :param args: The parameters a build must meet, including project, buildVariant, and revision. + :return: Build_id that matches the parameters. + """ + prefix = "{}_{}_{}_".format(args.project, args.buildVariant, args.revision) + # The "project" and "buildVariant" passed in may contain "-", but the "builds" listed from + # Evergreen only contain "_". Replace the hyphens before searching for the build. + prefix = prefix.replace("-", "_") + build_id_pattern = re.compile(prefix) + for build_id in builds: + if build_id_pattern.search(build_id): + return build_id + return None + + def main(): # pylint: disable=too-many-locals,too-many-statements """Execute Main entry. @@ -329,19 +348,8 @@ def main(): # pylint: disable=too-many-locals,too-many-statements revision_url = "{}/rest/v1/projects/{}/revisions/{}".format(api_server, args.project, args.revision) revisions = requests_get_json(revision_url) - - prefix = "{}_{}_{}_".format(args.project, args.buildVariant, args.revision) - # The "project" and "buildVariant" passed in may contain "-", but the "builds" listed from - # Evergreen only contain "_". Replace the hyphens before searching for the build. - prefix = prefix.replace("-", "_") - build_id_pattern = re.compile(prefix) - build_id = None - for build_id in revisions["builds"]: - # Find a suitable build_id - match = build_id_pattern.search(build_id) - if match: - break - else: + build_id = find_suitable_build_id(revisions["builds"], args) + if not build_id: print("Could not find build id for revision {} on project {}." " Default compile bypass to false.".format(args.revision, args.project)) return diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py index b036370fdf9..5a9c72ca602 100644 --- a/buildscripts/tests/test_burn_in_tags.py +++ b/buildscripts/tests/test_burn_in_tags.py @@ -44,11 +44,10 @@ def get_evergreen_config(): class TestCreateEvgBuildVariantMap(unittest.TestCase): - def test_create_evg_buildvariant_map(self): - expansions_file_data = { - "base_variants": - "enterprise-rhel-62-64-bit-majority-read-concern-off enterprise-rhel-62-64-bit-inmem" - } + @mock.patch(ns("evergreen")) + def test_create_evg_buildvariant_map(self, evergreen_mock): + evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config() + expansions_file_data = {"build_variant": "enterprise-rhel-62-64-bit"} buildvariant_map = burn_in_tags._create_evg_buildvariant_map(expansions_file_data) expected_buildvariant_map = { "enterprise-rhel-62-64-bit-majority-read-concern-off": @@ -58,8 +57,10 @@ class TestCreateEvgBuildVariantMap(unittest.TestCase): } self.assertEqual(buildvariant_map, expected_buildvariant_map) - def test_create_evg_buildvariant_map_no_base_variants(self): - expansions_file_data = {"base_variants": ""} + @mock.patch(ns("evergreen")) + def test_create_evg_buildvariant_map_no_base_variants(self, evergreen_mock): + evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config() + expansions_file_data = {"build_variant": "buildvariant-without-burn-in-tag-buildvariants"} buildvariant_map = burn_in_tags._create_evg_buildvariant_map(expansions_file_data) self.assertEqual(buildvariant_map, {}) @@ -69,17 +70,22 @@ class TestGenerateEvgBuildVariants(unittest.TestCase): def test_generate_evg_buildvariant_one_base_variant(self, evergreen_mock): evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config() base_variant = "enterprise-rhel-62-64-bit-inmem" - expected_variant_data = get_evergreen_config().get_variant(base_variant) - + generated_variant = "enterprise-rhel-62-64-bit-inmem-required" + burn_in_tags_gen_variant = "enterprise-rhel-62-64-bit" shrub_config = Configuration() - burn_in_tags._generate_evg_buildvariant(shrub_config, base_variant, - "enterprise-rhel-62-64-bit-inmem-required") + burn_in_tags._generate_evg_buildvariant(shrub_config, base_variant, generated_variant, + burn_in_tags_gen_variant) + + expected_variant_data = get_evergreen_config().get_variant(base_variant) generated_buildvariants = shrub_config.to_map()["buildvariants"] self.assertEqual(len(generated_buildvariants), 1) generated_build_variant = generated_buildvariants[0] - self.assertEqual(generated_build_variant["name"], f"{expected_variant_data}-required") + self.assertEqual(generated_build_variant["name"], generated_variant) self.assertEqual(generated_build_variant["modules"], expected_variant_data.modules) - self.assertEqual(generated_build_variant["expansions"], expected_variant_data.expansions) + generated_expansions = generated_build_variant["expansions"] + burn_in_bypass_expansion_value = generated_expansions.pop("burn_in_bypass") + self.assertEqual(burn_in_bypass_expansion_value, burn_in_tags_gen_variant) + self.assertEqual(generated_expansions, expected_variant_data.expansions) class TestGenerateEvgTasks(unittest.TestCase): diff --git a/buildscripts/tests/test_burn_in_tags_evergreen.yml b/buildscripts/tests/test_burn_in_tags_evergreen.yml index 165cb669345..815906704f9 100644 --- a/buildscripts/tests/test_burn_in_tags_evergreen.yml +++ b/buildscripts/tests/test_burn_in_tags_evergreen.yml @@ -15,14 +15,40 @@ tasks: depends_on: [] commands: - func: "fetch source" +- name: burn_in_tags_gen + depends_on: [] + commands: + - func: "fake command" buildvariants: +- name: enterprise-rhel-62-64-bit + display_name: "! Enterprise RHEL 6.2" + expansions: + multiversion_platform: rhel62 + burn_in_tag_buildvariants: enterprise-rhel-62-64-bit-majority-read-concern-off enterprise-rhel-62-64-bit-inmem + tasks: + - name: burn_in_tags_gen +- name: buildvariant-without-burn-in-tag-buildvariants + display_name: "Buildvariant without burn in tag buildvariants expansion" + expansions: + multiversion_platform: rhel62 + tasks: + - name: burn_in_tags_gen - name: enterprise-rhel-62-64-bit-majority-read-concern-off display_name: "Enterprise RHEL 6.2 (majority read concern off)" modules: ["enterprise"] run_on: - rhel62-small expansions: &enterprise-rhel-62-64-bit-majority-read-concern-off-expansions + multiversion_edition: enterprise + tasks: + - name: compile +- name: enterprise-rhel-62-64-bit-inmem + display_name: Enterprise RHEL 6.2 (inMemory) + modules: ["enterprise"] + run_on: + - rhel62-small + expansions: &enterprise-rhel-62-64-bit-inmem-expansions test_flags: >- --majorityReadConcern=off --excludeWithAnyTags=requires_majority_read_concern,uses_prepare_transaction,uses_multi_shard_transaction,uses_atclustertime @@ -32,24 +58,6 @@ buildvariants: --release --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars MONGO_DISTMOD=rhel62 - multiversion_platform: rhel62 - multiversion_edition: enterprise - repo_edition: enterprise - scons_cache_scope: shared - tooltags: "ssl sasl gssapi" - large_distro_name: rhel62-large - tasks: - - name: compile -- name: enterprise-rhel-62-64-bit-inmem - display_name: Enterprise RHEL 6.2 (inMemory) - modules: - - enterprise - run_on: - - rhel62-small - batchtime: 1440 # 1 day - expansions: &enterprise-rhel-62-64-bit-inmem-expansions - test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling - compile_flags: --ssl MONGO_DISTMOD=rhel62 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars multiversion_platform: rhel62 multiversion_edition: enterprise scons_cache_scope: shared diff --git a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py index 1adde7bc239..b93c7440504 100644 --- a/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py +++ b/buildscripts/tests/test_bypass_compile_and_fetch_binaries.py @@ -2,6 +2,7 @@ import unittest +from collections import namedtuple from mock import mock_open, patch, MagicMock import buildscripts.bypass_compile_and_fetch_binaries as under_test @@ -138,3 +139,28 @@ pytests/test2.py open_mock.return_value.__enter__.return_value = git_changes.splitlines() self.assertFalse(under_test.should_bypass_compile(MagicMock())) + + +def generate_args(): + project = "project-master" + build_variant = "rhel-62-64-bit" + revision = "fakesha" + Args = namedtuple("Args", ["project", "buildVariant", "revision"]) + return Args(project=project, buildVariant=build_variant, revision=revision) + + +class TestFindSuitableBuildID(unittest.TestCase): + def test_builds(self): + expected_build_id = "project_master_rhel_62_64_bit_fakesha_19_07_10_15_48_53" + builds = ["build1", "build2", expected_build_id] + args = generate_args() + build_id = under_test.find_suitable_build_id(builds, args) + + self.assertEqual(build_id, expected_build_id) + + def test_no_builds(self): + builds = [] + args = generate_args() + build_id = under_test.find_suitable_build_id(builds, args) + + self.assertEqual(build_id, None) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index fbb2c55f3cf..197c2ee5b0a 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1153,6 +1153,18 @@ functions: set -o verbose set -o errexit + if [ -n "${burn_in_bypass}" ]; then + ${activate_virtualenv} + # Evergreen executable is in $HOME, so add that to the path. + PATH=$PATH:$HOME $python buildscripts/burn_in_tags_bypass_compile_and_fetch_binaries.py \ + --project ${project} \ + --buildVariant ${burn_in_bypass} \ + --revision ${revision} \ + --patchFile patch_files.txt \ + --outFile bypass_compile_expansions.yml \ + --jsonArtifact artifacts.json + fi + # For patch builds determine if we can bypass compile. if [[ "${is_patch}" = "true" && "${task_name}" = "compile" ]]; then ${activate_virtualenv} @@ -5714,6 +5726,8 @@ tasks: fallback_num_sub_suites: 4 - name: burn_in_tags_gen + depends_on: + - name: compile commands: - command: manifest.load - func: "git get project" @@ -5727,7 +5741,6 @@ tasks: repeat_tests_secs: 600 repeat_tests_min: 2 repeat_tests_max: 1000 - base_variants: enterprise-rhel-62-64-bit-majority-read-concern-off enterprise-rhel-62-64-bit-inmem linux-64-duroff - name: auth_audit_gen tags: ["auth", "audit"] @@ -10231,6 +10244,7 @@ buildvariants: jstestfuzz_concurrent_num_files: 10 target_resmoke_time: 10 large_distro_name: rhel62-large + burn_in_tag_buildvariants: enterprise-rhel-62-64-bit-majority-read-concern-off enterprise-rhel-62-64-bit-inmem linux-64-duroff display_tasks: - *dbtest - *unittests |