diff options
author | XueruiFa <xuerui.fa@mongodb.com> | 2021-09-02 20:00:17 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-09-20 16:20:44 +0000 |
commit | efdac6418b6cd6012839c3e87a7b6dc4893d6ef0 (patch) | |
tree | 555f16b81f98eff696aa7d1640ba925a758e5074 | |
parent | d14837390af247b23cb6ca70908280107dccf986 (diff) | |
download | mongo-efdac6418b6cd6012839c3e87a7b6dc4893d6ef0.tar.gz |
SERVER-59012: Calculate FCV constants in resmoke
26 files changed, 325 insertions, 69 deletions
diff --git a/.gitignore b/.gitignore index 21394b3ab3c..2434c53fb6f 100644 --- a/.gitignore +++ b/.gitignore @@ -202,8 +202,10 @@ resmoke.ini # UndoDB Recordings *.undo -# Resmoke runtime trackers +# Resmoke runtime configuration and trackers. .resmoke_start_time.yml +.resmoke_mongo_version.yml +.resmoke_mongo_release_values.yml # libfuzzer artifacts default.profraw diff --git a/buildscripts/resmokeconfig/matrix_suites/tags.py b/buildscripts/resmokeconfig/matrix_suites/tags.py index d7b30cec275..571da6b638e 100644 --- a/buildscripts/resmokeconfig/matrix_suites/tags.py +++ b/buildscripts/resmokeconfig/matrix_suites/tags.py @@ -1,5 +1,4 @@ """Dynamically Generated tags.""" -from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG # TODO SERVER-55857: Let this file be the single source of truth for tags. Remove dupe definitions. @@ -10,6 +9,7 @@ class Tags(object): BACKPORT_REQUIRED_TAG = "backport_required_multiversion" + from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG # Base exclusion tag list. EXCLUDE_TAGS_TEMPLATE = f"{REQUIRES_FCV_TAG},multiversion_incompatible,{BACKPORT_REQUIRED_TAG}" diff --git a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml index 90232faa64a..59ab80667f3 100644 --- a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml +++ b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml @@ -4,15 +4,6 @@ # # PLEASE DO NOT REMOVE ANYTHING, unless a build variant no longer runs in any Evergreen project. - -evergreen_projects: - - mongodb-mongo-master - - mongodb-mongo-v5.0 - - mongodb-mongo-v4.4 - - mongodb-mongo-v4.2 - - mongodb-mongo-v4.0 - - evergreen_buildvariants: - name: ubuntu1604 edition: targeted diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml index e0f94ecd936..e587fa8107a 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml @@ -5,6 +5,8 @@ selector: - jstests/replsets/*.js exclude_with_any_tags: - multiversion_incompatible + # TODO (SERVER-55857): Remove this hardcoded tag once tests are filtered out using the + # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'. - requires_fcv_51 - backport_required_multiversion - replica_sets_multiversion_backport_required_multiversion diff --git a/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml index 5b0cb6a2727..6962ab75ca4 100644 --- a/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml +++ b/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml @@ -8,6 +8,8 @@ selector: - jstests/sharding/change_streams/*.js exclude_with_any_tags: - multiversion_incompatible + # TODO (SERVER-55857): Remove this hardcoded tag once tests are filtered out using the + # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'. - requires_fcv_51 - backport_required_multiversion - sharding_last_lts_mongos_and_mixed_shards_backport_required_multiversion diff --git a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml index 24aaa47e96c..0ab1b77d1e8 100644 --- a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml +++ b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml @@ -7,6 +7,8 @@ selector: - jstests/sharding/query/*.js exclude_with_any_tags: - multiversion_incompatible + # TODO (SERVER-55857): Remove this hardcoded tag once tests are filtered out using the + # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'. - requires_fcv_51 - backport_required_multiversion - replica_sets_multiversion_backport_required_multiversion diff --git a/buildscripts/resmokelib/__init__.py b/buildscripts/resmokelib/__init__.py index d96348ee2e8..a4d8ac6df6e 100644 --- a/buildscripts/resmokelib/__init__.py +++ b/buildscripts/resmokelib/__init__.py @@ -3,7 +3,6 @@ from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib import logging -from buildscripts.resmokelib import multiversionconstants from buildscripts.resmokelib import parser from buildscripts.resmokelib import reportfile from buildscripts.resmokelib import sighandler diff --git a/buildscripts/resmokelib/generate_fcv_constants/__init__.py b/buildscripts/resmokelib/generate_fcv_constants/__init__.py new file mode 100644 index 00000000000..fab4be5abd9 --- /dev/null +++ b/buildscripts/resmokelib/generate_fcv_constants/__init__.py @@ -0,0 +1,52 @@ +"""Generate FCV constants for consumption by non-C++ integration tests.""" +import argparse + +from buildscripts.resmokelib.plugin import PluginInterface, Subcommand + +_COMMAND = "generate-fcv-constants" + + +class GenerateFCVConstants(Subcommand): + """Interact with generating FCV constants.""" + + def __init__(self): + """Constructor.""" + + def execute(self) -> None: + """ + Work your magic. + + :return: None + """ + # This will cause multiversion constants to be generated. + import buildscripts.resmokelib.multiversionconstants # pylint: disable=unused-import + pass + + +class GenerateFCVConstantsPlugin(PluginInterface): + """Interact with generating FCV constants.""" + + def add_subcommand(self, subparsers): + """ + Add 'generate-fcv-constants' subcommand. + + :param subparsers: argparse parser to add to + :return: None + """ + # Can't hide this subcommand due to a Python bug. https://bugs.python.org/issue22848. + subparsers.add_parser(_COMMAND, help=argparse.SUPPRESS) + + def parse(self, subcommand, parser, parsed_args, **kwargs): + """ + Return the FCV constants subcommand for execution. + + :param subcommand: equivalent to parsed_args.command + :param parser: parser used + :param parsed_args: output of parsing + :param kwargs: additional args + :return: None or a Subcommand + """ + if subcommand != _COMMAND: + return None + + return GenerateFCVConstants() diff --git a/buildscripts/resmokelib/multiversionconstants.py b/buildscripts/resmokelib/multiversionconstants.py index bc440209727..055f9587fc1 100644 --- a/buildscripts/resmokelib/multiversionconstants.py +++ b/buildscripts/resmokelib/multiversionconstants.py @@ -1,10 +1,154 @@ """FCV and Server binary version constants used for multiversion testing.""" -LAST_LTS_BIN_VERSION = "5.0" -LAST_CONTINUOUS_BIN_VERSION = "5.0" -LAST_LTS_FCV = "5.0" -LAST_CONTINUOUS_FCV = "5.0" -LATEST_FCV = "5.1" +from bisect import bisect_left, bisect_right +import os +import re +import shutil +from subprocess import call, CalledProcessError, check_output, STDOUT +import structlog +import yaml + +from packaging.version import Version +from buildscripts.resmokelib.setup_multiversion.config import USE_EXISTING_RELEASES_FILE + +LOGGER = structlog.getLogger(__name__) + +# These values must match the include paths for artifacts.tgz in evergreen.yml. +MONGO_VERSION_YAML = ".resmoke_mongo_version.yml" +RELEASES_YAML = ".resmoke_mongo_release_values.yml" + + +def generate_mongo_version_file(): + """Generate the mongo version data file. Should only be called in the root of the mongo directory.""" + try: + res = check_output("git describe", shell=True, text=True) + except CalledProcessError as exp: + raise ChildProcessError("Failed to run git describe to get the latest tag") from exp + + # Write the current MONGO_VERSION to a data file. + with open(MONGO_VERSION_YAML, 'w') as mongo_version_fh: + # E.g. res = 'r5.1.0-alpha-597-g8c345c6693\n' + res = res[1:] # Remove the leading "r" character. + mongo_version_fh.write("mongo_version: " + res) + + +def generate_releases_file(): + """Generate the releases constants file.""" + # Copy the 'releases.yml' file from the source tree. + releases_yaml_path = os.path.join("src", "mongo", "util", "version", "releases.yml") + if not os.path.isfile(releases_yaml_path): + LOGGER.info( + 'Skipping yml file generation because file .resmoke_mongo_release_values.yml does not exist at path {}.' + .format(releases_yaml_path)) + return + + shutil.copyfile(releases_yaml_path, RELEASES_YAML) + + +def in_git_root_dir(): + """Return True if we are in the root of a git directory.""" + if call(["git", "branch"], stderr=STDOUT, stdout=open(os.devnull, 'w')) != 0: + # We are not in a git directory. + return False + + git_root_dir = check_output("git rev-parse --show-toplevel", shell=True, text=True).strip() + # Always use forward slash for the cwd path to resolve inconsistent formatting with Windows. + curr_dir = os.getcwd().replace("\\", "/") + return git_root_dir == curr_dir + + +if in_git_root_dir(): + generate_mongo_version_file() +else: + LOGGER.info("Skipping generating mongo version file since we're not in the root of a git repo") + +# Avoiding regenerating the releases file if this flag is set. Should only be set if there are +# multiple processes attempting to set up multiversion concurrently. +if not USE_EXISTING_RELEASES_FILE: + generate_releases_file() +else: + LOGGER.info( + "Skipping generating releases file since the --useExistingReleasesFile flag has been set") + + +class FCVConstantValues(object): + """Object to hold the calculated FCV constants.""" + + def __init__(self, latest, last_continuous, last_lts, requires_fcv_tag_list, + fcvs_less_than_latest): + """ + Initialize the object. + + :param latest: Latest FCV. + :param last_continuous: Last continuous FCV. + :param last_lts: Last LTS FCV. + :param requires_fcv_tag_list: List of FCVs that we need to generate a tag for. + :param fcvs_less_than_latest: List of all FCVs that are less than latest, starting from v4.0. + """ + self.latest = latest + self.last_continuous = last_continuous + self.last_lts = last_lts + self.requires_fcv_tag_list = requires_fcv_tag_list + self.fcvs_less_than_latest = fcvs_less_than_latest + + +def calculate_fcv_constants(): + """Calculate multiversion constants from data files.""" + mongo_version_yml_file = open(MONGO_VERSION_YAML, 'r') + mongo_version_yml = yaml.safe_load(mongo_version_yml_file) + mongo_version = mongo_version_yml['mongo_version'] + latest = Version(re.match(r'^[0-9]+\.[0-9]+', mongo_version).group(0)) + + releases_yml_file = open(RELEASES_YAML, 'r') + releases_yml = yaml.safe_load(releases_yml_file) + + fcvs = releases_yml['featureCompatibilityVersions'] + fcvs = list(map(Version, fcvs)) + lts = releases_yml['longTermSupportReleases'] + lts = list(map(Version, lts)) + + mongo_version_yml_file.close() + releases_yml_file.close() + + # Highest release less than latest. + last_continuous = fcvs[bisect_left(fcvs, latest) - 1] + + # Highest LTS release less than latest. + last_lts = lts[bisect_left(lts, latest) - 1] + + # All FCVs greater than last LTS, up to latest. + requires_fcv_tag_list = fcvs[bisect_right(fcvs, last_lts):bisect_right(fcvs, latest)] + + # All FCVs less than latest. + fcvs_less_than_latest = fcvs[:bisect_left(fcvs, latest)] + + return FCVConstantValues(latest, last_continuous, last_lts, requires_fcv_tag_list, + fcvs_less_than_latest) + + +def version_str(version): + """Return a string of the given version in 'MAJOR.MINOR' form.""" + return '{}.{}'.format(version.major, version.minor) + + +def tag_str(version): + """Return a tag for the given version.""" + return 'requires_fcv_{}{}'.format(version.major, version.minor) + + +def evg_project_str(version): + """Return the evergreen project name for the given version.""" + return 'mongodb-mongo-v{}.{}'.format(version.major, version.minor) + + +fcv_constants = calculate_fcv_constants() + +LAST_LTS_BIN_VERSION = version_str(fcv_constants.last_lts) +LAST_CONTINUOUS_BIN_VERSION = version_str(fcv_constants.last_continuous) + +LAST_LTS_FCV = version_str(fcv_constants.last_lts) +LAST_CONTINUOUS_FCV = version_str(fcv_constants.last_continuous) +LATEST_FCV = version_str(fcv_constants.latest) LAST_CONTINUOUS_MONGO_BINARY = "mongo-" + LAST_CONTINUOUS_BIN_VERSION LAST_CONTINUOUS_MONGOD_BINARY = "mongod-" + LAST_CONTINUOUS_BIN_VERSION @@ -14,7 +158,12 @@ LAST_LTS_MONGO_BINARY = "mongo-" + LAST_LTS_BIN_VERSION LAST_LTS_MONGOD_BINARY = "mongod-" + LAST_LTS_BIN_VERSION LAST_LTS_MONGOS_BINARY = "mongos-" + LAST_LTS_BIN_VERSION -REQUIRES_FCV_TAG_LATEST = "requires_fcv_51" +REQUIRES_FCV_TAG_LATEST = tag_str(fcv_constants.latest) + +# Generate tags for all FCVS in (lastLTS, latest]. +# All multiversion tests should be run with these tags excluded. +REQUIRES_FCV_TAG = ",".join([tag_str(fcv) for fcv in fcv_constants.requires_fcv_tag_list]) -# TODO: rename this to REQUIRES_FCV_TAGS (plural) when `requires_fcv_52` is added to it. -REQUIRES_FCV_TAG = REQUIRES_FCV_TAG_LATEST +# Generate evergreen project names for all FCVs less than latest. +EVERGREEN_PROJECTS = ['mongodb-mongo-master'] +EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in fcv_constants.fcvs_less_than_latest]) diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 433152a8bcd..9442634edca 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -4,6 +4,7 @@ import argparse import shlex from buildscripts.resmokelib import configure_resmoke +from buildscripts.resmokelib.generate_fcv_constants import GenerateFCVConstantsPlugin from buildscripts.resmokelib.hang_analyzer import HangAnalyzerPlugin from buildscripts.resmokelib.powercycle import PowercyclePlugin from buildscripts.resmokelib.run import RunPlugin @@ -20,6 +21,7 @@ _PLUGINS = [ PowercyclePlugin(), SymbolizerPlugin(), BisectPlugin(), + GenerateFCVConstantsPlugin(), ] diff --git a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py index b26159e3b65..54c0a0ea3d8 100755 --- a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py +++ b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py @@ -9,7 +9,6 @@ from subprocess import check_output import requests from buildscripts.ciconfig import tags as _tags -from buildscripts.resmokelib import multiversionconstants from buildscripts.resmokelib.config import MultiversionOptions from buildscripts.resmokelib.core.programs import get_path_env_var from buildscripts.resmokelib.utils import is_windows @@ -88,6 +87,8 @@ def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Log # Get the state of the backports_required_for_multiversion_tests.yml file for the old # binary we are running tests against. We do this by using the commit hash from the old # mongo shell executable. + from buildscripts.resmokelib import multiversionconstants + shell_version = { MultiversionOptions.LAST_LTS: multiversionconstants.LAST_LTS_MONGO_BINARY, MultiversionOptions.LAST_CONTINUOUS: multiversionconstants.LAST_CONTINUOUS_MONGO_BINARY, diff --git a/buildscripts/resmokelib/setup_multiversion/config.py b/buildscripts/resmokelib/setup_multiversion/config.py index e81fd445f58..0a39ca5e808 100644 --- a/buildscripts/resmokelib/setup_multiversion/config.py +++ b/buildscripts/resmokelib/setup_multiversion/config.py @@ -6,6 +6,8 @@ SETUP_MULTIVERSION_CONFIG = "buildscripts/resmokeconfig/setup_multiversion/setup # Records the paths of installed multiversion binaries on Windows. WINDOWS_BIN_PATHS_FILE = "windows_binary_paths.txt" +USE_EXISTING_RELEASES_FILE = False + class Buildvariant: """Class represents buildvariant in setup multiversion config.""" diff --git a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py index 0017f68b5c4..03bd6e1e863 100644 --- a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py +++ b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py @@ -5,6 +5,8 @@ to include its version) into an install directory and symlinks the binaries with versions to another directory. This script supports community and enterprise builds. """ +from itertools import chain +import argparse import logging import os import platform @@ -18,6 +20,8 @@ import distro import structlog import yaml +from requests.exceptions import HTTPError + from buildscripts.resmokelib.plugin import PluginInterface, Subcommand from buildscripts.resmokelib.setup_multiversion import config, download, github_conn from buildscripts.resmokelib.utils import evergreen_conn, is_windows @@ -70,9 +74,9 @@ def get_merge_base_commit(version: str) -> str: cmd = ["git", "merge-base", "origin/master", f"origin/v{version}"] result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) if result.returncode: - LOGGER.error("Git merge-base command failed.", cmd=cmd, + LOGGER.error("Git merge-base command failed. Falling back to latest master", cmd=cmd, error=result.stderr.decode("utf-8").strip()) - exit(result.returncode) + return None commit_hash = result.stdout.decode("utf-8").strip() LOGGER.info("Found merge-base commit.", cmd=cmd, commit=commit_hash) return commit_hash @@ -84,8 +88,8 @@ class SetupMultiversion(Subcommand): # pylint: disable=too-many-instance-attributes def __init__(self, download_options, install_dir="", link_dir="", mv_platform=None, edition=None, architecture=None, use_latest=None, versions=None, - evergreen_config=None, github_oauth_token=None, debug=None, - ignore_failed_push=False): + install_last_lts=None, install_last_continuous=None, evergreen_config=None, + github_oauth_token=None, debug=None, ignore_failed_push=False): """Initialize.""" setup_logging(debug) self.install_dir = os.path.abspath(install_dir) @@ -97,6 +101,8 @@ class SetupMultiversion(Subcommand): self.architecture = architecture.lower() if architecture else None self.use_latest = use_latest self.versions = versions + self.install_last_lts = install_last_lts + self.install_last_continuous = install_last_continuous self.ignore_failed_push = ignore_failed_push self.download_binaries = download_options.download_binaries @@ -131,6 +137,13 @@ class SetupMultiversion(Subcommand): def execute(self): """Execute setup multiversion mongodb.""" + from buildscripts.resmokelib import multiversionconstants + + if self.install_last_lts: + self.versions.append(multiversionconstants.LAST_LTS_FCV) + if self.install_last_continuous: + self.versions.append(multiversionconstants.LAST_CONTINUOUS_FCV) + self.versions = list(set(self.versions)) for version in self.versions: LOGGER.info("Setting up version.", version=version) @@ -218,25 +231,26 @@ class SetupMultiversion(Subcommand): """Return latest urls.""" urls = {} + # Assuming that project names contain <major>.<minor> version evg_project = f"mongodb-mongo-v{version}" if version == "master": evg_project = "mongodb-mongo-master" - if evg_project not in self.config.evergreen_projects: - return urls - - LOGGER.debug("Found evergreen project.", evergreen_project=evg_project) - # Assuming that project names contain <major>.<minor> version - major_minor_version = version - - buildvariant_name = self.get_buildvariant_name(major_minor_version) - LOGGER.debug("Found buildvariant.", buildvariant_name=buildvariant_name) - evg_versions = evergreen_conn.get_evergreen_versions(self.evg_api, evg_project) + evg_version = None + try: + evg_version = next(evg_versions) + except HTTPError as err: + # Evergreen currently returns 500 if the version does not exist. + # TODO (SERVER-59675): Remove the check for 500 once evergreen returns 404 instead. + if not (err.response.status_code == 500 or err.response.status_code == 404): + raise + buildvariant_name = self.get_buildvariant_name(version) + LOGGER.debug("Found buildvariant.", buildvariant_name=buildvariant_name) found_start_revision = start_from_revision is None - for evg_version in evg_versions: + for evg_version in chain(iter([evg_version]), evg_versions): # Skip all versions until we get the revision we should start looking from if found_start_revision is False and evg_version.revision != start_from_revision: LOGGER.warning("Skipping evergreen version.", evg_version=evg_version) @@ -260,13 +274,12 @@ class SetupMultiversion(Subcommand): def get_urls(self, version: str, buildvariant_name: Optional[str] = None) -> Dict[str, Any]: """Return multiversion urls for a given version (as binary version or commit hash or evergreen_version_id).""" - evg_version = evergreen_conn.get_evergreen_version(self.config, self.evg_api, version) + evg_version = evergreen_conn.get_evergreen_version(self.evg_api, version) if evg_version is None: git_tag, commit_hash = github_conn.get_git_tag_and_commit(self.github_oauth_token, version) LOGGER.info("Found git attributes.", git_tag=git_tag, commit_hash=commit_hash) - evg_version = evergreen_conn.get_evergreen_version(self.config, self.evg_api, - commit_hash) + evg_version = evergreen_conn.get_evergreen_version(self.evg_api, commit_hash) if evg_version is None: return {} @@ -362,10 +375,14 @@ class SetupMultiversionPlugin(PluginInterface): da=args.download_artifacts, dv=args.download_python_venv) + if args.use_existing_releases_file: + config.USE_EXISTING_RELEASES_FILE = True + return SetupMultiversion( install_dir=args.install_dir, link_dir=args.link_dir, mv_platform=args.platform, edition=args.edition, architecture=args.architecture, use_latest=args.use_latest, - versions=args.versions, download_options=download_options, + versions=args.versions, install_last_lts=args.install_last_lts, + install_last_continuous=args.install_last_continuous, download_options=download_options, evergreen_config=args.evergreen_config, github_oauth_token=args.github_oauth_token, ignore_failed_push=(not args.require_push), debug=args.debug) @@ -400,6 +417,11 @@ class SetupMultiversionPlugin(PluginInterface): "Binary version examples: 4.0, 4.0.1, 4.0.0-rc0. If 'rc' is included in the version name, " "we'll use the exact rc, otherwise we'll pull the highest non-rc version compatible with the " "version specified.") + parser.add_argument("--installLastLTS", dest="install_last_lts", action="store_true", + help="If specified, the last LTS version will be installed") + parser.add_argument("--installLastContinuous", dest="install_last_continuous", + action="store_true", + help="If specified, the last continuous version will be installed") parser.add_argument("-db", "--downloadBinaries", dest="download_binaries", action="store_true", default=True, @@ -427,6 +449,11 @@ class SetupMultiversionPlugin(PluginInterface): parser.add_argument( "-rp", "--require-push", dest="require_push", action="store_true", default=False, help="Require the push task to be successful for assets to be downloaded") + # Hidden flag that determines if we should generate a new releases yaml file. This flag + # should be set to True if we are invoking setup_multiversion multiple times in parallel, + # to prevent multiple processes from modifying the releases yaml file simultaneously. + parser.add_argument("--useExistingReleasesFile", dest="use_existing_releases_file", + action="store_true", default=False, help=argparse.SUPPRESS) def add_subcommand(self, subparsers): """Create and add the parser for the subcommand.""" diff --git a/buildscripts/resmokelib/testing/fixtures/_builder.py b/buildscripts/resmokelib/testing/fixtures/_builder.py index f3d90fc4240..7cf4bf0ef38 100644 --- a/buildscripts/resmokelib/testing/fixtures/_builder.py +++ b/buildscripts/resmokelib/testing/fixtures/_builder.py @@ -7,7 +7,7 @@ from git import Repo import buildscripts.resmokelib.utils.registry as registry import buildscripts.resmokelib.config as config -from buildscripts.resmokelib import errors, multiversionconstants +from buildscripts.resmokelib import errors from buildscripts.resmokelib.utils import default_if_none from buildscripts.resmokelib.utils import autoloader from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib @@ -86,6 +86,7 @@ class ReplSetBuilder(FixtureBuilder): num_nodes = kwargs["num_nodes"] latest_mongod = mongod_executable + from buildscripts.resmokelib import multiversionconstants fcv = multiversionconstants.LATEST_FCV executables = {BinVersionEnum.NEW: latest_mongod} diff --git a/buildscripts/resmokelib/testing/fixtures/fixturelib.py b/buildscripts/resmokelib/testing/fixtures/fixturelib.py index f38b71e944f..60d01badef8 100644 --- a/buildscripts/resmokelib/testing/fixtures/fixturelib.py +++ b/buildscripts/resmokelib/testing/fixtures/fixturelib.py @@ -8,7 +8,6 @@ from buildscripts.resmokelib import logging from buildscripts.resmokelib.core import network from buildscripts.resmokelib.utils.history import make_historic as _make_historic from buildscripts.resmokelib.testing.fixtures import _builder -from buildscripts.resmokelib.multiversionconstants import LAST_LTS_MONGOD_BINARY, LAST_LTS_MONGOS_BINARY, LAST_CONTINUOUS_MONGOD_BINARY, LAST_CONTINUOUS_MONGOS_BINARY class FixtureLib(object): @@ -101,6 +100,8 @@ class _FixtureConfig(object): # pylint: disable=too-many-instance-attributes def __init__(self): """Initialize FixtureConfig, setting values.""" + from buildscripts.resmokelib.multiversionconstants import LAST_LTS_MONGOD_BINARY, LAST_LTS_MONGOS_BINARY, LAST_CONTINUOUS_MONGOD_BINARY, LAST_CONTINUOUS_MONGOS_BINARY + # pylint: disable=invalid-name self.MONGOD_EXECUTABLE = config.MONGOD_EXECUTABLE self.DEFAULT_MONGOD_EXECUTABLE = config.DEFAULT_MONGOD_EXECUTABLE diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index 025bbfc4ea6..006b19fd6df 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -9,7 +9,6 @@ from typing import List import pymongo import pymongo.errors -import buildscripts.resmokelib.multiversionconstants as multiversion import buildscripts.resmokelib.utils.registry as registry _VERSIONS = {} # type: ignore diff --git a/buildscripts/resmokelib/utils/evergreen_conn.py b/buildscripts/resmokelib/utils/evergreen_conn.py index 4104c76f4b4..41e3e6e91aa 100644 --- a/buildscripts/resmokelib/utils/evergreen_conn.py +++ b/buildscripts/resmokelib/utils/evergreen_conn.py @@ -114,13 +114,15 @@ def get_generic_buildvariant_name(config, major_minor_version): return generic_buildvariant_name -def get_evergreen_version(config: SetupMultiversionConfig, evg_api: RetryingEvergreenApi, - evg_ref: str) -> Optional[Version]: +def get_evergreen_version(evg_api: RetryingEvergreenApi, evg_ref: str) -> Optional[Version]: """Return evergreen version by reference (commit_hash or evergreen_version_id).""" + from buildscripts.resmokelib import multiversionconstants + # Evergreen reference as evergreen_version_id evg_refs = [evg_ref] # Evergreen reference as {project_name}_{commit_hash} - evg_refs.extend(f"{proj.replace('-', '_')}_{evg_ref}" for proj in config.evergreen_projects) + evg_refs.extend( + f"{proj.replace('-', '_')}_{evg_ref}" for proj in multiversionconstants.EVERGREEN_PROJECTS) for ref in evg_refs: try: diff --git a/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py b/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py index e0f6730f3c9..7c843601e90 100644 --- a/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py +++ b/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py @@ -179,7 +179,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): def test_no_compile_artifacts(self, mock_get_compile_artifact_urls, mock_versions_by_project, mock_version): mock_version.build_variants_map = {self.buildvariant_name: "build_id"} - mock_versions_by_project.return_value = [mock_version] + mock_versions_by_project.return_value = iter([mock_version]) mock_get_compile_artifact_urls.return_value = {} urls = self.setup_multiversion.get_latest_urls("4.4") @@ -196,7 +196,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): } mock_version.build_variants_map = {self.buildvariant_name: "build_id"} - mock_versions_by_project.return_value = [mock_version] + mock_versions_by_project.return_value = iter([mock_version]) mock_get_compile_artifact_urls.return_value = expected_urls urls = self.setup_multiversion.get_latest_urls("4.4") @@ -218,7 +218,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase): mock_expected_version.build_variants_map = {self.buildvariant_name: "build_id"} evg_versions = [mock_version for _ in range(3)] evg_versions.append(mock_expected_version) - mock_versions_by_project.return_value = evg_versions + mock_versions_by_project.return_value = iter(evg_versions) mock_get_compile_artifact_urls.side_effect = lambda evg_api, evg_version, buildvariant_name, ignore_failed_push: { (self.setup_multiversion.evg_api, mock_version, self.buildvariant_name, False): {}, (self.setup_multiversion.evg_api, mock_expected_version, self.buildvariant_name, False): diff --git a/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py b/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py index 2de1dbe7008..7bddcc7e4ab 100644 --- a/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py +++ b/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py @@ -103,13 +103,6 @@ class TestGetGenericBuildvariantName(unittest.TestCase): class TestGetEvergreenProjectAndVersion(unittest.TestCase): - def setUp(self): - raw_yaml = {"evergreen_projects": [ - "mongodb-mongo-master", - "mongodb-mongo-v4.4", - ]} - self.config = SetupMultiversionConfig(raw_yaml) - @patch("evergreen.version.Version") @patch("evergreen.api.EvergreenApi") def test_version_by_commit_hash_found(self, mock_evg_api, mock_version): @@ -123,7 +116,7 @@ class TestGetEvergreenProjectAndVersion(unittest.TestCase): raise HTTPError() mock_evg_api.version_by_id.side_effect = version_by_id_side_effect - evg_version = evergreen_conn.get_evergreen_version(self.config, mock_evg_api, commit_hash) + evg_version = evergreen_conn.get_evergreen_version(mock_evg_api, commit_hash) self.assertEqual(mock_version, evg_version) self.assertEqual(mock_version.version_id, expected_evergreen_version_id) @@ -139,15 +132,14 @@ class TestGetEvergreenProjectAndVersion(unittest.TestCase): raise HTTPError() mock_evg_api.version_by_id.side_effect = version_by_id_side_effect - evg_version = evergreen_conn.get_evergreen_version(self.config, mock_evg_api, - evergreen_version_id) + evg_version = evergreen_conn.get_evergreen_version(mock_evg_api, evergreen_version_id) self.assertEqual(mock_version, evg_version) self.assertEqual(mock_version.version_id, evergreen_version_id) @patch("evergreen.api.EvergreenApi") def test_version_not_found(self, mock_evg_api): mock_evg_api.version_by_id.side_effect = HTTPError - evg_version = evergreen_conn.get_evergreen_version(self.config, mock_evg_api, "dummy") + evg_version = evergreen_conn.get_evergreen_version(mock_evg_api, "dummy") self.assertIsNone(evg_version) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 68d229df5d7..2af9a1ef0bc 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1978,11 +1978,21 @@ tasks: args: - "src/evergreen/gen_feature_flags.sh" + - *f_expansions_write + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/get_bin_and_fcv_versions.sh" + - command: archive.targz_pack params: target: "artifacts.tgz" source_dir: "src" include: + - ".resmoke_mongo_version.yml" + - ".resmoke_mongo_release_values.yml" - "patch_test_tags.tgz" - "./build/**.gcno" - "./etc/*san.suppressions" @@ -9547,6 +9557,8 @@ buildvariants: run_on: - rhel80-small expansions: &enterprise-rhel-80-64-bit-multiversion-expansions-template + # TODO (SERVER-55857): Remove 'requires_fcv_51' once tests are filtered out using the + # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'. test_flags: >- --excludeWithAnyTags=requires_fcv_51,multiversion_incompatible,disabled_due_to_server_58295 compile_flags: >- @@ -9576,6 +9588,8 @@ buildvariants: display_name: "Enterprise RHEL 8.0 (implicit multiversion & all feature flags)" expansions: <<: *enterprise-rhel-80-64-bit-multiversion-expansions-template + # TODO (SERVER-55857): Remove 'requires_fcv_51' once tests are filtered out using the + # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'. test_flags: >- --excludeWithAnyTags=requires_fcv_51,multiversion_incompatible,disabled_due_to_server_58295 --runAllFeatureFlagTests diff --git a/evergreen/get_bin_and_fcv_versions.sh b/evergreen/get_bin_and_fcv_versions.sh new file mode 100644 index 00000000000..0bc91fe4baa --- /dev/null +++ b/evergreen/get_bin_and_fcv_versions.sh @@ -0,0 +1,9 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit + +activate_venv +python buildscripts/resmoke.py generate-fcv-constants diff --git a/evergreen/multiversion_setup.sh b/evergreen/multiversion_setup.sh index 5af1ebcf725..64bd44a6aa7 100644 --- a/evergreen/multiversion_setup.sh +++ b/evergreen/multiversion_setup.sh @@ -70,4 +70,6 @@ $python buildscripts/resmoke.py setup-multiversion \ --edition $edition \ --platform $platform \ --architecture $architecture \ - --useLatest 4.4 5.0 + --useLatest \ + --installLastLTS \ + --installLastContinuous 4.4 diff --git a/src/mongo/db/wire_version.h b/src/mongo/db/wire_version.h index 61b52dc190b..61e5aab759e 100644 --- a/src/mongo/db/wire_version.h +++ b/src/mongo/db/wire_version.h @@ -99,8 +99,8 @@ enum WireVersion { // Supports features available from 5.1 and onwards. WIRE_VERSION_51 = 14, - // Calculate the latest wire version using the number of releases since 4.4. - LATEST_WIRE_VERSION = RESUMABLE_INITIAL_SYNC + multiversion::kSince_4_4, + // Calculate the latest wire version using the number of releases since 4.0. + LATEST_WIRE_VERSION = REPLICA_SET_TRANSACTIONS + multiversion::kSince_4_0, // Set this to LATEST_WIRE_VERSION - 1. LAST_CONT_WIRE_VERSION = LATEST_WIRE_VERSION - 1, diff --git a/src/mongo/idl/feature_flag_test.cpp b/src/mongo/idl/feature_flag_test.cpp index e4a41e48c67..10ecfa9824f 100644 --- a/src/mongo/idl/feature_flag_test.cpp +++ b/src/mongo/idl/feature_flag_test.cpp @@ -115,9 +115,12 @@ TEST_F(FeatureFlagTest, ServerStatus) { _featureFlagBlender->append(nullptr, builder, "blender"); - ASSERT_BSONOBJ_EQ(builder.obj(), - BSON("blender" << BSON("value" << true << "version" - << "5.1"))); + ASSERT_BSONOBJ_EQ( + builder.obj(), + // (Generic FCV reference): feature flag test. + BSON("blender" << BSON("value" + << true << "version" + << multiversion::toString(multiversion::GenericFCV::kLatest)))); } { diff --git a/src/mongo/util/version/releases.h.tpl b/src/mongo/util/version/releases.h.tpl index 4232e1c6222..787494840cf 100644 --- a/src/mongo/util/version/releases.h.tpl +++ b/src/mongo/util/version/releases.h.tpl @@ -161,8 +161,8 @@ enum class FeatureCompatibilityVersion { #end for }; -## Calculate number of versions since v4.4. -constexpr size_t kSince_$underscores(Version('4.4')) = ${bisect_left(fcvs, latest)}; +## Calculate number of versions since v4.0. +constexpr size_t kSince_$underscores(Version('4.0')) = ${bisect_left(fcvs, latest)}; // Last LTS was "$last_lts". constexpr size_t kSinceLastLTS = ${bisect_left(fcvs, latest) - bisect_left(fcvs, last_lts)}; diff --git a/src/mongo/util/version/releases.yml b/src/mongo/util/version/releases.yml index ea37d239bb1..d55c84dd4ad 100644 --- a/src/mongo/util/version/releases.yml +++ b/src/mongo/util/version/releases.yml @@ -3,8 +3,10 @@ # multiversion_constants.h. Please consult the STAR team before # modifying this file. -# All FCVs starting with 4.4. +# All FCVs starting with 4.0. featureCompatibilityVersions: + - "4.0" + - "4.2" - "4.4" - "4.7" - "4.8" |