summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorXueruiFa <xuerui.fa@mongodb.com>2021-09-02 20:00:17 +0000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2021-09-20 16:20:44 +0000
commitefdac6418b6cd6012839c3e87a7b6dc4893d6ef0 (patch)
tree555f16b81f98eff696aa7d1640ba925a758e5074
parentd14837390af247b23cb6ca70908280107dccf986 (diff)
downloadmongo-efdac6418b6cd6012839c3e87a7b6dc4893d6ef0.tar.gz
SERVER-59012: Calculate FCV constants in resmoke
-rw-r--r--.gitignore4
-rw-r--r--buildscripts/resmokeconfig/matrix_suites/tags.py2
-rw-r--r--buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml9
-rw-r--r--buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml2
-rw-r--r--buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml2
-rw-r--r--buildscripts/resmokeconfig/suites/sharding_multiversion.yml2
-rw-r--r--buildscripts/resmokelib/__init__.py1
-rw-r--r--buildscripts/resmokelib/generate_fcv_constants/__init__.py52
-rw-r--r--buildscripts/resmokelib/multiversionconstants.py165
-rw-r--r--buildscripts/resmokelib/parser.py2
-rwxr-xr-xbuildscripts/resmokelib/run/generate_multiversion_exclude_tags.py3
-rw-r--r--buildscripts/resmokelib/setup_multiversion/config.py2
-rw-r--r--buildscripts/resmokelib/setup_multiversion/setup_multiversion.py65
-rw-r--r--buildscripts/resmokelib/testing/fixtures/_builder.py3
-rw-r--r--buildscripts/resmokelib/testing/fixtures/fixturelib.py3
-rw-r--r--buildscripts/resmokelib/testing/fixtures/interface.py1
-rw-r--r--buildscripts/resmokelib/utils/evergreen_conn.py8
-rw-r--r--buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py6
-rw-r--r--buildscripts/tests/resmokelib/utils/test_evergreen_conn.py14
-rw-r--r--etc/evergreen.yml14
-rw-r--r--evergreen/get_bin_and_fcv_versions.sh9
-rw-r--r--evergreen/multiversion_setup.sh4
-rw-r--r--src/mongo/db/wire_version.h4
-rw-r--r--src/mongo/idl/feature_flag_test.cpp9
-rw-r--r--src/mongo/util/version/releases.h.tpl4
-rw-r--r--src/mongo/util/version/releases.yml4
26 files changed, 325 insertions, 69 deletions
diff --git a/.gitignore b/.gitignore
index 21394b3ab3c..2434c53fb6f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -202,8 +202,10 @@ resmoke.ini
# UndoDB Recordings
*.undo
-# Resmoke runtime trackers
+# Resmoke runtime configuration and trackers.
.resmoke_start_time.yml
+.resmoke_mongo_version.yml
+.resmoke_mongo_release_values.yml
# libfuzzer artifacts
default.profraw
diff --git a/buildscripts/resmokeconfig/matrix_suites/tags.py b/buildscripts/resmokeconfig/matrix_suites/tags.py
index d7b30cec275..571da6b638e 100644
--- a/buildscripts/resmokeconfig/matrix_suites/tags.py
+++ b/buildscripts/resmokeconfig/matrix_suites/tags.py
@@ -1,5 +1,4 @@
"""Dynamically Generated tags."""
-from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG
# TODO SERVER-55857: Let this file be the single source of truth for tags. Remove dupe definitions.
@@ -10,6 +9,7 @@ class Tags(object):
BACKPORT_REQUIRED_TAG = "backport_required_multiversion"
+ from buildscripts.resmokelib.multiversionconstants import REQUIRES_FCV_TAG
# Base exclusion tag list.
EXCLUDE_TAGS_TEMPLATE = f"{REQUIRES_FCV_TAG},multiversion_incompatible,{BACKPORT_REQUIRED_TAG}"
diff --git a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml
index 90232faa64a..59ab80667f3 100644
--- a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml
+++ b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml
@@ -4,15 +4,6 @@
#
# PLEASE DO NOT REMOVE ANYTHING, unless a build variant no longer runs in any Evergreen project.
-
-evergreen_projects:
- - mongodb-mongo-master
- - mongodb-mongo-v5.0
- - mongodb-mongo-v4.4
- - mongodb-mongo-v4.2
- - mongodb-mongo-v4.0
-
-
evergreen_buildvariants:
- name: ubuntu1604
edition: targeted
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
index e0f94ecd936..e587fa8107a 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
@@ -5,6 +5,8 @@ selector:
- jstests/replsets/*.js
exclude_with_any_tags:
- multiversion_incompatible
+ # TODO (SERVER-55857): Remove this hardcoded tag once tests are filtered out using the
+ # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'.
- requires_fcv_51
- backport_required_multiversion
- replica_sets_multiversion_backport_required_multiversion
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml
index 5b0cb6a2727..6962ab75ca4 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_lts_mongos_and_mixed_shards.yml
@@ -8,6 +8,8 @@ selector:
- jstests/sharding/change_streams/*.js
exclude_with_any_tags:
- multiversion_incompatible
+ # TODO (SERVER-55857): Remove this hardcoded tag once tests are filtered out using the
+ # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'.
- requires_fcv_51
- backport_required_multiversion
- sharding_last_lts_mongos_and_mixed_shards_backport_required_multiversion
diff --git a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
index 24aaa47e96c..0ab1b77d1e8 100644
--- a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
@@ -7,6 +7,8 @@ selector:
- jstests/sharding/query/*.js
exclude_with_any_tags:
- multiversion_incompatible
+ # TODO (SERVER-55857): Remove this hardcoded tag once tests are filtered out using the
+ # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'.
- requires_fcv_51
- backport_required_multiversion
- replica_sets_multiversion_backport_required_multiversion
diff --git a/buildscripts/resmokelib/__init__.py b/buildscripts/resmokelib/__init__.py
index d96348ee2e8..a4d8ac6df6e 100644
--- a/buildscripts/resmokelib/__init__.py
+++ b/buildscripts/resmokelib/__init__.py
@@ -3,7 +3,6 @@
from buildscripts.resmokelib import config
from buildscripts.resmokelib import errors
from buildscripts.resmokelib import logging
-from buildscripts.resmokelib import multiversionconstants
from buildscripts.resmokelib import parser
from buildscripts.resmokelib import reportfile
from buildscripts.resmokelib import sighandler
diff --git a/buildscripts/resmokelib/generate_fcv_constants/__init__.py b/buildscripts/resmokelib/generate_fcv_constants/__init__.py
new file mode 100644
index 00000000000..fab4be5abd9
--- /dev/null
+++ b/buildscripts/resmokelib/generate_fcv_constants/__init__.py
@@ -0,0 +1,52 @@
+"""Generate FCV constants for consumption by non-C++ integration tests."""
+import argparse
+
+from buildscripts.resmokelib.plugin import PluginInterface, Subcommand
+
+_COMMAND = "generate-fcv-constants"
+
+
+class GenerateFCVConstants(Subcommand):
+ """Interact with generating FCV constants."""
+
+ def __init__(self):
+ """Constructor."""
+
+ def execute(self) -> None:
+ """
+ Work your magic.
+
+ :return: None
+ """
+ # This will cause multiversion constants to be generated.
+ import buildscripts.resmokelib.multiversionconstants # pylint: disable=unused-import
+ pass
+
+
+class GenerateFCVConstantsPlugin(PluginInterface):
+ """Interact with generating FCV constants."""
+
+ def add_subcommand(self, subparsers):
+ """
+ Add 'generate-fcv-constants' subcommand.
+
+ :param subparsers: argparse parser to add to
+ :return: None
+ """
+ # Can't hide this subcommand due to a Python bug. https://bugs.python.org/issue22848.
+ subparsers.add_parser(_COMMAND, help=argparse.SUPPRESS)
+
+ def parse(self, subcommand, parser, parsed_args, **kwargs):
+ """
+ Return the FCV constants subcommand for execution.
+
+ :param subcommand: equivalent to parsed_args.command
+ :param parser: parser used
+ :param parsed_args: output of parsing
+ :param kwargs: additional args
+ :return: None or a Subcommand
+ """
+ if subcommand != _COMMAND:
+ return None
+
+ return GenerateFCVConstants()
diff --git a/buildscripts/resmokelib/multiversionconstants.py b/buildscripts/resmokelib/multiversionconstants.py
index bc440209727..055f9587fc1 100644
--- a/buildscripts/resmokelib/multiversionconstants.py
+++ b/buildscripts/resmokelib/multiversionconstants.py
@@ -1,10 +1,154 @@
"""FCV and Server binary version constants used for multiversion testing."""
-LAST_LTS_BIN_VERSION = "5.0"
-LAST_CONTINUOUS_BIN_VERSION = "5.0"
-LAST_LTS_FCV = "5.0"
-LAST_CONTINUOUS_FCV = "5.0"
-LATEST_FCV = "5.1"
+from bisect import bisect_left, bisect_right
+import os
+import re
+import shutil
+from subprocess import call, CalledProcessError, check_output, STDOUT
+import structlog
+import yaml
+
+from packaging.version import Version
+from buildscripts.resmokelib.setup_multiversion.config import USE_EXISTING_RELEASES_FILE
+
+LOGGER = structlog.getLogger(__name__)
+
+# These values must match the include paths for artifacts.tgz in evergreen.yml.
+MONGO_VERSION_YAML = ".resmoke_mongo_version.yml"
+RELEASES_YAML = ".resmoke_mongo_release_values.yml"
+
+
+def generate_mongo_version_file():
+ """Generate the mongo version data file. Should only be called in the root of the mongo directory."""
+ try:
+ res = check_output("git describe", shell=True, text=True)
+ except CalledProcessError as exp:
+ raise ChildProcessError("Failed to run git describe to get the latest tag") from exp
+
+ # Write the current MONGO_VERSION to a data file.
+ with open(MONGO_VERSION_YAML, 'w') as mongo_version_fh:
+ # E.g. res = 'r5.1.0-alpha-597-g8c345c6693\n'
+ res = res[1:] # Remove the leading "r" character.
+ mongo_version_fh.write("mongo_version: " + res)
+
+
+def generate_releases_file():
+ """Generate the releases constants file."""
+ # Copy the 'releases.yml' file from the source tree.
+ releases_yaml_path = os.path.join("src", "mongo", "util", "version", "releases.yml")
+ if not os.path.isfile(releases_yaml_path):
+ LOGGER.info(
+ 'Skipping yml file generation because file .resmoke_mongo_release_values.yml does not exist at path {}.'
+ .format(releases_yaml_path))
+ return
+
+ shutil.copyfile(releases_yaml_path, RELEASES_YAML)
+
+
+def in_git_root_dir():
+ """Return True if we are in the root of a git directory."""
+ if call(["git", "branch"], stderr=STDOUT, stdout=open(os.devnull, 'w')) != 0:
+ # We are not in a git directory.
+ return False
+
+ git_root_dir = check_output("git rev-parse --show-toplevel", shell=True, text=True).strip()
+ # Always use forward slash for the cwd path to resolve inconsistent formatting with Windows.
+ curr_dir = os.getcwd().replace("\\", "/")
+ return git_root_dir == curr_dir
+
+
+if in_git_root_dir():
+ generate_mongo_version_file()
+else:
+ LOGGER.info("Skipping generating mongo version file since we're not in the root of a git repo")
+
+# Avoiding regenerating the releases file if this flag is set. Should only be set if there are
+# multiple processes attempting to set up multiversion concurrently.
+if not USE_EXISTING_RELEASES_FILE:
+ generate_releases_file()
+else:
+ LOGGER.info(
+ "Skipping generating releases file since the --useExistingReleasesFile flag has been set")
+
+
+class FCVConstantValues(object):
+ """Object to hold the calculated FCV constants."""
+
+ def __init__(self, latest, last_continuous, last_lts, requires_fcv_tag_list,
+ fcvs_less_than_latest):
+ """
+ Initialize the object.
+
+ :param latest: Latest FCV.
+ :param last_continuous: Last continuous FCV.
+ :param last_lts: Last LTS FCV.
+ :param requires_fcv_tag_list: List of FCVs that we need to generate a tag for.
+ :param fcvs_less_than_latest: List of all FCVs that are less than latest, starting from v4.0.
+ """
+ self.latest = latest
+ self.last_continuous = last_continuous
+ self.last_lts = last_lts
+ self.requires_fcv_tag_list = requires_fcv_tag_list
+ self.fcvs_less_than_latest = fcvs_less_than_latest
+
+
+def calculate_fcv_constants():
+ """Calculate multiversion constants from data files."""
+ mongo_version_yml_file = open(MONGO_VERSION_YAML, 'r')
+ mongo_version_yml = yaml.safe_load(mongo_version_yml_file)
+ mongo_version = mongo_version_yml['mongo_version']
+ latest = Version(re.match(r'^[0-9]+\.[0-9]+', mongo_version).group(0))
+
+ releases_yml_file = open(RELEASES_YAML, 'r')
+ releases_yml = yaml.safe_load(releases_yml_file)
+
+ fcvs = releases_yml['featureCompatibilityVersions']
+ fcvs = list(map(Version, fcvs))
+ lts = releases_yml['longTermSupportReleases']
+ lts = list(map(Version, lts))
+
+ mongo_version_yml_file.close()
+ releases_yml_file.close()
+
+ # Highest release less than latest.
+ last_continuous = fcvs[bisect_left(fcvs, latest) - 1]
+
+ # Highest LTS release less than latest.
+ last_lts = lts[bisect_left(lts, latest) - 1]
+
+ # All FCVs greater than last LTS, up to latest.
+ requires_fcv_tag_list = fcvs[bisect_right(fcvs, last_lts):bisect_right(fcvs, latest)]
+
+ # All FCVs less than latest.
+ fcvs_less_than_latest = fcvs[:bisect_left(fcvs, latest)]
+
+ return FCVConstantValues(latest, last_continuous, last_lts, requires_fcv_tag_list,
+ fcvs_less_than_latest)
+
+
+def version_str(version):
+ """Return a string of the given version in 'MAJOR.MINOR' form."""
+ return '{}.{}'.format(version.major, version.minor)
+
+
+def tag_str(version):
+ """Return a tag for the given version."""
+ return 'requires_fcv_{}{}'.format(version.major, version.minor)
+
+
+def evg_project_str(version):
+ """Return the evergreen project name for the given version."""
+ return 'mongodb-mongo-v{}.{}'.format(version.major, version.minor)
+
+
+fcv_constants = calculate_fcv_constants()
+
+LAST_LTS_BIN_VERSION = version_str(fcv_constants.last_lts)
+LAST_CONTINUOUS_BIN_VERSION = version_str(fcv_constants.last_continuous)
+
+LAST_LTS_FCV = version_str(fcv_constants.last_lts)
+LAST_CONTINUOUS_FCV = version_str(fcv_constants.last_continuous)
+LATEST_FCV = version_str(fcv_constants.latest)
LAST_CONTINUOUS_MONGO_BINARY = "mongo-" + LAST_CONTINUOUS_BIN_VERSION
LAST_CONTINUOUS_MONGOD_BINARY = "mongod-" + LAST_CONTINUOUS_BIN_VERSION
@@ -14,7 +158,12 @@ LAST_LTS_MONGO_BINARY = "mongo-" + LAST_LTS_BIN_VERSION
LAST_LTS_MONGOD_BINARY = "mongod-" + LAST_LTS_BIN_VERSION
LAST_LTS_MONGOS_BINARY = "mongos-" + LAST_LTS_BIN_VERSION
-REQUIRES_FCV_TAG_LATEST = "requires_fcv_51"
+REQUIRES_FCV_TAG_LATEST = tag_str(fcv_constants.latest)
+
+# Generate tags for all FCVS in (lastLTS, latest].
+# All multiversion tests should be run with these tags excluded.
+REQUIRES_FCV_TAG = ",".join([tag_str(fcv) for fcv in fcv_constants.requires_fcv_tag_list])
-# TODO: rename this to REQUIRES_FCV_TAGS (plural) when `requires_fcv_52` is added to it.
-REQUIRES_FCV_TAG = REQUIRES_FCV_TAG_LATEST
+# Generate evergreen project names for all FCVs less than latest.
+EVERGREEN_PROJECTS = ['mongodb-mongo-master']
+EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in fcv_constants.fcvs_less_than_latest])
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index 433152a8bcd..9442634edca 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -4,6 +4,7 @@ import argparse
import shlex
from buildscripts.resmokelib import configure_resmoke
+from buildscripts.resmokelib.generate_fcv_constants import GenerateFCVConstantsPlugin
from buildscripts.resmokelib.hang_analyzer import HangAnalyzerPlugin
from buildscripts.resmokelib.powercycle import PowercyclePlugin
from buildscripts.resmokelib.run import RunPlugin
@@ -20,6 +21,7 @@ _PLUGINS = [
PowercyclePlugin(),
SymbolizerPlugin(),
BisectPlugin(),
+ GenerateFCVConstantsPlugin(),
]
diff --git a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py
index b26159e3b65..54c0a0ea3d8 100755
--- a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py
+++ b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py
@@ -9,7 +9,6 @@ from subprocess import check_output
import requests
from buildscripts.ciconfig import tags as _tags
-from buildscripts.resmokelib import multiversionconstants
from buildscripts.resmokelib.config import MultiversionOptions
from buildscripts.resmokelib.core.programs import get_path_env_var
from buildscripts.resmokelib.utils import is_windows
@@ -88,6 +87,8 @@ def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Log
# Get the state of the backports_required_for_multiversion_tests.yml file for the old
# binary we are running tests against. We do this by using the commit hash from the old
# mongo shell executable.
+ from buildscripts.resmokelib import multiversionconstants
+
shell_version = {
MultiversionOptions.LAST_LTS: multiversionconstants.LAST_LTS_MONGO_BINARY,
MultiversionOptions.LAST_CONTINUOUS: multiversionconstants.LAST_CONTINUOUS_MONGO_BINARY,
diff --git a/buildscripts/resmokelib/setup_multiversion/config.py b/buildscripts/resmokelib/setup_multiversion/config.py
index e81fd445f58..0a39ca5e808 100644
--- a/buildscripts/resmokelib/setup_multiversion/config.py
+++ b/buildscripts/resmokelib/setup_multiversion/config.py
@@ -6,6 +6,8 @@ SETUP_MULTIVERSION_CONFIG = "buildscripts/resmokeconfig/setup_multiversion/setup
# Records the paths of installed multiversion binaries on Windows.
WINDOWS_BIN_PATHS_FILE = "windows_binary_paths.txt"
+USE_EXISTING_RELEASES_FILE = False
+
class Buildvariant:
"""Class represents buildvariant in setup multiversion config."""
diff --git a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py
index 0017f68b5c4..03bd6e1e863 100644
--- a/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py
+++ b/buildscripts/resmokelib/setup_multiversion/setup_multiversion.py
@@ -5,6 +5,8 @@ to include its version) into an install directory and symlinks the binaries
with versions to another directory. This script supports community and
enterprise builds.
"""
+from itertools import chain
+import argparse
import logging
import os
import platform
@@ -18,6 +20,8 @@ import distro
import structlog
import yaml
+from requests.exceptions import HTTPError
+
from buildscripts.resmokelib.plugin import PluginInterface, Subcommand
from buildscripts.resmokelib.setup_multiversion import config, download, github_conn
from buildscripts.resmokelib.utils import evergreen_conn, is_windows
@@ -70,9 +74,9 @@ def get_merge_base_commit(version: str) -> str:
cmd = ["git", "merge-base", "origin/master", f"origin/v{version}"]
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
if result.returncode:
- LOGGER.error("Git merge-base command failed.", cmd=cmd,
+ LOGGER.error("Git merge-base command failed. Falling back to latest master", cmd=cmd,
error=result.stderr.decode("utf-8").strip())
- exit(result.returncode)
+ return None
commit_hash = result.stdout.decode("utf-8").strip()
LOGGER.info("Found merge-base commit.", cmd=cmd, commit=commit_hash)
return commit_hash
@@ -84,8 +88,8 @@ class SetupMultiversion(Subcommand):
# pylint: disable=too-many-instance-attributes
def __init__(self, download_options, install_dir="", link_dir="", mv_platform=None,
edition=None, architecture=None, use_latest=None, versions=None,
- evergreen_config=None, github_oauth_token=None, debug=None,
- ignore_failed_push=False):
+ install_last_lts=None, install_last_continuous=None, evergreen_config=None,
+ github_oauth_token=None, debug=None, ignore_failed_push=False):
"""Initialize."""
setup_logging(debug)
self.install_dir = os.path.abspath(install_dir)
@@ -97,6 +101,8 @@ class SetupMultiversion(Subcommand):
self.architecture = architecture.lower() if architecture else None
self.use_latest = use_latest
self.versions = versions
+ self.install_last_lts = install_last_lts
+ self.install_last_continuous = install_last_continuous
self.ignore_failed_push = ignore_failed_push
self.download_binaries = download_options.download_binaries
@@ -131,6 +137,13 @@ class SetupMultiversion(Subcommand):
def execute(self):
"""Execute setup multiversion mongodb."""
+ from buildscripts.resmokelib import multiversionconstants
+
+ if self.install_last_lts:
+ self.versions.append(multiversionconstants.LAST_LTS_FCV)
+ if self.install_last_continuous:
+ self.versions.append(multiversionconstants.LAST_CONTINUOUS_FCV)
+ self.versions = list(set(self.versions))
for version in self.versions:
LOGGER.info("Setting up version.", version=version)
@@ -218,25 +231,26 @@ class SetupMultiversion(Subcommand):
"""Return latest urls."""
urls = {}
+ # Assuming that project names contain <major>.<minor> version
evg_project = f"mongodb-mongo-v{version}"
if version == "master":
evg_project = "mongodb-mongo-master"
- if evg_project not in self.config.evergreen_projects:
- return urls
-
- LOGGER.debug("Found evergreen project.", evergreen_project=evg_project)
- # Assuming that project names contain <major>.<minor> version
- major_minor_version = version
-
- buildvariant_name = self.get_buildvariant_name(major_minor_version)
- LOGGER.debug("Found buildvariant.", buildvariant_name=buildvariant_name)
-
evg_versions = evergreen_conn.get_evergreen_versions(self.evg_api, evg_project)
+ evg_version = None
+ try:
+ evg_version = next(evg_versions)
+ except HTTPError as err:
+ # Evergreen currently returns 500 if the version does not exist.
+ # TODO (SERVER-59675): Remove the check for 500 once evergreen returns 404 instead.
+ if not (err.response.status_code == 500 or err.response.status_code == 404):
+ raise
+ buildvariant_name = self.get_buildvariant_name(version)
+ LOGGER.debug("Found buildvariant.", buildvariant_name=buildvariant_name)
found_start_revision = start_from_revision is None
- for evg_version in evg_versions:
+ for evg_version in chain(iter([evg_version]), evg_versions):
# Skip all versions until we get the revision we should start looking from
if found_start_revision is False and evg_version.revision != start_from_revision:
LOGGER.warning("Skipping evergreen version.", evg_version=evg_version)
@@ -260,13 +274,12 @@ class SetupMultiversion(Subcommand):
def get_urls(self, version: str, buildvariant_name: Optional[str] = None) -> Dict[str, Any]:
"""Return multiversion urls for a given version (as binary version or commit hash or evergreen_version_id)."""
- evg_version = evergreen_conn.get_evergreen_version(self.config, self.evg_api, version)
+ evg_version = evergreen_conn.get_evergreen_version(self.evg_api, version)
if evg_version is None:
git_tag, commit_hash = github_conn.get_git_tag_and_commit(self.github_oauth_token,
version)
LOGGER.info("Found git attributes.", git_tag=git_tag, commit_hash=commit_hash)
- evg_version = evergreen_conn.get_evergreen_version(self.config, self.evg_api,
- commit_hash)
+ evg_version = evergreen_conn.get_evergreen_version(self.evg_api, commit_hash)
if evg_version is None:
return {}
@@ -362,10 +375,14 @@ class SetupMultiversionPlugin(PluginInterface):
da=args.download_artifacts,
dv=args.download_python_venv)
+ if args.use_existing_releases_file:
+ config.USE_EXISTING_RELEASES_FILE = True
+
return SetupMultiversion(
install_dir=args.install_dir, link_dir=args.link_dir, mv_platform=args.platform,
edition=args.edition, architecture=args.architecture, use_latest=args.use_latest,
- versions=args.versions, download_options=download_options,
+ versions=args.versions, install_last_lts=args.install_last_lts,
+ install_last_continuous=args.install_last_continuous, download_options=download_options,
evergreen_config=args.evergreen_config, github_oauth_token=args.github_oauth_token,
ignore_failed_push=(not args.require_push), debug=args.debug)
@@ -400,6 +417,11 @@ class SetupMultiversionPlugin(PluginInterface):
"Binary version examples: 4.0, 4.0.1, 4.0.0-rc0. If 'rc' is included in the version name, "
"we'll use the exact rc, otherwise we'll pull the highest non-rc version compatible with the "
"version specified.")
+ parser.add_argument("--installLastLTS", dest="install_last_lts", action="store_true",
+ help="If specified, the last LTS version will be installed")
+ parser.add_argument("--installLastContinuous", dest="install_last_continuous",
+ action="store_true",
+ help="If specified, the last continuous version will be installed")
parser.add_argument("-db", "--downloadBinaries", dest="download_binaries",
action="store_true", default=True,
@@ -427,6 +449,11 @@ class SetupMultiversionPlugin(PluginInterface):
parser.add_argument(
"-rp", "--require-push", dest="require_push", action="store_true", default=False,
help="Require the push task to be successful for assets to be downloaded")
+ # Hidden flag that determines if we should generate a new releases yaml file. This flag
+ # should be set to True if we are invoking setup_multiversion multiple times in parallel,
+ # to prevent multiple processes from modifying the releases yaml file simultaneously.
+ parser.add_argument("--useExistingReleasesFile", dest="use_existing_releases_file",
+ action="store_true", default=False, help=argparse.SUPPRESS)
def add_subcommand(self, subparsers):
"""Create and add the parser for the subcommand."""
diff --git a/buildscripts/resmokelib/testing/fixtures/_builder.py b/buildscripts/resmokelib/testing/fixtures/_builder.py
index f3d90fc4240..7cf4bf0ef38 100644
--- a/buildscripts/resmokelib/testing/fixtures/_builder.py
+++ b/buildscripts/resmokelib/testing/fixtures/_builder.py
@@ -7,7 +7,7 @@ from git import Repo
import buildscripts.resmokelib.utils.registry as registry
import buildscripts.resmokelib.config as config
-from buildscripts.resmokelib import errors, multiversionconstants
+from buildscripts.resmokelib import errors
from buildscripts.resmokelib.utils import default_if_none
from buildscripts.resmokelib.utils import autoloader
from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib
@@ -86,6 +86,7 @@ class ReplSetBuilder(FixtureBuilder):
num_nodes = kwargs["num_nodes"]
latest_mongod = mongod_executable
+ from buildscripts.resmokelib import multiversionconstants
fcv = multiversionconstants.LATEST_FCV
executables = {BinVersionEnum.NEW: latest_mongod}
diff --git a/buildscripts/resmokelib/testing/fixtures/fixturelib.py b/buildscripts/resmokelib/testing/fixtures/fixturelib.py
index f38b71e944f..60d01badef8 100644
--- a/buildscripts/resmokelib/testing/fixtures/fixturelib.py
+++ b/buildscripts/resmokelib/testing/fixtures/fixturelib.py
@@ -8,7 +8,6 @@ from buildscripts.resmokelib import logging
from buildscripts.resmokelib.core import network
from buildscripts.resmokelib.utils.history import make_historic as _make_historic
from buildscripts.resmokelib.testing.fixtures import _builder
-from buildscripts.resmokelib.multiversionconstants import LAST_LTS_MONGOD_BINARY, LAST_LTS_MONGOS_BINARY, LAST_CONTINUOUS_MONGOD_BINARY, LAST_CONTINUOUS_MONGOS_BINARY
class FixtureLib(object):
@@ -101,6 +100,8 @@ class _FixtureConfig(object): # pylint: disable=too-many-instance-attributes
def __init__(self):
"""Initialize FixtureConfig, setting values."""
+ from buildscripts.resmokelib.multiversionconstants import LAST_LTS_MONGOD_BINARY, LAST_LTS_MONGOS_BINARY, LAST_CONTINUOUS_MONGOD_BINARY, LAST_CONTINUOUS_MONGOS_BINARY
+
# pylint: disable=invalid-name
self.MONGOD_EXECUTABLE = config.MONGOD_EXECUTABLE
self.DEFAULT_MONGOD_EXECUTABLE = config.DEFAULT_MONGOD_EXECUTABLE
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index 025bbfc4ea6..006b19fd6df 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -9,7 +9,6 @@ from typing import List
import pymongo
import pymongo.errors
-import buildscripts.resmokelib.multiversionconstants as multiversion
import buildscripts.resmokelib.utils.registry as registry
_VERSIONS = {} # type: ignore
diff --git a/buildscripts/resmokelib/utils/evergreen_conn.py b/buildscripts/resmokelib/utils/evergreen_conn.py
index 4104c76f4b4..41e3e6e91aa 100644
--- a/buildscripts/resmokelib/utils/evergreen_conn.py
+++ b/buildscripts/resmokelib/utils/evergreen_conn.py
@@ -114,13 +114,15 @@ def get_generic_buildvariant_name(config, major_minor_version):
return generic_buildvariant_name
-def get_evergreen_version(config: SetupMultiversionConfig, evg_api: RetryingEvergreenApi,
- evg_ref: str) -> Optional[Version]:
+def get_evergreen_version(evg_api: RetryingEvergreenApi, evg_ref: str) -> Optional[Version]:
"""Return evergreen version by reference (commit_hash or evergreen_version_id)."""
+ from buildscripts.resmokelib import multiversionconstants
+
# Evergreen reference as evergreen_version_id
evg_refs = [evg_ref]
# Evergreen reference as {project_name}_{commit_hash}
- evg_refs.extend(f"{proj.replace('-', '_')}_{evg_ref}" for proj in config.evergreen_projects)
+ evg_refs.extend(
+ f"{proj.replace('-', '_')}_{evg_ref}" for proj in multiversionconstants.EVERGREEN_PROJECTS)
for ref in evg_refs:
try:
diff --git a/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py b/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py
index e0f6730f3c9..7c843601e90 100644
--- a/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py
+++ b/buildscripts/tests/resmokelib/setup_multiversion/test_setup_multiversion.py
@@ -179,7 +179,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase):
def test_no_compile_artifacts(self, mock_get_compile_artifact_urls, mock_versions_by_project,
mock_version):
mock_version.build_variants_map = {self.buildvariant_name: "build_id"}
- mock_versions_by_project.return_value = [mock_version]
+ mock_versions_by_project.return_value = iter([mock_version])
mock_get_compile_artifact_urls.return_value = {}
urls = self.setup_multiversion.get_latest_urls("4.4")
@@ -196,7 +196,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase):
}
mock_version.build_variants_map = {self.buildvariant_name: "build_id"}
- mock_versions_by_project.return_value = [mock_version]
+ mock_versions_by_project.return_value = iter([mock_version])
mock_get_compile_artifact_urls.return_value = expected_urls
urls = self.setup_multiversion.get_latest_urls("4.4")
@@ -218,7 +218,7 @@ class TestSetupMultiversionGetLatestUrls(TestSetupMultiversionBase):
mock_expected_version.build_variants_map = {self.buildvariant_name: "build_id"}
evg_versions = [mock_version for _ in range(3)]
evg_versions.append(mock_expected_version)
- mock_versions_by_project.return_value = evg_versions
+ mock_versions_by_project.return_value = iter(evg_versions)
mock_get_compile_artifact_urls.side_effect = lambda evg_api, evg_version, buildvariant_name, ignore_failed_push: {
(self.setup_multiversion.evg_api, mock_version, self.buildvariant_name, False): {},
(self.setup_multiversion.evg_api, mock_expected_version, self.buildvariant_name, False):
diff --git a/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py b/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py
index 2de1dbe7008..7bddcc7e4ab 100644
--- a/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py
+++ b/buildscripts/tests/resmokelib/utils/test_evergreen_conn.py
@@ -103,13 +103,6 @@ class TestGetGenericBuildvariantName(unittest.TestCase):
class TestGetEvergreenProjectAndVersion(unittest.TestCase):
- def setUp(self):
- raw_yaml = {"evergreen_projects": [
- "mongodb-mongo-master",
- "mongodb-mongo-v4.4",
- ]}
- self.config = SetupMultiversionConfig(raw_yaml)
-
@patch("evergreen.version.Version")
@patch("evergreen.api.EvergreenApi")
def test_version_by_commit_hash_found(self, mock_evg_api, mock_version):
@@ -123,7 +116,7 @@ class TestGetEvergreenProjectAndVersion(unittest.TestCase):
raise HTTPError()
mock_evg_api.version_by_id.side_effect = version_by_id_side_effect
- evg_version = evergreen_conn.get_evergreen_version(self.config, mock_evg_api, commit_hash)
+ evg_version = evergreen_conn.get_evergreen_version(mock_evg_api, commit_hash)
self.assertEqual(mock_version, evg_version)
self.assertEqual(mock_version.version_id, expected_evergreen_version_id)
@@ -139,15 +132,14 @@ class TestGetEvergreenProjectAndVersion(unittest.TestCase):
raise HTTPError()
mock_evg_api.version_by_id.side_effect = version_by_id_side_effect
- evg_version = evergreen_conn.get_evergreen_version(self.config, mock_evg_api,
- evergreen_version_id)
+ evg_version = evergreen_conn.get_evergreen_version(mock_evg_api, evergreen_version_id)
self.assertEqual(mock_version, evg_version)
self.assertEqual(mock_version.version_id, evergreen_version_id)
@patch("evergreen.api.EvergreenApi")
def test_version_not_found(self, mock_evg_api):
mock_evg_api.version_by_id.side_effect = HTTPError
- evg_version = evergreen_conn.get_evergreen_version(self.config, mock_evg_api, "dummy")
+ evg_version = evergreen_conn.get_evergreen_version(mock_evg_api, "dummy")
self.assertIsNone(evg_version)
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 68d229df5d7..2af9a1ef0bc 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -1978,11 +1978,21 @@ tasks:
args:
- "src/evergreen/gen_feature_flags.sh"
+ - *f_expansions_write
+ - command: subprocess.exec
+ type: test
+ params:
+ binary: bash
+ args:
+ - "src/evergreen/get_bin_and_fcv_versions.sh"
+
- command: archive.targz_pack
params:
target: "artifacts.tgz"
source_dir: "src"
include:
+ - ".resmoke_mongo_version.yml"
+ - ".resmoke_mongo_release_values.yml"
- "patch_test_tags.tgz"
- "./build/**.gcno"
- "./etc/*san.suppressions"
@@ -9547,6 +9557,8 @@ buildvariants:
run_on:
- rhel80-small
expansions: &enterprise-rhel-80-64-bit-multiversion-expansions-template
+ # TODO (SERVER-55857): Remove 'requires_fcv_51' once tests are filtered out using the
+ # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'.
test_flags: >-
--excludeWithAnyTags=requires_fcv_51,multiversion_incompatible,disabled_due_to_server_58295
compile_flags: >-
@@ -9576,6 +9588,8 @@ buildvariants:
display_name: "Enterprise RHEL 8.0 (implicit multiversion & all feature flags)"
expansions:
<<: *enterprise-rhel-80-64-bit-multiversion-expansions-template
+ # TODO (SERVER-55857): Remove 'requires_fcv_51' once tests are filtered out using the
+ # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'.
test_flags: >-
--excludeWithAnyTags=requires_fcv_51,multiversion_incompatible,disabled_due_to_server_58295
--runAllFeatureFlagTests
diff --git a/evergreen/get_bin_and_fcv_versions.sh b/evergreen/get_bin_and_fcv_versions.sh
new file mode 100644
index 00000000000..0bc91fe4baa
--- /dev/null
+++ b/evergreen/get_bin_and_fcv_versions.sh
@@ -0,0 +1,9 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+
+activate_venv
+python buildscripts/resmoke.py generate-fcv-constants
diff --git a/evergreen/multiversion_setup.sh b/evergreen/multiversion_setup.sh
index 5af1ebcf725..64bd44a6aa7 100644
--- a/evergreen/multiversion_setup.sh
+++ b/evergreen/multiversion_setup.sh
@@ -70,4 +70,6 @@ $python buildscripts/resmoke.py setup-multiversion \
--edition $edition \
--platform $platform \
--architecture $architecture \
- --useLatest 4.4 5.0
+ --useLatest \
+ --installLastLTS \
+ --installLastContinuous 4.4
diff --git a/src/mongo/db/wire_version.h b/src/mongo/db/wire_version.h
index 61b52dc190b..61e5aab759e 100644
--- a/src/mongo/db/wire_version.h
+++ b/src/mongo/db/wire_version.h
@@ -99,8 +99,8 @@ enum WireVersion {
// Supports features available from 5.1 and onwards.
WIRE_VERSION_51 = 14,
- // Calculate the latest wire version using the number of releases since 4.4.
- LATEST_WIRE_VERSION = RESUMABLE_INITIAL_SYNC + multiversion::kSince_4_4,
+ // Calculate the latest wire version using the number of releases since 4.0.
+ LATEST_WIRE_VERSION = REPLICA_SET_TRANSACTIONS + multiversion::kSince_4_0,
// Set this to LATEST_WIRE_VERSION - 1.
LAST_CONT_WIRE_VERSION = LATEST_WIRE_VERSION - 1,
diff --git a/src/mongo/idl/feature_flag_test.cpp b/src/mongo/idl/feature_flag_test.cpp
index e4a41e48c67..10ecfa9824f 100644
--- a/src/mongo/idl/feature_flag_test.cpp
+++ b/src/mongo/idl/feature_flag_test.cpp
@@ -115,9 +115,12 @@ TEST_F(FeatureFlagTest, ServerStatus) {
_featureFlagBlender->append(nullptr, builder, "blender");
- ASSERT_BSONOBJ_EQ(builder.obj(),
- BSON("blender" << BSON("value" << true << "version"
- << "5.1")));
+ ASSERT_BSONOBJ_EQ(
+ builder.obj(),
+ // (Generic FCV reference): feature flag test.
+ BSON("blender" << BSON("value"
+ << true << "version"
+ << multiversion::toString(multiversion::GenericFCV::kLatest))));
}
{
diff --git a/src/mongo/util/version/releases.h.tpl b/src/mongo/util/version/releases.h.tpl
index 4232e1c6222..787494840cf 100644
--- a/src/mongo/util/version/releases.h.tpl
+++ b/src/mongo/util/version/releases.h.tpl
@@ -161,8 +161,8 @@ enum class FeatureCompatibilityVersion {
#end for
};
-## Calculate number of versions since v4.4.
-constexpr size_t kSince_$underscores(Version('4.4')) = ${bisect_left(fcvs, latest)};
+## Calculate number of versions since v4.0.
+constexpr size_t kSince_$underscores(Version('4.0')) = ${bisect_left(fcvs, latest)};
// Last LTS was "$last_lts".
constexpr size_t kSinceLastLTS = ${bisect_left(fcvs, latest) - bisect_left(fcvs, last_lts)};
diff --git a/src/mongo/util/version/releases.yml b/src/mongo/util/version/releases.yml
index ea37d239bb1..d55c84dd4ad 100644
--- a/src/mongo/util/version/releases.yml
+++ b/src/mongo/util/version/releases.yml
@@ -3,8 +3,10 @@
# multiversion_constants.h. Please consult the STAR team before
# modifying this file.
-# All FCVs starting with 4.4.
+# All FCVs starting with 4.0.
featureCompatibilityVersions:
+ - "4.0"
+ - "4.2"
- "4.4"
- "4.7"
- "4.8"