diff options
| -rwxr-xr-x | buildscripts/evergreen_gen_fuzzer_tests.py | 1 | ||||
| -rwxr-xr-x | buildscripts/evergreen_gen_multiversion_tests.py | 1 | ||||
| -rw-r--r-- | buildscripts/patch_builds/task_generation.py | 1 | ||||
| -rw-r--r-- | buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml (renamed from etc/setup_multiversion_config.yml) | 2 | ||||
| -rw-r--r-- | buildscripts/resmokelib/setup_multiversion/config.py | 2 | ||||
| -rwxr-xr-x | buildscripts/setup_multiversion_mongodb.py | 484 | ||||
| -rw-r--r-- | buildscripts/tests/patch_builds/test_task_generation.py | 8 | ||||
| -rw-r--r-- | buildscripts/tests/test_burn_in_tests_multiversion.py | 2 | ||||
| -rw-r--r-- | buildscripts/tests/test_evergreen_gen_fuzzer_tests.py | 4 | ||||
| -rw-r--r-- | etc/evergreen.yml | 15 | ||||
| -rw-r--r-- | jstests/hooks/run_fuzzer_restore_settings.js | 16 | ||||
| -rw-r--r-- | jstests/multiVersion/genericSetFCVUsage/major_version_upgrade.js | 94 |
12 files changed, 41 insertions, 589 deletions
diff --git a/buildscripts/evergreen_gen_fuzzer_tests.py b/buildscripts/evergreen_gen_fuzzer_tests.py index eb750149348..1c4b90e6b28 100755 --- a/buildscripts/evergreen_gen_fuzzer_tests.py +++ b/buildscripts/evergreen_gen_fuzzer_tests.py @@ -105,6 +105,7 @@ def build_fuzzer_sub_task(task_name: str, task_index: int, options: ConfigOption commands = [ FunctionCall("do setup"), + FunctionCall("configure evergreen api credentials") if options.use_multiversion else None, FunctionCall("do multiversion setup") if options.use_multiversion else None, FunctionCall("setup jstestfuzz"), FunctionCall("run jstestfuzz", run_jstestfuzz_vars), diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py index 11e3f64a93f..a2334a0d794 100755 --- a/buildscripts/evergreen_gen_multiversion_tests.py +++ b/buildscripts/evergreen_gen_multiversion_tests.py @@ -178,6 +178,7 @@ class EvergreenMultiversionConfigGenerator(object): commands = [ FunctionCall("do setup"), # Fetch and download the proper mongod binaries before running multiversion tests. + FunctionCall("configure evergreen api credentials"), FunctionCall("do multiversion setup"), FunctionCall("run generated tests", run_tests_vars), ] diff --git a/buildscripts/patch_builds/task_generation.py b/buildscripts/patch_builds/task_generation.py index 2ca8a7e184f..c4ad822fc0e 100644 --- a/buildscripts/patch_builds/task_generation.py +++ b/buildscripts/patch_builds/task_generation.py @@ -26,6 +26,7 @@ def resmoke_commands(run_tests_fn_name: str, run_tests_vars: Dict[str, Any], commands = [ timeout_info.cmd, FunctionCall("do setup"), + FunctionCall("configure evergreen api credentials") if use_multiversion else None, FunctionCall("do multiversion setup") if use_multiversion else None, FunctionCall(run_tests_fn_name, run_tests_vars), ] diff --git a/etc/setup_multiversion_config.yml b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml index 642f58db157..4e58821c427 100644 --- a/etc/setup_multiversion_config.yml +++ b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml @@ -94,7 +94,7 @@ evergreen_buildvariants: - name: enterprise-linux-64-amazon-ami edition: enterprise - platform: amazon + platform: amzn64 architecture: x86_64 - name: amazon2 diff --git a/buildscripts/resmokelib/setup_multiversion/config.py b/buildscripts/resmokelib/setup_multiversion/config.py index 72cb82a4fb3..f2c63921ef3 100644 --- a/buildscripts/resmokelib/setup_multiversion/config.py +++ b/buildscripts/resmokelib/setup_multiversion/config.py @@ -1,7 +1,7 @@ """Setup multiversion config.""" from typing import List -SETUP_MULTIVERSION_CONFIG = "etc/setup_multiversion_config.yml" +SETUP_MULTIVERSION_CONFIG = "buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml" class Buildvariant: diff --git a/buildscripts/setup_multiversion_mongodb.py b/buildscripts/setup_multiversion_mongodb.py index 94e5d056816..3f089504d1d 100755 --- a/buildscripts/setup_multiversion_mongodb.py +++ b/buildscripts/setup_multiversion_mongodb.py @@ -1,481 +1,9 @@ #!/usr/bin/env python3 -"""Install multiple versions of MongoDB on a machine.""" - -import contextlib -import errno -import json -import optparse -import os -import re -import shutil -import signal -import sys -import tarfile -import tempfile -import threading -import traceback -import urllib.parse -import zipfile - -import requests -import requests.exceptions - - -def dump_stacks(_signal_num, _frame): # pylint: disable=unused-argument - """Dump stacks when SIGUSR1 is received.""" - print("======================================") - print("DUMPING STACKS due to SIGUSR1 signal") - print("======================================") - threads = threading.enumerate() - - print("Total Threads: {:d}".format(len(threads))) - - for tid, stack in list(sys._current_frames().items()): # pylint: disable=protected-access - print("Thread {:d}".format(tid)) - print("".join(traceback.format_stack(stack))) - print("======================================") - - -def get_version_parts(version, for_sorting=False): - """Return a list containing the components of the version string as numeric values. - - This function can be used for numeric sorting - of version strings such as '2.6.0-rc1' and '2.4.0' when the - 'for_sorting' parameter is specified as true. - """ - - rc_offset = -100 - version_parts = re.split(r"\.|-", version) - - if version_parts[-1] == "pre": - # Prior to improvements for how the version string is managed within the server - # (SERVER-17782), the binary archives would contain a trailing "-pre". - version_parts.pop() - - if version_parts[-1].startswith("rc"): - # RC versions are weighted down to allow future RCs and general - # releases to be sorted in ascending order (e.g., 2.6.0-rc1, - # 2.6.0-rc2, 2.6.0). - version_parts[-1] = int(version_parts[-1][2:]) + rc_offset - elif version_parts[0].startswith("v") and version_parts[-1] == "latest": - version_parts[0] = version_parts[0][1:] - # The "<branchname>-latest" versions are weighted the highest when a particular major - # release is requested. - version_parts[-1] = float("inf") - elif for_sorting: - # We want to have the number of components in the resulting version parts match the number - # of components in the 'version' string if we aren't going to be using them for sorting. - # Otherwise, we append an additional 0 to non-RC releases so that version lists like - # [2, 6, 0, -100] and [2, 6, 0, 0] sort in ascending order. - version_parts.append(0) - - return [float(part) for part in version_parts] - - -def download_file(url, file_name, download_retries=5): - """Return True if download was successful. Raises error if download fails.""" - - while download_retries > 0: - - with requests.Session() as session: - adapter = requests.adapters.HTTPAdapter(max_retries=download_retries) - session.mount(url, adapter) - response = session.get(url, stream=True) - response.raise_for_status() - - with open(file_name, "wb") as file_handle: - try: - for block in response.iter_content(1024 * 1000): - file_handle.write(block) - except requests.exceptions.ChunkedEncodingError as err: - download_retries -= 1 - if download_retries == 0: - raise Exception("Incomplete download for URL {}: {}".format(url, err)) - continue - - # Check if file download was completed. - if "Content-length" in response.headers: - url_content_length = int(response.headers["Content-length"]) - file_size = os.path.getsize(file_name) - # Retry download if file_size has an unexpected size. - if url_content_length != file_size: - download_retries -= 1 - if download_retries == 0: - raise Exception("Downloaded file size ({} bytes) doesn't match content length" - "({} bytes) for URL {}".format(file_size, url_content_length, - url)) - continue - - return True - - raise Exception("Unknown download problem for {} to file {}".format(url, file_name)) - - -class MultiVersionDownloader(object): # pylint: disable=too-many-instance-attributes - """Class to support multiversion downloads.""" - - def __init__( # pylint: disable=too-many-arguments - self, install_dir, link_dir, edition, platform, architecture, use_latest=False): - """Initialize MultiVersionDownloader.""" - self.install_dir = install_dir - self.link_dir = link_dir - self.edition = edition.lower() - self.platform = platform.lower() - self.architecture = architecture.lower() - self.generic_platform = "linux" - self.generic_architecture = "x86_64" - self.use_latest = use_latest - self._links = None - self._generic_links = None - - @property - def generic_links(self): - """Get a list of generic links.""" - if self._generic_links is None: - self._links, self._generic_links = self.download_links() - return self._generic_links - - @property - def links(self): - """Get a list of links.""" - if self._links is None: - self._links, self._generic_links = self.download_links() - return self._links - - @staticmethod - def is_major_minor_version(version): - """Return True if the version is specified as M.m.""" - if re.match(r"^\d+?\.\d+?$", version) is None: - return False - return True - - def download_links(self): - """Return the download and generic download links.""" - temp_file = tempfile.mktemp() - download_file("https://downloads.mongodb.org/full.json", temp_file) - with open(temp_file) as file_handle: - full_json = json.load(file_handle) - os.remove(temp_file) - if "versions" not in full_json: - raise Exception("No versions field in JSON: \n" + str(full_json)) - - links = {} - generic_links = {} - # The generic target contains a platform and architecture. - generic_target = "{}_{}".format(self.generic_platform, self.generic_architecture) - for json_version in full_json["versions"]: - if "version" not in json_version or 'downloads' not in json_version: - continue - version = json_version["version"] - for download in json_version["downloads"]: - if "target" not in download or "edition" not in download: - continue - if (download["target"].lower() == self.platform - and download["arch"].lower() == self.architecture - and download["edition"].lower() == self.edition): - links[version] = download["archive"]["url"] - elif (download["target"].lower() == generic_target - and download["edition"].lower() == "base"): - generic_links[version] = download["archive"]["url"] - - return links, generic_links - - def download_install(self, version): - """Download and install the version specified.""" - dl_file = self.download_version(version) - if dl_file: - installed_dir = self.uncompress_download(dl_file) - self.symlink_version(version, installed_dir) - - def download_version(self, version): # pylint: disable=too-many-branches,too-many-locals,too-many-statements - """Download the version specified and return file location. - - If no download occurs, file location is None. - """ - - try: - os.makedirs(self.install_dir) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(self.install_dir): - pass - else: - raise - - urls = [] - requested_version_parts = get_version_parts(version) - for link_version, link_url in self.links.items(): - link_version_parts = get_version_parts(link_version) - if link_version_parts[:len(requested_version_parts)] == requested_version_parts: - # The 'link_version' is a candidate for the requested 'version' if - # (a) it is a prefix of the requested version, or if - # (b) it is the "<branchname>-latest" version and the requested version is for a - # particular major release. - # This is equivalent to the 'link_version' having components equal to all of the - # version parts that make up 'version'. - if "-" in version: - # The requested 'version' contains a hyphen, so we only consider exact matches - # to that version. - if link_version != version: - continue - urls.append((link_version, link_url)) - - if not urls: - print( - "Cannot find a link for version {}, versions {} found.".format(version, self.links), - file=sys.stderr) - for ver, generic_url in self.generic_links.items(): - parts = get_version_parts(ver) - if parts[:len(requested_version_parts)] == requested_version_parts: - if "-" in version and ver != version: - continue - urls.append((ver, generic_url)) - if not urls: - if version not in ["4.7", "4.8"]: - raise Exception( - "No fall-back generic link available or version {}.".format(version)) - - print( - "manually constructing URL for {} releases; please note that only the latest release is available" - .format(version)) - - url_template = "https://downloads.mongodb.com/{bucket}/mongodb-{os_family}-{arch}-enterprise-{platform}-{version}.{suffix}" - - suffix = "tgz" - - if self.platform == "windows": - os_family = self.platform - bucket = self.platform - suffix = "zip" - elif self.platform == "osx": - os_family = "macos" - bucket = self.platform - else: - os_family = "linux" - bucket = "linux" - - if self.platform == "osx" or self.platform == "windows": - # macOS and Windows URLs don't have the platform portion. - platform = "" - elif self.platform == "amazon": - # The community version and the enterprise version have different platform names. - platform = "amzn64" - else: - platform = self.platform - - url = url_template.format(bucket=bucket, os_family=os_family, - arch=self.architecture, platform=platform, - version=version, suffix=suffix) - - # URLs with missing sections lead to double dashes. - url = url.replace("--", "-") - urls.append((version, url)) - else: - print("Falling back to generic architecture.") - - urls.sort(key=lambda link: get_version_parts(link[0], for_sorting=True)) - full_version = urls[-1][0] - url = urls[-1][1] - extract_dir = url.split("/")[-1][:-4] - file_suffix = os.path.splitext(urllib.parse.urlparse(url).path)[1] - - # Only download if we don't already have the directory. - # Note, we cannot detect if 'latest' has already been downloaded, as the name - # of the 'extract_dir' cannot be derived from the URL, since it contains the githash. - already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir)) - if already_downloaded: - print("Skipping download for version {} ({}) since the dest already exists '{}'".format( - version, full_version, extract_dir)) - return None - else: - temp_file = tempfile.mktemp(suffix=file_suffix) - - latest_downloaded = False - # We try to download 'v<version>-latest' if the 'version' is specified - # as Major.minor. If that fails, we then try to download the version that - # was specified. - if self.use_latest and self.is_major_minor_version(version): - latest_version = "v{}-latest".format(version) - latest_url = url.replace(full_version, latest_version) - print("Trying to download {}...".format(latest_version)) - print("Download url is {}".format(latest_url)) - try: - download_file(latest_url, temp_file) - full_version = latest_version - latest_downloaded = True - except requests.exceptions.HTTPError: - print("Failed to download {}".format(latest_url)) - - if not latest_downloaded: - print("Downloading data for version {} ({})...".format(version, full_version)) - print("Download url is {}".format(url)) - download_file(url, temp_file) - return temp_file - - def uncompress_download(self, dl_file): - """Download the version specified and return root of extracted directory.""" - - print("Uncompressing data to {}...".format(self.install_dir)) - first_file = "" - temp_dir = tempfile.mkdtemp() - _, file_suffix = os.path.splitext(dl_file) - if file_suffix == ".zip": - # Support .zip downloads, used for Windows binaries. - with zipfile.ZipFile(dl_file) as zip_handle: - # Use the name of the root directory in the archive as the name of the directory - # to extract the binaries into inside 'self.install_dir'. The name of the root - # directory nearly always matches the parsed URL text, with the exception of - # versions such as "v3.2-latest" that instead contain the githash. - first_file = zip_handle.namelist()[0] - zip_handle.extractall(temp_dir) - elif file_suffix == ".tgz": - # Support .tgz downloads, used for Linux binaries. - with contextlib.closing(tarfile.open(dl_file, "r:gz")) as tar_handle: - # Use the name of the root directory in the archive as the name of the directory - # to extract the binaries into inside 'self.install_dir'. The name of the root - # directory nearly always matches the parsed URL text, with the exception of - # versions such as "v3.2-latest" that instead contain the githash. - first_file = tar_handle.getnames()[0] - tar_handle.extractall(path=temp_dir) - else: - raise Exception("Unsupported file extension {}".format(file_suffix)) - - # Sometimes the zip will contain the root directory as the first file and - # os.path.dirname() will return ''. - extract_dir = os.path.dirname(first_file) - if not extract_dir: - extract_dir = first_file - temp_install_dir = os.path.join(temp_dir, extract_dir) - - # We may not have been able to determine whether we already downloaded the requested - # version due to the ambiguity in the parsed URL text, so we check for it again using - # the adjusted 'extract_dir' value. - already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir)) - if not already_downloaded: - shutil.move(temp_install_dir, self.install_dir) - - shutil.rmtree(temp_dir) - os.remove(dl_file) - - return os.path.abspath(os.path.join(self.install_dir, extract_dir)) - - def symlink_version(self, version, installed_dir): - """Symlink the binaries in the 'installed_dir' to the 'link_dir'.""" - try: - os.makedirs(self.link_dir) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(self.link_dir): - pass - else: - raise - - for executable in os.listdir(os.path.join(installed_dir, "bin")): - - executable_name, executable_extension = os.path.splitext(executable) - link_name = "{}-{}{}".format(executable_name, version, executable_extension) - - try: - executable = os.path.join(installed_dir, "bin", executable) - executable_link = os.path.join(self.link_dir, link_name) - if os.name == "nt": - # os.symlink is not supported on Windows, use a direct method instead. - def symlink_ms(source, link_name): - """Provide symlink for Windows.""" - import ctypes - csl = ctypes.windll.kernel32.CreateSymbolicLinkW - csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) - csl.restype = ctypes.c_ubyte - flags = 1 if os.path.isdir(source) else 0 - if csl(link_name, source.replace("/", "\\"), flags) == 0: - raise ctypes.WinError() - - os.symlink = symlink_ms - os.symlink(executable, executable_link) - except OSError as exc: - if exc.errno == errno.EEXIST: - pass - else: - raise - - -def main(): - """Execute Main program.""" - - # Listen for SIGUSR1 and dump stack if received. - try: - signal.signal(signal.SIGUSR1, dump_stacks) - except AttributeError: - print("Cannot catch signals on Windows") - - parser = optparse.OptionParser(usage=""" -Downloads and installs particular mongodb versions (each binary is renamed -to include its version) into an install directory and symlinks the binaries -with versions to another directory. This script supports community and -enterprise builds. - -Usage: setup_multiversion_mongodb.py [options] ver1 [vers2 ...] - -Ex: setup_multiversion_mongodb.py --installDir ./install - --linkDir ./link - --edition base - --platform Linux 2.0.6 2.0.3-rc0 - --architecture x86_64 - 2.0 2.2 2.3 -Ex: setup_multiversion_mongodb.py --installDir ./install - --linkDir ./link - --edition enterprise - --platform osx - 2.4 2.2 - -After running the script you will have a directory structure like this: - ./install/[mongodb-osx-x86_64-2.4.9, mongodb-osx-x86_64-2.2.7] - ./link/[mongod-2.4.9, mongod-2.2.7, mongo-2.4.9...] - -You should then add ./link/ to your path so multi-version tests will work. - -Note: If "rc" is included in the version name, we'll use the exact rc, otherwise -we'll pull the highest non-rc version compatible with the version specified. -""") - - parser.add_option("-i", "--installDir", dest="install_dir", - help="Directory to install the download archive. [REQUIRED]", default=None) - parser.add_option( - "-l", "--linkDir", dest="link_dir", - help=("Directory to contain links to all binaries for each version in" - " the install directory. [REQUIRED]"), default=None) - editions = ["base", "enterprise", "targeted"] - parser.add_option( - "-e", "--edition", dest="edition", choices=editions, - help=("Edition of the build to download, choose from {}, [default:" - " '%default'].".format(editions)), default="base") - parser.add_option( - "-p", "--platform", dest="platform", - help=("Platform to download [REQUIRED]. Examples include: 'linux'," - " 'osx', 'rhel62', 'windows'."), default=None) - parser.add_option( - "-a", "--architecture", dest="architecture", - help=("Architecture to download, [default: '%default']. Examples include:" - " 'arm64', 'ppc64le', 's390x' and 'x86_64'."), default="x86_64") - parser.add_option( - "-u", "--useLatest", dest="use_latest", action="store_true", - help=("If specified, the latest (nightly) version will be downloaded," - " if it exists, for the version specified. For example, if specifying" - " version 3.2 for download, the nightly version for 3.2 will be" - " downloaded if it exists, otherwise the 'highest' version will be" - " downloaded, i.e., '3.2.17'"), default=False) - - options, versions = parser.parse_args() - - # Check for required options. - if not versions or not options.install_dir or not options.link_dir or not options.platform: - parser.print_help() - parser.exit(1) - - downloader = MultiVersionDownloader(options.install_dir, options.link_dir, options.edition, - options.platform, options.architecture, options.use_latest) - - for version in versions: - downloader.download_install(version) - +"""Stub file pointing users to the new invocation.""" if __name__ == "__main__": - main() + print( + "Hello! It seems you've executed 'buildscripts/setup_multiversion_mongodb.py'. We have\n" + "repackaged the setup multiversion as a subcommand of resmoke. It can now be invoked as\n" + "'./buildscripts/resmoke.py setup-multiversion' with all of the same arguments as before.\n" + "Please use './buildscripts/resmoke.py setup-multiversion --help' for more details.") diff --git a/buildscripts/tests/patch_builds/test_task_generation.py b/buildscripts/tests/patch_builds/test_task_generation.py index 336c86d9c90..e0609d1d6f3 100644 --- a/buildscripts/tests/patch_builds/test_task_generation.py +++ b/buildscripts/tests/patch_builds/test_task_generation.py @@ -25,8 +25,8 @@ class TestResmokeCommand(unittest.TestCase): commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info, use_multiversion="multiversion") - # 3 expected command = 1 for setup + 1 for running tests + 1 for multiversion setup. - self.assertEqual(3, len(commands)) + # 4 expected command = 1 for setup + 1 for running tests + 2 for multiversion setup. + self.assertEqual(4, len(commands)) def test_with_timeout(self): run_tests = "run tests" @@ -46,9 +46,9 @@ class TestResmokeCommand(unittest.TestCase): commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info, use_multiversion="multiversion") - # 4 expected command = 1 for setup + 1 for running tests + 1 for multiversion setup + + # 5 expected command = 1 for setup + 1 for running tests + 2 for multiversion setup + # 1 for timeout. - self.assertEqual(4, len(commands)) + self.assertEqual(5, len(commands)) class TestTimeoutInfo(unittest.TestCase): diff --git a/buildscripts/tests/test_burn_in_tests_multiversion.py b/buildscripts/tests/test_burn_in_tests_multiversion.py index 3a334a7fce0..090c9e84358 100644 --- a/buildscripts/tests/test_burn_in_tests_multiversion.py +++ b/buildscripts/tests/test_burn_in_tests_multiversion.py @@ -236,7 +236,7 @@ class TestCreateGenerateTasksConfig(unittest.TestCase): evg_config_dict = shrub_project.as_dict() tasks = evg_config_dict["tasks"] self.assertEqual(n_tasks * n_tests, len(tasks)) - self.assertEqual(multiversion_path, tasks[0]["commands"][2]["vars"]["task_path_suffix"]) + self.assertEqual(multiversion_path, tasks[0]["commands"][3]["vars"]["task_path_suffix"]) class TestGatherTaskInfo(unittest.TestCase): diff --git a/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py b/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py index 012b9777194..c078bd51a6b 100644 --- a/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py +++ b/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py @@ -64,6 +64,6 @@ class TestCreateFuzzerTask(unittest.TestCase): shrub_project = ShrubProject.empty().add_build_variant(build_variant) config = shrub_project.as_dict() - self.assertEqual("do multiversion setup", config["tasks"][0]["commands"][1]["func"]) + self.assertEqual("do multiversion setup", config["tasks"][0]["commands"][2]["func"]) self.assertEqual("/data/multiversion", - config["tasks"][0]["commands"][4]["vars"]["task_path_suffix"]) + config["tasks"][0]["commands"][5]["vars"]["task_path_suffix"]) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 7c6662b4197..eeb4ee55fcd 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1296,16 +1296,17 @@ functions: rm -rf /data/install /data/multiversion edition="${multiversion_edition|base}" - platform="${multiversion_platform|linux}" + platform="${multiversion_platform|linux_x86_64}" architecture="${multiversion_architecture|x86_64}" - $python buildscripts/setup_multiversion_mongodb.py \ + $python buildscripts/resmoke.py setup-multiversion \ --installDir /data/install \ --linkDir /data/multiversion \ --edition $edition \ --platform $platform \ --architecture $architecture \ - --useLatest 3.2 3.4 3.6 4.0 + --githubOauthToken "${github_token}" \ + --useLatest 3.6 4.0 # The platform and architecture for how some of the binaries are reported in # https://downloads.mongodb.org/full.json changed between MongoDB 4.0 and MongoDB 4.2. @@ -1324,12 +1325,13 @@ functions: architecture="${multiversion_architecture_42_or_later}" fi - $python buildscripts/setup_multiversion_mongodb.py \ + $python buildscripts/resmoke.py setup-multiversion \ --installDir /data/install \ --linkDir /data/multiversion \ --edition $edition \ --platform $platform \ --architecture $architecture \ + --githubOauthToken "${github_token}" \ --useLatest 4.2 4.2.1 # The platform and architecture for how some of the binaries are reported in @@ -1349,12 +1351,13 @@ functions: architecture="${multiversion_architecture_44_or_later}" fi - $python buildscripts/setup_multiversion_mongodb.py \ + $python buildscripts/resmoke.py setup-multiversion \ --installDir /data/install \ --linkDir /data/multiversion \ --edition $edition \ --platform $platform \ --architecture $architecture \ + --githubOauthToken "${github_token}" \ --useLatest 4.4 4.7 4.8 "execute resmoke tests": &execute_resmoke_tests @@ -4935,6 +4938,7 @@ tasks: suite: jstestfuzz_replication is_jstestfuzz: true use_multiversion: /data/multiversion + task_path_suffix: /data/multiversion npm_command: jstestfuzz ## jstestfuzz initial sync replica set ## @@ -4991,6 +4995,7 @@ tasks: suite: jstestfuzz_sharded is_jstestfuzz: true use_multiversion: /data/multiversion + task_path_suffix: /data/multiversion npm_command: jstestfuzz ## jstestfuzz sharded cluster causal consistency ## diff --git a/jstests/hooks/run_fuzzer_restore_settings.js b/jstests/hooks/run_fuzzer_restore_settings.js index a2858f4e915..3a8318ea2bd 100644 --- a/jstests/hooks/run_fuzzer_restore_settings.js +++ b/jstests/hooks/run_fuzzer_restore_settings.js @@ -1,15 +1,25 @@ (function() { 'use strict'; -assert.commandWorked(db.adminCommand({setParameter: 1, requireApiVersion: false, apiVersion: "1"})); +// Ignore "attempted to set unrecognized parameter" error from setParameter on old versions +let res; +try { + res = db.adminCommand({setParameter: 1, requireApiVersion: false, apiVersion: "1"}); +} catch (e) { + print("Failed to set requireApiVersion."); + print(e); +} +if (res && res.ok === 0 && !res.errmsg.includes("attempted to set unrecognized parameter")) { + assert.commandWorked(res); +} // Unsetting read/write settings. This command will also cause the server to refresh and get -// the new settings. A standalone or mongos will return an error; ignore it. +// the new settings. A standalone, mongos or old version will return an error; ignore it. const result = db.adminCommand({ setDefaultRWConcern: 1, defaultReadConcern: {}, defaultWriteConcern: {}, writeConcern: {w: 1} }); -assert.commandWorkedOrFailedWithCode(result, [51300, 51301]); +assert.commandWorkedOrFailedWithCode(result, [51300, 51301, 40415]); })(); diff --git a/jstests/multiVersion/genericSetFCVUsage/major_version_upgrade.js b/jstests/multiVersion/genericSetFCVUsage/major_version_upgrade.js index f0a163105aa..a6a80a61c44 100644 --- a/jstests/multiVersion/genericSetFCVUsage/major_version_upgrade.js +++ b/jstests/multiVersion/genericSetFCVUsage/major_version_upgrade.js @@ -31,8 +31,6 @@ const defaultOptions = { // TODO SERVER-26792: In the future, we should have a common place from which both the // multiversion setup procedure and this test get information about supported major releases. const versions = [ - {binVersion: '3.2', testCollection: 'three_two'}, - {binVersion: '3.4', featureCompatibilityVersion: '3.4', testCollection: 'three_four'}, {binVersion: '3.6', featureCompatibilityVersion: '3.6', testCollection: 'three_six'}, {binVersion: '4.0', featureCompatibilityVersion: '4.0', testCollection: 'four_zero'}, {binVersion: '4.2', featureCompatibilityVersion: '4.2', testCollection: 'four_two'}, @@ -42,80 +40,6 @@ const versions = [ {binVersion: 'latest', featureCompatibilityVersion: latestFCV, testCollection: 'latest'}, ]; -// These key patterns are considered valid for existing v:0 and v:1 indexes, but are considered -// invalid for v:2 indexes or new index builds. -var invalidIndexSpecs = [ - {a: 0}, - {a: NaN}, - {a: true}, -]; - -// When running the oldest supported version, insert indexes with bad key patterns. -function insertBadIndexes(testDB) { - invalidIndexSpecs.forEach((spec) => { - // Generate a unique and identifiable collection name. - let collName = 'bad_index_' + tojson(spec.a); - assert.commandWorked(testDB[collName].createIndex(spec, {name: 'badkp'}), - 'failed to create index with key pattern' + tojson(spec)); - }); -} - -// When running the newest version, check that the indexes with bad key patterns are readable. -function validateBadIndexesStandalone(testDB) { - invalidIndexSpecs.forEach((spec) => { - // Generate a unique and identifiable collection name. - let collName = 'bad_index_' + tojson(spec.a); - let indexSpec = GetIndexHelpers.findByName(testDB[collName].getIndexes(), 'badkp'); - assert.neq(null, indexSpec, 'could not find index "badkp"'); - assert.eq(1, indexSpec.v, tojson(indexSpec)); - - // Collection compact command should succeed, despite the presence of the v:1 index - // which would fail v:2 validation rules. - assert.commandWorked(testDB.runCommand({compact: collName})); - - // reIndex will fail because when featureCompatibilityVersion>=3.4, reIndex - // automatically upgrades v=1 indexes to v=2. - assert.commandFailed(testDB[collName].reIndex()); - - // reIndex should not drop the index. - indexSpec = GetIndexHelpers.findByName(testDB[collName].getIndexes(), 'badkp'); - assert.neq(null, indexSpec, 'could not find index "badkp" after reIndex'); - assert.eq(1, indexSpec.v, tojson(indexSpec)); - - // A query that hints the index should succeed. - assert.commandWorked(testDB.runCommand({find: collName, hint: "badkp"})); - - // Newly created indexes will do stricter validation and should fail if the - // key pattern is invalid. - assert.commandWorked(testDB[collName].dropIndexes()); - assert.commandFailedWithCode( - testDB[collName].createIndex(spec), - ErrorCodes.CannotCreateIndex, - 'creating index with key pattern ' + tojson(spec) + ' unexpectedly succeeded'); - // Index build should also fail if v:1 or v:2 is explicitly requested. - assert.commandFailedWithCode( - testDB[collName].createIndex(spec, {v: 1}), - ErrorCodes.CannotCreateIndex, - 'creating index with key pattern ' + tojson(spec) + ' unexpectedly succeeded'); - assert.commandFailedWithCode( - testDB[collName].createIndex(spec, {v: 2}), - ErrorCodes.CannotCreateIndex, - 'creating index with key pattern ' + tojson(spec) + ' unexpectedly succeeded'); - }); -} - -// Check that secondary nodes have the v:1 indexes. -function validateBadIndexesSecondary(testDB) { - invalidIndexSpecs.forEach((spec) => { - // Generate a unique and identifiable collection name. - let collName = 'bad_index_' + tojson(spec.a); - // Verify that the secondary has the v:1 index. - let indexSpec = GetIndexHelpers.findByName(testDB[collName].getIndexes(), 'badkp'); - assert.neq(null, indexSpec, 'could not find index "badkp"'); - assert.eq(1, indexSpec.v, tojson(indexSpec)); - }); -} - // Standalone // Iterate from earliest to latest versions specified in the versions list, and follow the steps // outlined at the top of this test file. @@ -187,14 +111,6 @@ for (let i = 0; i < versions.length; i++) { // Create an index on the new collection. assert.commandWorked(testDB[version.testCollection].createIndex({a: 1})); - if (i === 0) { - // We're on the earliest version, insert indexes with bad key patterns. - insertBadIndexes(testDB); - } else if (i === versions.length - 1) { - // We're on the latest version, check bad indexes are still readable. - validateBadIndexesStandalone(testDB); - } - // Set the appropriate featureCompatibilityVersion upon upgrade, if applicable. if (version.hasOwnProperty('featureCompatibilityVersion')) { let adminDB = conn.getDB("admin"); @@ -250,16 +166,6 @@ for (let i = 0; i < versions.length; i++) { // Create an index on the new collection. assert.commandWorked(testDB[version.testCollection].createIndex({a: 1})); - if (i === 0) { - // We're on the earliest version, insert indexes with bad key patterns. - insertBadIndexes(testDB); - } else if (i === versions.length - 1) { - // We're on the latest version, check bad indexes are still readable. - for (let secondary of rst.getSecondaries()) { - validateBadIndexesSecondary(secondary.getDB('test')); - } - } - // Do the index creation and insertion again after upgrading the primary node. primary = rst.upgradePrimary(primary, {binVersion: version.binVersion}); assert.neq( |