From abf903ba3ba186de7dc4471cb63bbea212cc8330 Mon Sep 17 00:00:00 2001 From: Ryan Egesdahl Date: Tue, 31 Jan 2023 19:14:56 +0000 Subject: SERVER-73123 Make package tests compatible with older distros --- buildscripts/package_test.py | 16 ++-- buildscripts/package_test_internal.py | 165 ++++++++++++++++++++-------------- 2 files changed, 104 insertions(+), 77 deletions(-) (limited to 'buildscripts') diff --git a/buildscripts/package_test.py b/buildscripts/package_test.py index 4a258d17f0a..d927bbda49c 100644 --- a/buildscripts/package_test.py +++ b/buildscripts/package_test.py @@ -12,7 +12,7 @@ import uuid from concurrent import futures from pathlib import Path -from typing import Any, Dict, Generator, List, Optional, Tuple, Set +from typing import Any, Dict, Generator, List, Optional, Set, Tuple import docker import docker.errors @@ -21,8 +21,7 @@ import requests from docker.client import DockerClient from docker.models.containers import Container from docker.models.images import Image - -from simple_report import Result, Report +from simple_report import Report, Result root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -114,9 +113,10 @@ OS_DOCKER_LOOKUP = { 'ubuntu1204': None, 'ubuntu1404': None, 'ubuntu1604': ('ubuntu:16.04', "apt", - frozenset( - ["python", "python3", "wget", "pkg-config", "systemd", "procps", "file"]), - "python3"), + frozenset([ + "apt-utils", "python", "python3", "wget", "pkg-config", "systemd", "procps", + "file" + ]), "python3"), 'ubuntu1804': ('ubuntu:18.04', "apt", frozenset( ["python", "python3", "wget", "pkg-config", "systemd", "procps", "file"]), @@ -215,7 +215,7 @@ def run_test(test: Test, client: DockerClient) -> Result: logging.debug(test_external_root) log_external_path = Path.joinpath(test_external_root, log_name) - commands: List[str] = [] + commands: List[str] = ["export PYTHONIOENCODING=UTF-8"] if test.os_name.startswith('rhel'): if test.os_name.startswith('rhel7'): @@ -224,7 +224,7 @@ def run_test(test: Test, client: DockerClient) -> Result: "yum -y install centos-release-scl", "yum-config-manager --enable centos-sclo-rh", ] - # RHEL distros need EPEL for Compass dependencies + # RHEL distros need EPEL for Compass dependencies commands += [ "yum -y install yum-utils epel-release", "yum-config-manager --enable epel", diff --git a/buildscripts/package_test_internal.py b/buildscripts/package_test_internal.py index fc80d139f66..d1f5719ffa9 100644 --- a/buildscripts/package_test_internal.py +++ b/buildscripts/package_test_internal.py @@ -1,7 +1,7 @@ # This script needs to be compatible with odler versions of python since it runs on older versions of OSs when testing packaging # For example ubuntu 1604 uses python3.5 -#pylint: disable=redefined-outer-name,invalid-name +# pylint: disable=redefined-outer-name,invalid-name,subprocess-run-check import grp import logging @@ -10,8 +10,8 @@ import pathlib import platform import pwd import re -import sys import subprocess +import sys import tarfile import time import traceback @@ -24,7 +24,7 @@ root.setLevel(logging.DEBUG) stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setLevel(logging.DEBUG) -file_handler = WatchedFileHandler(sys.argv[1], mode='w') +file_handler = WatchedFileHandler(sys.argv[1], mode='w', encoding='utf8') file_handler.setLevel(logging.DEBUG) formatter = logging.Formatter('[%(asctime)s]%(levelname)s:%(message)s') stdout_handler.setFormatter(formatter) @@ -39,8 +39,9 @@ JOURNALCTL_URL = DOCKER_SYSTEMCTL_REPO + "/master/files/docker/journalctl3.py" TestArgs = Dict[str, Union[str, int, List[str]]] -def run_and_log(cmd: str, end_on_error: bool = True) -> 'subprocess.CompletedProcess[bytes]': - proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # pylint: disable=subprocess-run-check +def run_and_log(cmd: str, end_on_error: bool = True): + # type: (str, bool) -> 'subprocess.CompletedProcess[bytes]' + proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) logging.debug(cmd) logging.debug(proc.stdout.decode("UTF-8").strip()) if end_on_error and proc.returncode != 0: @@ -57,7 +58,7 @@ def download_extract_package(package: str) -> List[str]: if not package.endswith(".tgz"): return [downloaded_file] - extracted_paths = [] + extracted_paths = [] # type: List[str] with tarfile.open(downloaded_file) as tf: for member in tf.getmembers(): if member.name.endswith('.deb') or member.name.endswith('.rpm'): @@ -68,7 +69,7 @@ def download_extract_package(package: str) -> List[str]: def download_extract_all_packages(package_urls: List[str]) -> List[str]: - all_packages: List[str] = [] + all_packages = [] # type: List[str] for package_url in package_urls: package_names = download_extract_package(package_url) all_packages.extend(["./" + package_name for package_name in package_names]) @@ -90,14 +91,14 @@ def run_zypper_test(packages: List[str]): run_and_log("zypper -n --no-gpg-checks install {}".format(' '.join(packages))) -def run_mongo_query(query: str, should_fail: bool = False, tries: int = 60, - interval: float = 1.0) -> Optional['subprocess.CompletedProcess[bytes]']: +def run_mongo_query(query, should_fail=False, tries=60, interval=1.0): + # type: (str, bool, int, float) -> Optional[subprocess.CompletedProcess[bytes]] assert tries >= 1 - exec_result: Union[subprocess.CompletedProcess[bytes], None] = None + exec_result = None # type: Union[subprocess.CompletedProcess[bytes], None] current_try = 1 - command = f"mongosh --eval '{query}'" + command = "mongosh --eval '{}'".format(query) # Keep trying the query until we either get a successful return code or we # run out of tries. @@ -126,7 +127,7 @@ def run_mongo_query(query: str, should_fail: bool = False, tries: int = 60, def parse_os_release(path: str) -> Dict[str, str]: - result: Dict[str, str] = {} + result = {} # type: Dict[str, str] with open(path, 'r', encoding='utf-8') as os_release: for line in os_release: try: @@ -163,32 +164,32 @@ def parse_ulimits(pid: int) -> Dict[str, Tuple[int, int, Optional[str]]]: ulimit_line_re = re.compile( r'(?P.*?)\s{2,}(?P\S+)\s+(?P\S+)(?:\s+(?P\S+))?', re.MULTILINE) - result: Dict[str, Tuple[int, int, Optional[str]]] = {} - with open(f"/proc/{pid}/limits", 'r', encoding='utf-8') as ulimits_file: + result = {} # type: Dict[str, Tuple[int, int, Optional[str]]] + with open("/proc/{}/limits".format(pid), 'r', encoding='utf-8') as ulimits_file: next(ulimits_file) for line in ulimits_file: limits = ulimit_line_re.match(line) if limits is None: continue try: - soft_limit = int(limits['soft']) + soft_limit = int(limits.group('soft')) except ValueError: # unlimited soft_limit = -1 try: - hard_limit = int(limits['hard']) + hard_limit = int(limits.group('hard')) except ValueError: # unlimited hard_limit = -1 - result[limits['name']] = (soft_limit, hard_limit, limits['units']) + result[limits.group('name')] = (soft_limit, hard_limit, limits.group('units')) return result def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs: # Set up data for later tests - test_args: TestArgs = {} + test_args = {} # type: TestArgs test_args['package_manager'] = package_manager test_args['package_files'] = package_files @@ -200,6 +201,8 @@ def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs: test_args['systemd_units_dir'] = run_and_log( "pkg-config systemd --variable=systemdsystemunitdir").stdout.decode('utf-8').strip() + test_args['systemd_presets_dir'] = run_and_log( + "pkg-config systemd --variable=systemdsystempresetdir").stdout.decode('utf-8').strip() if package_manager in ('yum', 'zypper'): test_args['mongo_username'] = 'mongod' @@ -225,24 +228,30 @@ def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs: def get_package_name(package_file: str) -> str: if package_manager in ('yum', 'zypper'): - result = run_and_log(f"rpm --nosignature -qp --queryformat '%{{NAME}}' {package_file}") + result = run_and_log( + "rpm --nosignature -qp --queryformat '%{{NAME}}' {0}".format(package_file)) return result.stdout.decode('utf-8').strip() else: - result = run_and_log(f"dpkg -I {package_file}") + result = run_and_log("dpkg -I {}".format(package_file)) match = deb_output_re.search(result.stdout.decode('utf-8')) if match is not None: - return match[0] + return match.group(0) return '' - package_names: List[str] = [] + package_names = [] # type: List[str] for package in package_files: package_names.append(get_package_name(package)) test_args['package_names'] = package_names + if pathlib.Path("/usr/bin/systemd").exists(): + test_args["systemd_path"] = "/usr/bin" + else: + test_args["systemd_path"] = "/bin" + return test_args -def setup(): +def setup(test_args: TestArgs): logging.info("Setting up test environment.") # TODO SERVER-70425: We can remove these once we have figured out why @@ -250,9 +259,8 @@ def setup(): # Remove the PIDFile, PermissionsStartOnly, and Type configurations from # the systemd service file because they are not needed for simple-type # (non-forking) services and confuse the systemd emulator script. - run_and_log( - "sed -Ei '/^PIDFile=|PermissionsStartOnly=|Type=/d' $(pkg-config systemd --variable=systemdsystemunitdir)/mongod.service" - ) + run_and_log("sed -Ei '/^PIDFile=|PermissionsStartOnly=|Type=/d' {}/mongod.service".format( + test_args["systemd_units_dir"])) # Remove the journal: line (and the next) from mongod.conf, which is a # removed configuration. The Debian version of the config never got updated. run_and_log("sed -i '/journal:/,+1d' /etc/mongod.conf") @@ -260,26 +268,38 @@ def setup(): # a non-forking service under systemd. run_and_log("sed -Ei '/fork:|pidFilePath:/d' /etc/mongod.conf") - # TODO SERVER-70426: Remove these when we have a fake systemd package - # that does all of this for us. - # Ensure that mongod doesn't start automatically so we can do it as a test. + # Ensure systemd doesn't try to start anything automatically so we can do + # it in our tests run_and_log("mkdir -p /run/systemd/system") - run_and_log("mkdir -p $(pkg-config systemd --variable=systemdsystempresetdir)") - run_and_log( - "echo 'disable *' > $(pkg-config systemd --variable=systemdsystempresetdir)/00-test.preset") + run_and_log("mkdir -p {}".format(test_args["systemd_presets_dir"])) + run_and_log("echo 'disable *' > {}/00-test.preset".format(test_args["systemd_presets_dir"])) + + +def install_fake_systemd(test_args: TestArgs): + # TODO SERVER-70426: Remove this when we have a fake systemd package + # that does all of it for us. + # Ensure that mongod doesn't start automatically so we can do it as a test. + logging.info("Installing systemd emulator script.") + # Install a systemd emulator script so we can test services the way they are # actually installed on user systems. - run_and_log("wget -P /usr/bin " + SYSTEMCTL_URL) - run_and_log("wget -P /usr/bin " + JOURNALCTL_URL) - run_and_log("chmod +x /usr/bin/systemctl3.py /usr/bin/journalctl3.py") + run_and_log("wget -P /usr/local/bin " + SYSTEMCTL_URL) + run_and_log("wget -P /usr/local/bin " + JOURNALCTL_URL) + run_and_log("chmod +x /usr/local/bin/systemctl3.py /usr/local/bin/journalctl3.py") + # Overwrite the installed systemd with the emulator script and set up # the required symlinks so it all works correctly. - run_and_log("ln -sf /usr/bin/systemctl3.py /bin/systemctl") - run_and_log("ln -sf /usr/bin/systemctl3.py /usr/bin/systemctl") - run_and_log("ln -sf /usr/bin/systemctl3.py /bin/systemd") - run_and_log("ln -sf /usr/bin/systemctl3.py /usr/bin/systemd") - run_and_log("ln -sf /usr/bin/journalctl3.py /bin/journalctl") - run_and_log("ln -sf /usr/bin/journalctl3.py /usr/bin/journalctl") + run_and_log("ln -sf /usr/local/bin/systemctl3.py /usr/bin/systemd") + run_and_log("ln -sf /usr/local/bin/systemctl3.py /usr/bin/systemctl") + run_and_log("ln -sf /usr/local/bin/journalctl3.py /usr/bin/journalctl") + run_and_log("ln -sf /usr/local/bin/systemctl3.py /bin/systemd") + run_and_log("ln -sf /usr/local/bin/systemctl3.py /bin/systemctl") + run_and_log("ln -sf /usr/local/bin/journalctl3.py /bin/journalctl") + + # Do some workarounds for problems in older Ubuntu policykit-1 packages + # that cause package installation failures. + run_and_log("sed -iE '/--runtime/{s/# //;n;s/# //}' /usr/local/bin/systemctl3.py") + run_and_log("touch {}/polkitd.service".format(test_args["systemd_units_dir"])) def test_start(): @@ -295,16 +315,16 @@ def test_start(): def test_install_is_complete(test_args: TestArgs): logging.info("Checking that the installation is complete.") - required_files: List[pathlib.Path] = [ + required_files = [ pathlib.Path('/etc/mongod.conf'), pathlib.Path('/usr/bin/mongod'), pathlib.Path('/var/log/mongodb/mongod.log'), pathlib.Path(test_args['systemd_units_dir']) / "mongod.service", - ] + ] # type: List[pathlib.Path] - required_dirs: List[pathlib.Path] = [ + required_dirs = [ pathlib.Path(test_args['mongo_work_dir']), - ] + ] # type: List[pathlib.Path] if test_args['package_manager'] in ('yum', 'zypper'): # Only RPM-based distros create the home directory. Debian/Ubuntu @@ -313,21 +333,21 @@ def test_install_is_complete(test_args: TestArgs): for path in required_files: if not (path.exists() and path.is_file()): - raise RuntimeError(f"Required file missing: {path}") + raise RuntimeError("Required file missing: {}".format(path)) for path in required_dirs: if not (path.exists() and path.is_dir()): - raise RuntimeError(f"Required directory missing: {path}") + raise RuntimeError("Required directory missing: {}".format(path)) try: user_info = pwd.getpwnam(test_args['mongo_username']) except KeyError: - raise RuntimeError(f"Required user missing: {test_args['mongo_username']}") + raise RuntimeError("Required user missing: {}".format(test_args['mongo_username'])) try: grp.getgrnam(test_args['mongo_groupname']) except KeyError: - raise RuntimeError(f"Required group missing: {test_args['mongo_username']}") + raise RuntimeError("Required group missing: {}".format(test_args['mongo_username'])) # All of the supplemental groups (the .deb pattern) mongo_user_groups = [ @@ -338,16 +358,17 @@ def test_install_is_complete(test_args: TestArgs): mongo_user_groups.append(grp.getgrgid(user_info.pw_gid).gr_name) if test_args['mongo_groupname'] not in mongo_user_groups: - raise RuntimeError((f"Required group `{test_args['mongo_groupname']}' " - f"is not in configured groups: {mongo_user_groups}")) + raise RuntimeError("Required group `{}' is not in configured groups: {}".format( + test_args['mongo_groupname'], mongo_user_groups)) if user_info.pw_dir != test_args['mongo_home_dir']: - raise RuntimeError((f"Configured home directory `{user_info.pw_dir}' " - f"does not match required path `{test_args['mongo_home_dir']}'")) + raise RuntimeError( + "Configured home directory `{}' does not match required path `{}'".format( + user_info.pw_dir, test_args['mongo_home_dir'])) if user_info.pw_shell != test_args['mongo_user_shell']: - raise RuntimeError((f"Configured user shell `{user_info.pw_shell}' " - f"does not match required path `{test_args['mongo_user_shell']}'")) + raise RuntimeError("Configured user shell `{}' does not match required path `{}'".format( + user_info.pw_shell, test_args['mongo_user_shell'])) def test_ulimits_correct(): @@ -359,22 +380,22 @@ def test_ulimits_correct(): ulimits = parse_ulimits(mongod_pid) if ulimits['Max file size'][0] != -1: - raise RuntimeError(f"RLMIT_FSIZE != unlimited: {ulimits['Max file size']}") + raise RuntimeError("RLMIT_FSIZE != unlimited: {}".format(ulimits['Max file size'])) if ulimits['Max cpu time'][0] != -1: - raise RuntimeError(f"RLMIT_CPU != unlimited: {ulimits['Max cpu time']}") + raise RuntimeError("RLMIT_CPU != unlimited: {}".format(ulimits['Max cpu time'])) if ulimits['Max address space'][0] != -1: - raise RuntimeError(f"RLMIT_AS != unlimited: {ulimits['Max address space']}") + raise RuntimeError("RLMIT_AS != unlimited: {}".format(ulimits['Max address space'])) if ulimits['Max open files'][0] != -1 and ulimits['Max open files'][0] < 64000: - raise RuntimeError(f"RLMIT_NOFILE < 64000: {ulimits['Max open files']}") + raise RuntimeError("RLMIT_NOFILE < 64000: {}".format(ulimits['Max open files'])) if ulimits['Max resident set'][0] != -1: - raise RuntimeError(f"RLMIT_RSS != unlimited: {ulimits['Max resident set']}") + raise RuntimeError("RLMIT_RSS != unlimited: {}".format(ulimits['Max resident set'])) if ulimits['Max processes'][0] != -1 and ulimits['Max processes'][0] < 64000: - raise RuntimeError(f"RLMIT_NPROC < 64000: {ulimits['Max processes']}") + raise RuntimeError("RLMIT_NPROC < 64000: {}".format(ulimits['Max processes'])) def test_restart(): @@ -402,7 +423,12 @@ def test_stop(): def test_install_compass(test_args: TestArgs): logging.info("Installing Compass.") - exec_result = run_and_log("install_compass", end_on_error=False) + cmd = [] # type: List[str] + if test_args["package_manager"] == "apt": + cmd += ["DEBIAN_FRONTEND=noninteractive"] + cmd += ["install_compass"] + + exec_result = run_and_log(" ".join(cmd), end_on_error=False) if exec_result.returncode != 0: if test_args['arch'] == 'x86_64' and test_args['package_manager'] != 'zypper': @@ -414,7 +440,7 @@ def test_install_compass(test_args: TestArgs): def test_uninstall(test_args: TestArgs): logging.info("Uninstalling packages:\n\t%s", '\n\t'.join(test_args['package_names'])) - command: str + command = '' # type: str if test_args['package_manager'] == 'apt': command = 'apt-get remove -y {}' elif test_args['package_manager'] == 'yum': @@ -422,8 +448,8 @@ def test_uninstall(test_args: TestArgs): elif test_args['package_manager'] == 'zypper': command = 'zypper -n remove {}' else: - raise RuntimeError( - "Don't know how to uninstall with package manager: {test_args['package_manager']}") + raise RuntimeError("Don't know how to uninstall with package manager: {}".format( + test_args['package_manager'])) run_and_log(command.format(' '.join(test_args['package_names']))) @@ -434,11 +460,11 @@ def test_uninstall_is_complete(test_args: TestArgs): leftover_files = [ pathlib.Path('/usr/bin/mongod'), pathlib.Path(test_args['systemd_units_dir']) / 'mongod.service', - ] + ] # type: List[pathlib.Path] for path in leftover_files: if path.exists(): - raise RuntimeError(f"Failed to uninstall cleanly, found: {path}") + raise RuntimeError("Failed to uninstall cleanly, found: {}".format(path)) package_urls = sys.argv[2:] @@ -449,7 +475,7 @@ if len(package_urls) == 0: package_files = download_extract_all_packages(package_urls) -package_manager: str +package_manager = '' # type: str apt_proc = run_and_log("apt --help", end_on_error=False) yum_proc = run_and_log("yum --help", end_on_error=False) zypper_proc = run_and_log("zypper -n --help", end_on_error=False) @@ -468,7 +494,8 @@ else: sys.exit(1) test_args = get_test_args(package_manager, package_files) -setup() +setup(test_args) +install_fake_systemd(test_args) test_start() test_install_is_complete(test_args) -- cgit v1.2.1