SERVER-73123 Make package tests compatible with older distros

2 files changed, 104 insertions, 77 deletions

diff --git a/buildscripts/package_test.py b/buildscripts/package_test.py
index 4a258d17f0a..d927bbda49c 100644
--- a/buildscripts/package_test.py
+++ b/buildscripts/package_test.py
@@ -12,7 +12,7 @@ import uuid
 
 from concurrent import futures
 from pathlib import Path
-from typing import Any, Dict, Generator, List, Optional, Tuple, Set
+from typing import Any, Dict, Generator, List, Optional, Set, Tuple
 
 import docker
 import docker.errors
@@ -21,8 +21,7 @@ import requests
 from docker.client import DockerClient
 from docker.models.containers import Container
 from docker.models.images import Image
-
-from simple_report import Result, Report
+from simple_report import Report, Result
 
 root = logging.getLogger()
 root.setLevel(logging.DEBUG)
@@ -114,9 +113,10 @@ OS_DOCKER_LOOKUP = {
     'ubuntu1204': None,
     'ubuntu1404': None,
     'ubuntu1604': ('ubuntu:16.04', "apt",
-                   frozenset(
-                       ["python", "python3", "wget", "pkg-config", "systemd", "procps", "file"]),
-                   "python3"),
+                   frozenset([
+                       "apt-utils", "python", "python3", "wget", "pkg-config", "systemd", "procps",
+                       "file"
+                   ]), "python3"),
     'ubuntu1804': ('ubuntu:18.04', "apt",
                    frozenset(
                        ["python", "python3", "wget", "pkg-config", "systemd", "procps", "file"]),
@@ -215,7 +215,7 @@ def run_test(test: Test, client: DockerClient) -> Result:
     logging.debug(test_external_root)
     log_external_path = Path.joinpath(test_external_root, log_name)
 
-    commands: List[str] = []
+    commands: List[str] = ["export PYTHONIOENCODING=UTF-8"]
 
     if test.os_name.startswith('rhel'):
         if test.os_name.startswith('rhel7'):
@@ -224,7 +224,7 @@ def run_test(test: Test, client: DockerClient) -> Result:
                 "yum -y install centos-release-scl",
                 "yum-config-manager --enable centos-sclo-rh",
             ]
-            # RHEL distros need EPEL for Compass dependencies
+        # RHEL distros need EPEL for Compass dependencies
         commands += [
             "yum -y install yum-utils epel-release",
             "yum-config-manager --enable epel",
diff --git a/buildscripts/package_test_internal.py b/buildscripts/package_test_internal.py
index fc80d139f66..d1f5719ffa9 100644
--- a/buildscripts/package_test_internal.py
+++ b/buildscripts/package_test_internal.py
@@ -1,7 +1,7 @@
 # This script needs to be compatible with odler versions of python since it runs on older versions of OSs when testing packaging
 # For example ubuntu 1604 uses python3.5
 
-#pylint: disable=redefined-outer-name,invalid-name
+# pylint: disable=redefined-outer-name,invalid-name,subprocess-run-check
 
 import grp
 import logging
@@ -10,8 +10,8 @@ import pathlib
 import platform
 import pwd
 import re
-import sys
 import subprocess
+import sys
 import tarfile
 import time
 import traceback
@@ -24,7 +24,7 @@ root.setLevel(logging.DEBUG)
 
 stdout_handler = logging.StreamHandler(sys.stdout)
 stdout_handler.setLevel(logging.DEBUG)
-file_handler = WatchedFileHandler(sys.argv[1], mode='w')
+file_handler = WatchedFileHandler(sys.argv[1], mode='w', encoding='utf8')
 file_handler.setLevel(logging.DEBUG)
 formatter = logging.Formatter('[%(asctime)s]%(levelname)s:%(message)s')
 stdout_handler.setFormatter(formatter)
@@ -39,8 +39,9 @@ JOURNALCTL_URL = DOCKER_SYSTEMCTL_REPO + "/master/files/docker/journalctl3.py"
 TestArgs = Dict[str, Union[str, int, List[str]]]
 
 
-def run_and_log(cmd: str, end_on_error: bool = True) -> 'subprocess.CompletedProcess[bytes]':
-    proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)  # pylint: disable=subprocess-run-check
+def run_and_log(cmd: str, end_on_error: bool = True):
+    # type: (str, bool) -> 'subprocess.CompletedProcess[bytes]'
+    proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     logging.debug(cmd)
     logging.debug(proc.stdout.decode("UTF-8").strip())
     if end_on_error and proc.returncode != 0:
@@ -57,7 +58,7 @@ def download_extract_package(package: str) -> List[str]:
     if not package.endswith(".tgz"):
         return [downloaded_file]
 
-    extracted_paths = []
+    extracted_paths = []  # type: List[str]
     with tarfile.open(downloaded_file) as tf:
         for member in tf.getmembers():
             if member.name.endswith('.deb') or member.name.endswith('.rpm'):
@@ -68,7 +69,7 @@ def download_extract_package(package: str) -> List[str]:
 
 
 def download_extract_all_packages(package_urls: List[str]) -> List[str]:
-    all_packages: List[str] = []
+    all_packages = []  # type: List[str]
     for package_url in package_urls:
         package_names = download_extract_package(package_url)
         all_packages.extend(["./" + package_name for package_name in package_names])
@@ -90,14 +91,14 @@ def run_zypper_test(packages: List[str]):
     run_and_log("zypper -n --no-gpg-checks install {}".format(' '.join(packages)))
 
 
-def run_mongo_query(query: str, should_fail: bool = False, tries: int = 60,
-                    interval: float = 1.0) -> Optional['subprocess.CompletedProcess[bytes]']:
+def run_mongo_query(query, should_fail=False, tries=60, interval=1.0):
+    # type: (str, bool, int, float) -> Optional[subprocess.CompletedProcess[bytes]]
     assert tries >= 1
 
-    exec_result: Union[subprocess.CompletedProcess[bytes], None] = None
+    exec_result = None  # type: Union[subprocess.CompletedProcess[bytes], None]
     current_try = 1
 
-    command = f"mongosh --eval '{query}'"
+    command = "mongosh --eval '{}'".format(query)
 
     # Keep trying the query until we either get a successful return code or we
     # run out of tries.
@@ -126,7 +127,7 @@ def run_mongo_query(query: str, should_fail: bool = False, tries: int = 60,
 
 
 def parse_os_release(path: str) -> Dict[str, str]:
-    result: Dict[str, str] = {}
+    result = {}  # type: Dict[str, str]
     with open(path, 'r', encoding='utf-8') as os_release:
         for line in os_release:
             try:
@@ -163,32 +164,32 @@ def parse_ulimits(pid: int) -> Dict[str, Tuple[int, int, Optional[str]]]:
     ulimit_line_re = re.compile(
         r'(?P<name>.*?)\s{2,}(?P<soft>\S+)\s+(?P<hard>\S+)(?:\s+(?P<units>\S+))?', re.MULTILINE)
 
-    result: Dict[str, Tuple[int, int, Optional[str]]] = {}
-    with open(f"/proc/{pid}/limits", 'r', encoding='utf-8') as ulimits_file:
+    result = {}  # type: Dict[str, Tuple[int, int, Optional[str]]]
+    with open("/proc/{}/limits".format(pid), 'r', encoding='utf-8') as ulimits_file:
         next(ulimits_file)
         for line in ulimits_file:
             limits = ulimit_line_re.match(line)
             if limits is None:
                 continue
             try:
-                soft_limit = int(limits['soft'])
+                soft_limit = int(limits.group('soft'))
             except ValueError:
                 # unlimited
                 soft_limit = -1
             try:
-                hard_limit = int(limits['hard'])
+                hard_limit = int(limits.group('hard'))
             except ValueError:
                 # unlimited
                 hard_limit = -1
 
-            result[limits['name']] = (soft_limit, hard_limit, limits['units'])
+            result[limits.group('name')] = (soft_limit, hard_limit, limits.group('units'))
 
     return result
 
 
 def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs:
     # Set up data for later tests
-    test_args: TestArgs = {}
+    test_args = {}  # type: TestArgs
 
     test_args['package_manager'] = package_manager
     test_args['package_files'] = package_files
@@ -200,6 +201,8 @@ def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs:
 
     test_args['systemd_units_dir'] = run_and_log(
         "pkg-config systemd --variable=systemdsystemunitdir").stdout.decode('utf-8').strip()
+    test_args['systemd_presets_dir'] = run_and_log(
+        "pkg-config systemd --variable=systemdsystempresetdir").stdout.decode('utf-8').strip()
 
     if package_manager in ('yum', 'zypper'):
         test_args['mongo_username'] = 'mongod'
@@ -225,24 +228,30 @@ def get_test_args(package_manager: str, package_files: List[str]) -> TestArgs:
 
     def get_package_name(package_file: str) -> str:
         if package_manager in ('yum', 'zypper'):
-            result = run_and_log(f"rpm --nosignature -qp --queryformat '%{{NAME}}' {package_file}")
+            result = run_and_log(
+                "rpm --nosignature -qp --queryformat '%{{NAME}}' {0}".format(package_file))
             return result.stdout.decode('utf-8').strip()
         else:
-            result = run_and_log(f"dpkg -I {package_file}")
+            result = run_and_log("dpkg -I {}".format(package_file))
             match = deb_output_re.search(result.stdout.decode('utf-8'))
             if match is not None:
-                return match[0]
+                return match.group(0)
             return ''
 
-    package_names: List[str] = []
+    package_names = []  # type: List[str]
     for package in package_files:
         package_names.append(get_package_name(package))
     test_args['package_names'] = package_names
 
+    if pathlib.Path("/usr/bin/systemd").exists():
+        test_args["systemd_path"] = "/usr/bin"
+    else:
+        test_args["systemd_path"] = "/bin"
+
     return test_args
 
 
-def setup():
+def setup(test_args: TestArgs):
     logging.info("Setting up test environment.")
 
     # TODO SERVER-70425: We can remove these once we have figured out why
@@ -250,9 +259,8 @@ def setup():
     # Remove the PIDFile, PermissionsStartOnly, and Type configurations from
     # the systemd service file because they are not needed for simple-type
     # (non-forking) services and confuse the systemd emulator script.
-    run_and_log(
-        "sed -Ei '/^PIDFile=|PermissionsStartOnly=|Type=/d' $(pkg-config systemd --variable=systemdsystemunitdir)/mongod.service"
-    )
+    run_and_log("sed -Ei '/^PIDFile=|PermissionsStartOnly=|Type=/d' {}/mongod.service".format(
+        test_args["systemd_units_dir"]))
     # Remove the journal: line (and the next) from mongod.conf, which is a
     # removed configuration. The Debian version of the config never got updated.
     run_and_log("sed -i '/journal:/,+1d' /etc/mongod.conf")
@@ -260,26 +268,38 @@ def setup():
     # a non-forking service under systemd.
     run_and_log("sed -Ei '/fork:|pidFilePath:/d' /etc/mongod.conf")
 
-    # TODO SERVER-70426: Remove these when we have a fake systemd package
-    # that does all of this for us.
-    # Ensure that mongod doesn't start automatically so we can do it as a test.
+    # Ensure systemd doesn't try to start anything automatically so we can do
+    # it in our tests
     run_and_log("mkdir -p /run/systemd/system")
-    run_and_log("mkdir -p $(pkg-config systemd --variable=systemdsystempresetdir)")
-    run_and_log(
-        "echo 'disable *' > $(pkg-config systemd --variable=systemdsystempresetdir)/00-test.preset")
+    run_and_log("mkdir -p {}".format(test_args["systemd_presets_dir"]))
+    run_and_log("echo 'disable *' > {}/00-test.preset".format(test_args["systemd_presets_dir"]))
+
+
+def install_fake_systemd(test_args: TestArgs):
+    # TODO SERVER-70426: Remove this when we have a fake systemd package
+    # that does all of it for us.
+    # Ensure that mongod doesn't start automatically so we can do it as a test.
+    logging.info("Installing systemd emulator script.")
+
     # Install a systemd emulator script so we can test services the way they are
     # actually installed on user systems.
-    run_and_log("wget -P /usr/bin " + SYSTEMCTL_URL)
-    run_and_log("wget -P /usr/bin " + JOURNALCTL_URL)
-    run_and_log("chmod +x /usr/bin/systemctl3.py /usr/bin/journalctl3.py")
+    run_and_log("wget -P /usr/local/bin " + SYSTEMCTL_URL)
+    run_and_log("wget -P /usr/local/bin " + JOURNALCTL_URL)
+    run_and_log("chmod +x /usr/local/bin/systemctl3.py /usr/local/bin/journalctl3.py")
+
     # Overwrite the installed systemd with the emulator script and set up
     # the required symlinks so it all works correctly.
-    run_and_log("ln -sf /usr/bin/systemctl3.py /bin/systemctl")
-    run_and_log("ln -sf /usr/bin/systemctl3.py /usr/bin/systemctl")
-    run_and_log("ln -sf /usr/bin/systemctl3.py /bin/systemd")
-    run_and_log("ln -sf /usr/bin/systemctl3.py /usr/bin/systemd")
-    run_and_log("ln -sf /usr/bin/journalctl3.py /bin/journalctl")
-    run_and_log("ln -sf /usr/bin/journalctl3.py /usr/bin/journalctl")
+    run_and_log("ln -sf /usr/local/bin/systemctl3.py /usr/bin/systemd")
+    run_and_log("ln -sf /usr/local/bin/systemctl3.py /usr/bin/systemctl")
+    run_and_log("ln -sf /usr/local/bin/journalctl3.py /usr/bin/journalctl")
+    run_and_log("ln -sf /usr/local/bin/systemctl3.py /bin/systemd")
+    run_and_log("ln -sf /usr/local/bin/systemctl3.py /bin/systemctl")
+    run_and_log("ln -sf /usr/local/bin/journalctl3.py /bin/journalctl")
+
+    # Do some workarounds for problems in older Ubuntu policykit-1 packages
+    # that cause package installation failures.
+    run_and_log("sed -iE '/--runtime/{s/# //;n;s/# //}' /usr/local/bin/systemctl3.py")
+    run_and_log("touch {}/polkitd.service".format(test_args["systemd_units_dir"]))
 
 
 def test_start():
@@ -295,16 +315,16 @@ def test_start():
 def test_install_is_complete(test_args: TestArgs):
     logging.info("Checking that the installation is complete.")
 
-    required_files: List[pathlib.Path] = [
+    required_files = [
         pathlib.Path('/etc/mongod.conf'),
         pathlib.Path('/usr/bin/mongod'),
         pathlib.Path('/var/log/mongodb/mongod.log'),
         pathlib.Path(test_args['systemd_units_dir']) / "mongod.service",
-    ]
+    ]  # type: List[pathlib.Path]
 
-    required_dirs: List[pathlib.Path] = [
+    required_dirs = [
         pathlib.Path(test_args['mongo_work_dir']),
-    ]
+    ]  # type: List[pathlib.Path]
 
     if test_args['package_manager'] in ('yum', 'zypper'):
         # Only RPM-based distros create the home directory. Debian/Ubuntu
@@ -313,21 +333,21 @@ def test_install_is_complete(test_args: TestArgs):
 
     for path in required_files:
         if not (path.exists() and path.is_file()):
-            raise RuntimeError(f"Required file missing: {path}")
+            raise RuntimeError("Required file missing: {}".format(path))
 
     for path in required_dirs:
         if not (path.exists() and path.is_dir()):
-            raise RuntimeError(f"Required directory missing: {path}")
+            raise RuntimeError("Required directory missing: {}".format(path))
 
     try:
         user_info = pwd.getpwnam(test_args['mongo_username'])
     except KeyError:
-        raise RuntimeError(f"Required user missing: {test_args['mongo_username']}")
+        raise RuntimeError("Required user missing: {}".format(test_args['mongo_username']))
 
     try:
         grp.getgrnam(test_args['mongo_groupname'])
     except KeyError:
-        raise RuntimeError(f"Required group missing: {test_args['mongo_username']}")
+        raise RuntimeError("Required group missing: {}".format(test_args['mongo_username']))
 
     # All of the supplemental groups (the .deb pattern)
     mongo_user_groups = [
@@ -338,16 +358,17 @@ def test_install_is_complete(test_args: TestArgs):
     mongo_user_groups.append(grp.getgrgid(user_info.pw_gid).gr_name)
 
     if test_args['mongo_groupname'] not in mongo_user_groups:
-        raise RuntimeError((f"Required group `{test_args['mongo_groupname']}' "
-                            f"is not in configured groups: {mongo_user_groups}"))
+        raise RuntimeError("Required group `{}' is not in configured groups: {}".format(
+            test_args['mongo_groupname'], mongo_user_groups))
 
     if user_info.pw_dir != test_args['mongo_home_dir']:
-        raise RuntimeError((f"Configured home directory `{user_info.pw_dir}' "
-                            f"does not match required path `{test_args['mongo_home_dir']}'"))
+        raise RuntimeError(
+            "Configured home directory `{}' does not match required path `{}'".format(
+                user_info.pw_dir, test_args['mongo_home_dir']))
 
     if user_info.pw_shell != test_args['mongo_user_shell']:
-        raise RuntimeError((f"Configured user shell `{user_info.pw_shell}' "
-                            f"does not match required path `{test_args['mongo_user_shell']}'"))
+        raise RuntimeError("Configured user shell `{}' does not match required path `{}'".format(
+            user_info.pw_shell, test_args['mongo_user_shell']))
 
 
 def test_ulimits_correct():
@@ -359,22 +380,22 @@ def test_ulimits_correct():
     ulimits = parse_ulimits(mongod_pid)
 
     if ulimits['Max file size'][0] != -1:
-        raise RuntimeError(f"RLMIT_FSIZE != unlimited: {ulimits['Max file size']}")
+        raise RuntimeError("RLMIT_FSIZE != unlimited: {}".format(ulimits['Max file size']))
 
     if ulimits['Max cpu time'][0] != -1:
-        raise RuntimeError(f"RLMIT_CPU != unlimited: {ulimits['Max cpu time']}")
+        raise RuntimeError("RLMIT_CPU != unlimited: {}".format(ulimits['Max cpu time']))
 
     if ulimits['Max address space'][0] != -1:
-        raise RuntimeError(f"RLMIT_AS != unlimited: {ulimits['Max address space']}")
+        raise RuntimeError("RLMIT_AS != unlimited: {}".format(ulimits['Max address space']))
 
     if ulimits['Max open files'][0] != -1 and ulimits['Max open files'][0] < 64000:
-        raise RuntimeError(f"RLMIT_NOFILE < 64000: {ulimits['Max open files']}")
+        raise RuntimeError("RLMIT_NOFILE < 64000: {}".format(ulimits['Max open files']))
 
     if ulimits['Max resident set'][0] != -1:
-        raise RuntimeError(f"RLMIT_RSS != unlimited: {ulimits['Max resident set']}")
+        raise RuntimeError("RLMIT_RSS != unlimited: {}".format(ulimits['Max resident set']))
 
     if ulimits['Max processes'][0] != -1 and ulimits['Max processes'][0] < 64000:
-        raise RuntimeError(f"RLMIT_NPROC < 64000: {ulimits['Max processes']}")
+        raise RuntimeError("RLMIT_NPROC < 64000: {}".format(ulimits['Max processes']))
 
 
 def test_restart():
@@ -402,7 +423,12 @@ def test_stop():
 def test_install_compass(test_args: TestArgs):
     logging.info("Installing Compass.")
 
-    exec_result = run_and_log("install_compass", end_on_error=False)
+    cmd = []  # type: List[str]
+    if test_args["package_manager"] == "apt":
+        cmd += ["DEBIAN_FRONTEND=noninteractive"]
+    cmd += ["install_compass"]
+
+    exec_result = run_and_log(" ".join(cmd), end_on_error=False)
 
     if exec_result.returncode != 0:
         if test_args['arch'] == 'x86_64' and test_args['package_manager'] != 'zypper':
@@ -414,7 +440,7 @@ def test_install_compass(test_args: TestArgs):
 def test_uninstall(test_args: TestArgs):
     logging.info("Uninstalling packages:\n\t%s", '\n\t'.join(test_args['package_names']))
 
-    command: str
+    command = ''  # type: str
     if test_args['package_manager'] == 'apt':
         command = 'apt-get remove -y {}'
     elif test_args['package_manager'] == 'yum':
@@ -422,8 +448,8 @@ def test_uninstall(test_args: TestArgs):
     elif test_args['package_manager'] == 'zypper':
         command = 'zypper -n remove {}'
     else:
-        raise RuntimeError(
-            "Don't know how to uninstall with package manager: {test_args['package_manager']}")
+        raise RuntimeError("Don't know how to uninstall with package manager: {}".format(
+            test_args['package_manager']))
 
     run_and_log(command.format(' '.join(test_args['package_names'])))
 
@@ -434,11 +460,11 @@ def test_uninstall_is_complete(test_args: TestArgs):
     leftover_files = [
         pathlib.Path('/usr/bin/mongod'),
         pathlib.Path(test_args['systemd_units_dir']) / 'mongod.service',
-    ]
+    ]  # type: List[pathlib.Path]
 
     for path in leftover_files:
         if path.exists():
-            raise RuntimeError(f"Failed to uninstall cleanly, found: {path}")
+            raise RuntimeError("Failed to uninstall cleanly, found: {}".format(path))
 
 
 package_urls = sys.argv[2:]
@@ -449,7 +475,7 @@ if len(package_urls) == 0:
 
 package_files = download_extract_all_packages(package_urls)
 
-package_manager: str
+package_manager = ''  # type: str
 apt_proc = run_and_log("apt --help", end_on_error=False)
 yum_proc = run_and_log("yum --help", end_on_error=False)
 zypper_proc = run_and_log("zypper -n --help", end_on_error=False)
@@ -468,7 +494,8 @@ else:
     sys.exit(1)
 
 test_args = get_test_args(package_manager, package_files)
-setup()
+setup(test_args)
+install_fake_systemd(test_args)
 
 test_start()
 test_install_is_complete(test_args)