SERVER-55019 Fix install_compass platform compatibility

The install_compass script did not execute on all platforms we support
on all branches. This change makes the script a bit more universal so it
will run with whatever Python version users happen to have installed.

(cherry picked from commit 92c4d25d07b94356a27c8455f320d4145195a7e3)
(cherry picked from commit 44936147c571d6f42821ba81b154e0746c3d7a14)

2 files changed, 398 insertions, 159 deletions

diff --git a/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb b/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb
index 25900e95196..ad85bb3ee76 100644
--- a/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb
+++ b/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb
@@ -55,17 +55,29 @@ else
   end
 end
 
-if os[:arch] == 'x86_64' and 
-  ((os[:name] == 'ubuntu' and os[:release].split('.')[0].to_i > 12) or 
-    (os[:family] == 'redhat' and os[:release].split('.')[0].to_i >= 7) or 
-    (os[:name] == 'debian' and os[:release].split('.')[0].to_i >= 10) or
-    os[:name] == 'amazon')
-  describe command("install_compass") do
-    its('exit_status') { should eq 0 }
+if os[:arch] == 'x86_64'
+  # install_compass does not run Amazon Linux but *does* run on Amazon Linux 2,
+  # but the 'redhat' family includes both, apparently. We need to specifically
+  # exclude Amazon Linux from the set of allowed distributions here because the
+  # version strings would otherwise pass it.
+  if ((os[:family] == 'redhat' and os[:name] != "amazon" and os[:release].split('.')[0].to_i >= 7) or
+      (os[:name] == 'ubuntu' and os[:release].split('.')[0].to_i >= 16) or
+      (os[:name] == 'debian' and os[:release].split('.')[0].to_i >= 9) or
+      (os[:name] == 'amazon' and os[:release].split('.')[0].to_i == 2))
+    describe command("install_compass") do
+      its('exit_status') { should eq 0 }
+      its('stderr') { should eq '' }
+    end
+  else
+    describe command("install_compass") do
+      its('exit_status') { should eq 1 }
+      its('stderr') { should match /You are using an unsupported Linux distribution/ }
+    end
   end
 else
   describe command("install_compass") do
-    its('exit_status') { should_not eq 0 }
+    its('exit_status') { should eq 1 }
+    its('stderr') { should match /Sorry, MongoDB Compass is only supported on 64-bit Intel platforms./ }
   end
 end
 
diff --git a/src/mongo/installer/compass/install_compass b/src/mongo/installer/compass/install_compass
index 2fea45944ca..04c081737f6 100755
--- a/src/mongo/installer/compass/install_compass
+++ b/src/mongo/installer/compass/install_compass
@@ -1,207 +1,434 @@
 #!/usr/bin/env python
-from __future__ import print_function
 
-# This script downloads the latest version of Compass and installs it on
-# non-windows platforms.
+"""This script downloads the latest version of Compass and installs it on
+non-windows platforms."""
 
+
+import itertools
+import logging
 import os
 import os.path as path
+import platform
+import re
 import shutil
 import subprocess
 import sys
 import tempfile
-import platform
 
+try:
+    from urllib2 import urlopen
+except ImportError:
+    from urllib.request import urlopen
+
+try:
+    from shlex import quote as cmd_quote
+except ImportError:
+    from pipes import quote as cmd_quote
+
+
+DOWNLOAD_BASE_URL = 'https://compass.mongodb.com/api/v2/download/latest/compass/stable'
+
+# Some systems may have multiple package managers installed - Debian systems may
+# have yum installed, and newer RHEL/Fedora systems may have both dnf and yum.
+# These must be checked in the given order for consistent results.
+SUPPORTED_LINUX_PKG_MANAGERS = ['apt', 'dnf', 'yum']
+
+LINUX_PKG_MANAGER_ARGS = {
+    'apt': ['install', '--yes'],
+    'yum': ['install', '--assumeyes'],
+    'dnf': ['install', '--assumeyes'],
+}
+
+PKG_MANAGER_SUFFIXES = {
+    'apt': '.deb',
+    'dnf': '.rpm',
+    'yum': '.rpm',
+    'dmg': '.dmg',
+}
+
+SUPPORTED_OSX_VERSION = 10.10
+SUPPORTED_RHEL_VERSION = 7
+SUPPORTED_DEBIAN_VERSION = 9
+SUPPORTED_DEBIAN_RELEASES = [
+    'stretch',
+    'buster',
+    'bullseye',
+    'bookworm',
+]
+
+
+class LogLevelFilter(logging.Filter):
+    """Filters log messages by level"""
+
+    def __init__(self, levelno, *args, **kwargs):
+        self._levelno = levelno
+        # In Python 2.6, logging.Filter is not a new-style class, so super()
+        # will not work with it. We need to call the base class __init__()
+        # directly here so it works with all versions of Python.
+        logging.Filter.__init__(self, *args, **kwargs)
+
+    def filter(self, record):
+        return record.levelno <= self._levelno
+
+
+class FileDownloadError(Exception):
+    """Exception raised on file download errors"""
+
+
+class InstallationError(Exception):
+    """Exception raised on installation errors"""
+
+
+def run_command(*args, **kwargs):
+    """Runs a command, with optional stdout and stderr."""
 
-def get_pkg_format():
-    """Determine the package manager for this Linux distro."""
     with open(os.devnull, 'w') as fnull:
-        try:
-            subprocess.call(['apt-get', '--help'], stdout=fnull, stderr=fnull)
-            return 'apt'
-        except:
-            pass
+        stdout = kwargs.pop('stdout', fnull)
+        stderr = kwargs.pop('stderr', fnull)
 
-        try:
-            subprocess.call(['dnf', '--help'], stdout=fnull, stderr=fnull)
-            return 'dnf'
-        except:
-            pass
+        process = subprocess.Popen([cmd_quote(arg) for arg in args],
+                                   stdout=stdout, stderr=stderr, **kwargs)
+        stdout, stderr = process.communicate()
 
-        try:
-            subprocess.call(['yum', '--help'], stdout=fnull, stderr=fnull)
-            return 'yum'
-        except:
-            pass
+        if stdout is not None:
+            stdout = stdout.decode('utf-8')
+            process.stdout.close()
+        if stderr is not None:
+            stderr = stderr.decode('utf-8')
+            process.stderr.close()
 
-    return ''
+    return (process.wait(), stdout, stderr)
+
+
+def get_package_manager():
+    """Determine the package manager for this Linux distro."""
+
+    if sys.platform == 'darwin':
+        return 'dmg'
+    else:
+        for package_manager in SUPPORTED_LINUX_PKG_MANAGERS:
+            try:
+                retcode, _, _ = run_command(package_manager, '--help')
+                if retcode == 0:
+                    return package_manager
+            except OSError:
+                pass
+
+    return None
 
 
 def download_progress(count, block_size, total_size):
     """Log the download progress."""
-    percent = int(count * block_size * 100 / total_size)
-    sys.stdout.write("\rDownloading Compass... %d%%" % percent)
-    sys.stdout.flush()
 
+    percent = min(int(count * block_size * 100 / total_size), 100)
 
-def download_pkg(link, pkg_format=''):
-    """Download the package from link, logging progress. Returns the filename."""
-    suf = ''
-    if pkg_format == 'apt':
-        suf = '.deb'
-    elif pkg_format == 'yum' or pkg_format == 'dnf':
-        suf = '.rpm'
+    if sys.stdout.isatty():
+        sys.stdout.write("\x1b[KDownloading ... " + str(percent) + "%\r")
+        sys.stdout.flush()
+    else:
+        # If we're not writing to a terminal, don't bother being too fancy. This
+        # prevents us from filling up logs with many progress lines.
+        if count == 1:
+            LOG.info('Downloading ...')
+        if count % 10 == 0:
+            LOG.info("    %dK (%d%%)", block_size / 1024, percent)
 
-    (_handle, filename) = tempfile.mkstemp(suffix=suf)
-    try:
-        subprocess.check_call(['curl', '--fail', '-L', '-o', filename, link],
-                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    except subprocess.CalledProcessError as error:
-        print('Unable to download MongoDB Compass, please check your internet' \
-            ' connection. If the issue persists go to' \
-            ' https://www.mongodb.com/download-center?jmp=hero#compass' \
-            ' to download the compass installer for your platform.')
+    if (count * block_size) >= total_size:
+        LOG.info("Complete!")
 
-    try:
-        out = subprocess.check_output(['file', filename]).decode('utf-8')
-    except subprocess.CalledProcessError as error:
-        print('Got an unexpected error checking file type %s' % error)
-        sys.exit(1)
 
-    if 'ASCII Text' in out:
-        print('Unknown package format downloaded. Appears there was an HTTP error.')
-        sys.exit(1)
+def download_file(url, filename, block_size=16 * 1024):
+    """Download the package from link, logging progress. Returns the filename."""
 
-    return filename
+    try:
+        response = urlopen(url)
+        total_size = int(response.info()["Content-Length"])
+        with open(filename, 'wb') as out_file:
+            for count in itertools.count(1):
+                chunk = response.read(block_size)
+                if not chunk:
+                    LOG.info('\nDownload complete!')
+                    break
+                download_progress(count, block_size, total_size)
+                out_file.write(chunk)
+        response.close()
+    except IOError:
+        raise FileDownloadError('Unable to download MongoDB Compass, please check your internet' \
+                                ' connection. If the issue persists go to' \
+                                ' https://www.mongodb.com/download-center?jmp=hero#compass' \
+                                ' to download the compass installer for your platform.')
+    else:
+        retcode, stdout, stderr = run_command('file', filename,
+                                              shell=False, stdout=subprocess.PIPE,
+                                              stderr=subprocess.PIPE)
+        if retcode != 0:
+            raise Exception(stderr)
+        if 'ASCII Text' in stdout:
+            raise FileDownloadError('Downloaded file is text, but binary was expected.')
 
 
 def install_mac(dmg):
     """Use CLI tools to mount the dmg and extract all .apps into Applications."""
+
     tmp = tempfile.mkdtemp()
-    with open(os.devnull, 'w') as fnull:
-        try:
-            subprocess.check_call(
-                ['hdiutil', 'attach', '-nobrowse', '-noautoopen', '-mountpoint', tmp, dmg],
-                stdout=fnull, stderr=fnull)
-        except subprocess.CalledProcessError as error:
-            print('Problem running hdiutil: %s' % error)
 
-        try:
-            apps = [f for f in os.listdir(tmp) if f.endswith('.app')]
-            for app in apps:
-                if path.isdir('/Applications/' + app):
-                    print('Old version found removing...')
-                    shutil.rmtree('/Applications/' + app)
-                print('Copying %s to /Applications' % app)
-                shutil.copytree(path.join(tmp, app), '/Applications/' + app)
-        # We don't really care about what errors come up here. Just log the failure
-        # and use the finally to make sure we always unmount the dmg.
-        except IOError:
-            print('Unknown error copying MongoDB Compass to /Applications/')
-        finally:
-            subprocess.check_call(['hdiutil', 'detach', tmp], stdout=fnull, stderr=fnull)
-
-        if path.isdir('/Applications/MongoDB Compass.app'):
-            subprocess.check_call(['open', '/Applications/MongoDB Compass.app'])
+    retcode, _, stderr = run_command(
+        'hdiutil', 'attach', '-nobrowse', '-noautoopen', '-mountpoint', tmp, dmg,
+        stderr=subprocess.PIPE)
+    if retcode != 0:
+        raise InstallationError('Problem running hdiutil: ' + stderr)
+
+    try:
+        apps = [f for f in os.listdir(tmp) if f.endswith('.app')]
+        for app in apps:
+            if path.isdir('/Applications/' + app):
+                LOG.info('Old version found removing...')
+                shutil.rmtree('/Applications/' + app)
+            LOG.info('Copying %s to /Applications', app)
+            shutil.copytree(path.join(tmp, app), '/Applications/' + app)
+    # We don't really care about what errors come up here. Just log the failure
+    # and use the finally to make sure we always unmount the dmg.
+    except IOError as error:
+        raise InstallationError('Error copying MongoDB Compass to /Applications: ' + error.message)
+    finally:
+        run_command('hdiutil', 'detach', tmp)
+
+    if path.isdir('/Applications/MongoDB Compass.app'):
+        retcode, _, stderr = run_command('open', '/Applications/MongoDB Compass.app',
+                                         stderr=subprocess.PIPE)
+        if retcode == 0:
             return
-        if path.isdir('/Applications/MongoDB Compass Community.app'):
-            subprocess.check_call(['open', '/Applications/MongoDB Compass Community.app'])
+
+    if path.isdir('/Applications/MongoDB Compass Community.app'):
+        retcode, _, stderr = run_command('open', '/Applications/MongoDB Compass Community.app',
+                                         stderr=subprocess.PIPE)
+        if retcode == 0:
             return
 
+    raise InstallationError('Problem opening application: ' + stderr)
 
-def install_linux(pkg_format, pkg_file):
+
+def install_linux(package_manager, pkg_file):
     """Use the package manager indicated by pkg_format to install pkg_file."""
-    if pkg_format == 'yum':
-        install = ['yum', 'localinstall', '--assumeyes', pkg_file]
-    elif pkg_format == 'apt':
-        # dpkg returns an error code when it fails to install dependencies
-        # so just run it and let apt-get tell us if something went wrong
-        subprocess.call(['dpkg', '--install', pkg_file])
-        install = ['apt-get', 'install', '-f', '--yes']
-    elif pkg_format == 'dnf':
-        install = ['dnf', 'install', '--assumeyes', pkg_file]
-    else:
-        print('No available installation methods.')
-        sys.exit(1)
 
-    subprocess.check_call(install)
+    install = tuple([package_manager] + LINUX_PKG_MANAGER_ARGS[package_manager] + [pkg_file])
 
+    retcode, _, stderr = run_command(*install, stderr=subprocess.PIPE)
+    if retcode != 0:
+        raise InstallationError('Problem running package manager: ' + stderr)
 
-def is_supported_distro(pkg_format):
-    if pkg_format in ('apt', 'yum', 'dnf'):
+
+def get_osx_version():
+    """Gets the OSX version."""
+
+    mac_version, _, _ = platform.mac_ver()
+    # Get Mac OSX Major.Minor verson as a float
+    return float('.'.join(mac_version.split('.')[:2]))
+
+
+def get_rhel_version():
+    """Gets the RHEL compatibility version."""
+
+    rhel_version_re = re.compile(r' release (\d+\.\d+)', re.MULTILINE)
+
+    with open('/etc/redhat-release', 'r') as release_file:
+        match = rhel_version_re.search(release_file.read())
+
+        if not match:
+            return None
+
+        return float('.'.join(match.group(1).split('.')))
+
+
+def get_debian_release():
+    """Gets the Debian release."""
+
+    # Matches from the beginning of the line to a slash or the end of the line
+    debian_version_re = re.compile(r'^([^/]+)(?:/|$)', re.MULTILINE)
+
+    with open('/etc/debian_version', 'r') as version_file:
+        match = debian_version_re.search(version_file.read())
+
+        if not match:
+            return None
+
+        return match.group(1)
+
+
+def is_supported_linux():  # pylint: disable=too-many-branches,too-many-return-statements
+    """Checks whether the Linux version is supported by the Compass package."""
+
+    # This isn't intended to be an exhaustive check of all compatible
+    # distributions. We're only checking the ones that are likely to be
+    # used when installing either a .deb or a .rpm package.
+    if os.path.exists('/etc/redhat-release'):
+        rhel_version = get_rhel_version()
+        if rhel_version is not None and rhel_version > SUPPORTED_RHEL_VERSION:
+            return True
+
+    if os.path.exists('/etc/debian_version'):
+        debian_release = get_debian_release()
+        try:
+            # The debian version string is sometimes just a float. This appears
+            # to be only for older releases, but we'll try to be safe here.
+            if float(debian_release) >= SUPPORTED_DEBIAN_VERSION:
+                return True
+            return False
+        except ValueError:
+            if debian_release is not None and debian_release in SUPPORTED_DEBIAN_RELEASES:
+                return True
+            return False
+
+    # Matches the key and value separated by an =, with the value optionally
+    # being enclosed in quotes.
+    os_release_re = re.compile(r'^([A-Z_]+)="?([^"]+)(?:"|$)')
+    if os.path.exists('/etc/os-release'):
+        os_id = None
+        version_id = None
+        with open('/etc/os-release', 'r') as os_release:
+            for line in os_release.readlines():
+                match = os_release_re.search(line)
+                if match:
+                    if match.group(1) == "ID":
+                        os_id = match.group(2)
+                    if match.group(1) == "VERSION_ID":
+                        version_id = match.group(2)
+        # The only two known incompatible distributions that have a
+        # /etc/os-release provided by systemd are these two. It's best here to
+        # assume the rest will succeed and take the ones that fail individually.
+        if os_id == "amzn":
+            if version_id.startswith("2018") or version_id.startswith("2016"):
+                return False
+        if os_id == "opensuse":
+            return False
         return True
+
     return False
 
 
-def is_supported_mac_version():
-    v, _, _ = platform.mac_ver()
-    # Get Mac OSX Major.Minor verson as a float
-    ver_float = float('.'.join(v.split('.')[:2]))
-    return ver_float >= 10.10
+def prerequisites_satisfied(package_manager):
+    """Check that prerequisites are satisfied before downloading and installing."""
+
+    if package_manager is None:
+        LOG.error('You are using an unsupported platform.\n' \
+              'Please visit: https://compass.mongodb.com/community-supported-platforms' \
+              ' to view available community supported packages.')
+        return False
+
+    if platform.machine() != 'x86_64':
+        LOG.error('Sorry, MongoDB Compass is only supported on 64-bit Intel platforms.' \
+              ' If you believe you\'re seeing this message in error please open a' \
+              ' ticket on the SERVER project at https://jira.mongodb.org/')
+        return False
+
+    if sys.platform == 'osx' and get_osx_version() < SUPPORTED_OSX_VERSION:
+        LOG.error('You are using an unsupported Mac OSX version. Please upgrade\n' \
+              'to at least version 10.10 (Yosemite) to install Compass.')
+        return False
+
+    if sys.platform.startswith('linux'):
+        if os.getuid() != 0:
+            LOG.error('You must run this script as root.')
+            return False
+
+        if not is_supported_linux():
+            LOG.error('You are using an unsupported Linux distribution.\n' \
+                      'Please visit: https://compass.mongodb.com/community-supported-platforms' \
+                      ' to view available community supported packages.')
+            return False
+
+    return True
+
+
+def get_package_type(package_manager):
+    """Gets the package type from the detected package manager."""
+
+    if sys.platform.startswith('linux'):
+        package_type = 'linux'
+        if package_manager == 'apt':
+            package_type += '_deb'
+        elif package_manager in ('yum', 'dnf'):
+            package_type += '_rpm'
+    elif sys.platform == 'darwin':
+        package_type = 'osx'
+
+    return package_type
 
 
 def download_and_install_compass():
     """Download and install compass for this platform."""
-    os_type = sys.platform
-    pkg_format = get_pkg_format()
-
-    # Sometimes sys.platform gives us 'linux2' and we only want 'linux'
-    if os_type.startswith('linux'):
-        os_type = 'linux'
-        if pkg_format == 'apt':
-            os_type += '_deb'
-        elif pkg_format == 'yum' or pkg_format == 'dnf':
-            os_type += '_rpm'
-    elif os_type == 'darwin':
-        os_type = 'osx'
-
-    if os_type.startswith('linux') and os.getuid() != 0:
-        print('You must run this script as root.')
-        sys.exit(1)
 
-    if os_type.startswith('linux') and not is_supported_distro(pkg_format):
-        print('You are using an unsupported Linux distribution.\n' \
-            'Please visit: https://compass.mongodb.com/community-supported-platforms' \
-            ' to view available community supported packages.')
-        sys.exit(1)
+    package_manager = get_package_manager()
 
-    if os_type == 'osx' and not is_supported_mac_version():
-        print('You are using an unsupported Mac OSX version. Please upgrade\n' \
-            'to at least version 10.10 (Yosemite) to install Compass.')
-        sys.exit(1)
+    if not prerequisites_satisfied(package_manager):
+        return sys.exit(1)
 
-    if platform.machine() != 'x86_64':
-        print('Sorry, MongoDB Compass is only supported on 64 bit platforms.' \
-            ' If you believe you\'re seeing this message in error please open a' \
-            ' ticket on the SERVER project at https://jira.mongodb.org/')
+    _, filename = tempfile.mkstemp(suffix=PKG_MANAGER_SUFFIXES[package_manager])
+    is_successful = True
+
+    url = DOWNLOAD_BASE_URL + "/" + get_package_type(package_manager)
+    LOG.info('Retrieving the Compass package from %s', url)
 
-    link = 'https://compass.mongodb.com/api/v2/download/latest/compass/stable/' + os_type
-    print('Downloading the package...')
-    try:
-        pkg = download_pkg(link, pkg_format=pkg_format)
-    # This should not execute we are catching errors in all of these functions
-    # but these are a catch all so we don't just dump Python stack traces to
-    # the user in the case we have a new failure case we didn't think about.
-    except Exception:
-        print('Unkown error downloading compass. Please open a ticket on the' \
-            ' SERVER project at https://jira.mongodb.org/')
-
-    print('Installing the package...')
     try:
-        if os_type == 'osx':
-            install_mac(pkg)
-        elif os_type.startswith('linux'):
-            install_linux(pkg_format, pkg)
-        else:
-            print('Unrecognized os_type: %s' % os_type)
-    except Exception:
-        print('Unkown error downloading compass. Please open a ticket on the' \
-            ' SERVER project at https://jira.mongodb.org/')
+        download_file(url=url, filename=filename)
+    except FileDownloadError as error:
+        LOG.exception(error)
+        is_successful = False
+    except Exception as error:  # pylint: disable=broad-except
+        LOG.info('Unexpected error when downloading: %s', error.message)
+        is_successful = False
+
+    else:
+        LOG.info('Installing the package...')
+        try:
+            if package_manager == 'dmg':
+                install_mac(filename)
+            else:
+                install_linux(package_manager, filename)
+        except InstallationError  as error:
+            LOG.exception(error)
+            is_successful = False
+        except Exception as error:  # pylint: disable=broad-except
+            LOG.error('Unexpected error when installing: %s', error.message)
+            is_successful = False
 
-    print('Cleaning up...')
-    os.remove(pkg)
-    print('Done!')
+    LOG.info('Cleaning up...')
+    os.remove(filename)
+    LOG.info('Done!')
+
+    return is_successful
+
+
+# Set up logging to stdout and stderr for the console only
+LOG = logging.getLogger()
+LOG.setLevel(logging.DEBUG)
+
+LOG_STDOUT_HANDLER = logging.StreamHandler(sys.stdout)
+LOG_STDOUT_HANDLER.setLevel(logging.INFO)
+LOG_STDOUT_HANDLER.addFilter(LogLevelFilter(logging.INFO))
+LOG.addHandler(LOG_STDOUT_HANDLER)
+
+LOG_STDERR_HANDLER = logging.StreamHandler(sys.stderr)
+LOG_STDERR_HANDLER.setLevel(logging.ERROR)
+LOG_STDERR_HANDLER.addFilter(LogLevelFilter(logging.ERROR))
+LOG.addHandler(LOG_STDERR_HANDLER)
 
 
 if __name__ == '__main__':
-    download_and_install_compass()
+    # Set up a log file for both stdout and stderr, but only when this is being
+    # executed as a script. This facilitates testing without polluting the test
+    # environment.
+    LOG_FILE_NAME = '/tmp/install_compass.log'
+    LOG_FILE_HANDLER = logging.FileHandler(LOG_FILE_NAME)
+    LOG_FILE_HANDLER.setLevel(logging.DEBUG)
+    LOG_FILE_FORMATTER = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    LOG_FILE_HANDLER.setFormatter(LOG_FILE_FORMATTER)
+    LOG.addHandler(LOG_FILE_HANDLER)
+
+    SUCCESS = download_and_install_compass()
+    LOG.info('A log file for this installation can be found at %s', LOG_FILE_NAME)
+
+    if not SUCCESS:
+        LOG.error('Please open a ticket with the log file attached on the SERVER ' \
+                  'project at https://jira.mongodb.org/')
+        sys.exit(1)