SERVER-40559 Python linters do not run after upgrade to Python 3

1 files changed, 22 insertions, 18 deletions

diff --git a/buildscripts/linter/runner.py b/buildscripts/linter/runner.py
index 8a3951d9635..40b68bb2aa0 100644
--- a/buildscripts/linter/runner.py
+++ b/buildscripts/linter/runner.py
@@ -4,6 +4,7 @@ import difflib
 import logging
 import os
 import re
+import site
 import subprocess
 import sys
 import threading
@@ -56,6 +57,7 @@ def _find_linter(linter, config_dict):
     Return a LinterInstance with the location of the linter binary if a linter binary with the
     matching version is found. None otherwise.
     """
+    # pylint: disable=too-many-branches,too-many-return-statements
 
     if linter.cmd_name in config_dict and config_dict[linter.cmd_name] is not None:
         cmd = [config_dict[linter.cmd_name]]
@@ -67,8 +69,9 @@ def _find_linter(linter, config_dict):
 
     # Search for tool
     # 1. In the same directory as the interpreter
-    # 2. The current path
-    # 3. In '/opt/mongodbtoolchain/v3/bin' if virtualenv is set up.
+    # 2. Check user base -- i.e. site.USERBASE. With "pip install --user" puts files
+    # 3. The current path
+    # 4. In '/opt/mongodbtoolchain/v3/bin' if virtualenv is set up.
     python_dir = os.path.dirname(sys.executable)
     if sys.platform == "win32":
         # On Windows, these scripts are installed in %PYTHONDIR%\scripts like
@@ -95,12 +98,17 @@ def _find_linter(linter, config_dict):
     logging.info("First version check failed for linter '%s', trying a different location.",
                  linter.cmd_name)
 
-    # Check 2: current path
+    # Check 2: Check USERBASE
+    cmd = [os.path.join(site.getuserbase(), "bin", linter.cmd_name)]
+    if _check_version(linter, cmd, linter.get_lint_version_cmd_args()):
+        return base.LinterInstance(linter, cmd)
+
+    # Check 3: current path
     cmd = [linter.cmd_name]
     if _check_version(linter, cmd, linter.get_lint_version_cmd_args()):
         return base.LinterInstance(linter, cmd)
 
-    # Check 3: When a virtualenv is setup the linter modules are not installed, so we need
+    # Check 4: When a virtualenv is setup the linter modules are not installed, so we need
     # to use the linters installed in '/opt/mongodbtoolchain/v3/bin'.
     cmd = [sys.executable, os.path.join('/opt/mongodbtoolchain/v3/bin', linter.cmd_name)]
     if _check_version(linter, cmd, linter.get_lint_version_cmd_args()):
@@ -123,7 +131,7 @@ Could not find the correct version of linter '%s', expected '%s'. Check your
 PATH environment variable or re-run with --verbose for more information.
 
 To fix, install the needed python modules for Python 3.x:
-   sudo pip3 install -r etc/pip/lint-requirements.txt
+   python3 -m pip install -r etc/pip/lint-requirements.txt
 
 These commands are typically available via packages with names like python-pip,
 or python3-pip. See your OS documentation for help.
@@ -156,10 +164,10 @@ class LintRunner(object):
     def run_lint(self, linter, file_name):
         # type: (base.LinterInstance, str) -> bool
         """Run the specified linter for the file."""
+        # pylint: disable=too-many-locals
 
-        cmd = linter.cmd_path
-        cmd += linter.linter.get_lint_cmd_args(file_name)
-        if cmd == linter.cmd_path:
+        linter_args = linter.linter.get_lint_cmd_args(file_name)
+        if not linter_args:
             # If args is empty it means we didn't get a valid command
             # to run and so should skip this file.
             #
@@ -167,7 +175,9 @@ class LintRunner(object):
             # for non-idl files since they shouldn't be type checked.
             return True
 
-        logging.debug(str(cmd))
+        cmd = linter.cmd_path + linter_args
+
+        logging.debug(' '.join(cmd))
 
         try:
             if linter.linter.needs_file_diff():
@@ -196,16 +206,10 @@ class LintRunner(object):
 
                     return False
             else:
-                output = subprocess.check_output(cmd).decode('utf-8')
-
-                # On Windows, mypy.bat returns 0 even if there are length failures so we need to
-                # check if there was any output
-                if output and sys.platform == "win32":
-                    self._safe_print("CMD [%s] output:\n%s" % (cmd, output))
-                    return False
+                subprocess.check_output(cmd).decode('utf-8')
 
         except subprocess.CalledProcessError as cpe:
-            self._safe_print("CMD [%s] failed:\n%s" % (cmd, cpe.output.decode('utf-8')))
+            self._safe_print("CMD [%s] failed:\n%s" % (' '.join(cmd), cpe.output.decode('utf-8')))
             return False
 
         return True
@@ -219,7 +223,7 @@ class LintRunner(object):
         try:
             subprocess.check_output(cmd).decode('utf-8')
         except subprocess.CalledProcessError as cpe:
-            self._safe_print("CMD [%s] failed:\n%s" % (cmd, cpe.output))
+            self._safe_print("CMD [%s] failed:\n%s" % (' '.join(cmd), cpe.output))
             return False
 
         return True