From 9e0c79b14f17165087b6cec54bc49914be91ca88 Mon Sep 17 00:00:00 2001 From: Ryan Egesdahl Date: Thu, 27 Aug 2020 11:05:57 -0700 Subject: SERVER-50592 Update mypy and GitPython pip requirements Update the mypy and GitPython pip modules that are required for linting builds before commits to compatible versions rather than hard version requirements. (cherry picked from commit ca4df25002a60910b38bfdd8d71eb5bff5a79b49) (cherry picked from commit 2990572b79734dd9178db207958f962d50c2daf2) --- buildscripts/clang_format.py | 1 + buildscripts/eslint.py | 1 + buildscripts/idl/idl/common.py | 2 ++ buildscripts/linter/mypy.py | 2 +- buildscripts/linter/pylint.py | 2 +- buildscripts/linter/runner.py | 26 +++++++++++++++-------- buildscripts/linter/yapf.py | 2 +- buildscripts/resmokelib/testing/hooks/stepdown.py | 1 + etc/pip/components/evergreen.req | 2 +- mypy.ini | 3 +++ 10 files changed, 29 insertions(+), 13 deletions(-) diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py index 6acdcaff192..47466975a2a 100755 --- a/buildscripts/clang_format.py +++ b/buildscripts/clang_format.py @@ -70,6 +70,7 @@ def callo(args, **kwargs): def get_tar_path(version, tar_path): """Return the path to clang-format in the llvm tarball.""" + # pylint: disable=too-many-function-args return CLANG_FORMAT_SOURCE_TAR_BASE.substitute(version=version, tar_path=tar_path) diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py index ed68e9e45a1..e0e8c7672c6 100755 --- a/buildscripts/eslint.py +++ b/buildscripts/eslint.py @@ -87,6 +87,7 @@ def get_eslint_from_cache(dest_file, platform, arch): print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION, url, temp_tar_file)) urllib.request.urlretrieve(url, temp_tar_file) + # pylint: disable=too-many-function-args eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(platform=platform, arch=arch) extract_eslint(temp_tar_file, eslint_distfile) shutil.move(eslint_distfile, dest_file) diff --git a/buildscripts/idl/idl/common.py b/buildscripts/idl/idl/common.py index 39f5f032183..768f0e8e11f 100644 --- a/buildscripts/idl/idl/common.py +++ b/buildscripts/idl/idl/common.py @@ -84,6 +84,7 @@ def template_format(template, template_params=None): # str. # See https://docs.python.org/2/library/string.html#template-strings template = _escape_template_string(template) + # pylint: disable=too-many-function-args return string.Template(template).substitute(template_params) # type: ignore @@ -94,6 +95,7 @@ def template_args(template, **kwargs): # str. # See https://docs.python.org/2/library/string.html#template-strings template = _escape_template_string(template) + # pylint: disable=too-many-function-args return string.Template(template).substitute(kwargs) # type: ignore diff --git a/buildscripts/linter/mypy.py b/buildscripts/linter/mypy.py index c11e5b3c249..e87d26c5e43 100644 --- a/buildscripts/linter/mypy.py +++ b/buildscripts/linter/mypy.py @@ -14,7 +14,7 @@ class MypyLinter(base.LinterBase): """Create a mypy linter.""" # User can override the location of mypy from an environment variable. - super(MypyLinter, self).__init__("mypy", "mypy 0.800", os.getenv("MYPY")) + super(MypyLinter, self).__init__("mypy", "0.800", os.getenv("MYPY")) def get_lint_version_cmd_args(self): # type: () -> List[str] diff --git a/buildscripts/linter/pylint.py b/buildscripts/linter/pylint.py index 58f452d09b6..7f5c921f032 100644 --- a/buildscripts/linter/pylint.py +++ b/buildscripts/linter/pylint.py @@ -13,7 +13,7 @@ class PyLintLinter(base.LinterBase): def __init__(self): # type: () -> None """Create a pylint linter.""" - super(PyLintLinter, self).__init__("pylint", "pylint 2.3.1") + super(PyLintLinter, self).__init__("pylint", "2.3.1") def get_lint_version_cmd_args(self): # type: () -> List[str] diff --git a/buildscripts/linter/runner.py b/buildscripts/linter/runner.py index 40b68bb2aa0..623a3c4295e 100644 --- a/buildscripts/linter/runner.py +++ b/buildscripts/linter/runner.py @@ -1,5 +1,7 @@ """Class to support running various linters in a common framework.""" +from typing import Dict, List, Optional + import difflib import logging import os @@ -8,7 +10,7 @@ import site import subprocess import sys import threading -from typing import Dict, List, Optional +import pkg_resources from . import base @@ -22,22 +24,28 @@ def _check_version(linter, cmd_path, args): logging.info(str(cmd)) process_handle = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, stderr = process_handle.communicate() - output = output.decode('utf-8') + decoded_output = output.decode('utf-8') if process_handle.returncode: logging.info( "Version check failed for [%s], return code '%d'." - "Standard Output:\n%s\nStandard Error:\n%s", cmd, process_handle.returncode, output, - stderr) + "Standard Output:\n%s\nStandard Error:\n%s", cmd, process_handle.returncode, + decoded_output, stderr) + + pattern = r"\b(?:(%s) )?(?P\S+)\b" % (linter.cmd_name) + required_version = pkg_resources.parse_version(linter.required_version) - required_version = re.escape(linter.required_version) + match = re.search(pattern, decoded_output) + if match: + found_version = match.group('version') + else: + found_version = '0.0' - pattern = r"\b%s\b" % (required_version) - if not re.search(pattern, output): + if pkg_resources.parse_version(found_version) < required_version: logging.info( - "Linter %s has wrong version for '%s'. Expected '%s'," + "Linter %s has wrong version for '%s'. Expected >= '%s'," "Standard Output:\n'%s'\nStandard Error:\n%s", linter.cmd_name, cmd, - required_version, output, stderr) + required_version, decoded_output, stderr) return False except OSError as os_error: diff --git a/buildscripts/linter/yapf.py b/buildscripts/linter/yapf.py index d787810a0da..e339e3fc763 100644 --- a/buildscripts/linter/yapf.py +++ b/buildscripts/linter/yapf.py @@ -11,7 +11,7 @@ class YapfLinter(base.LinterBase): def __init__(self): # type: () -> None """Create a yapf linter.""" - super(YapfLinter, self).__init__("yapf", "yapf 0.26.0") + super(YapfLinter, self).__init__("yapf", "0.26.0") def get_lint_version_cmd_args(self): # type: () -> List[str] diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py index 3cf2aa5b9ab..76f9c26464b 100644 --- a/buildscripts/resmokelib/testing/hooks/stepdown.py +++ b/buildscripts/resmokelib/testing/hooks/stepdown.py @@ -372,6 +372,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at self._is_idle_evt = threading.Event() self._is_idle_evt.set() + # pylint: disable=too-many-function-args self._step_up_stats = collections.Counter() def run(self): diff --git a/etc/pip/components/evergreen.req b/etc/pip/components/evergreen.req index 4853e5a6203..87b0aec8868 100644 --- a/etc/pip/components/evergreen.req +++ b/etc/pip/components/evergreen.req @@ -1,5 +1,5 @@ click ~= 7.0 -GitPython ~= 2.1.11 +GitPython ~= 3.1.7 psutil pydantic ~= 1.7.3 structlog ~= 19.2.0 diff --git a/mypy.ini b/mypy.ini index a3917513153..6c6bf55f4c3 100644 --- a/mypy.ini +++ b/mypy.ini @@ -9,6 +9,9 @@ follow_imports = silent # This will limit effectiveness but avoids mypy complaining about running code. ignore_missing_imports = True +# Make None compatible with every type (the default prior to v 0.600) +strict_optional = False + [mypy-idl.*] # Error if any code is missing type annotations. disallow_untyped_defs = True -- cgit v1.2.1