diff options
author | Mathew Robinson <chasinglogic@gmail.com> | 2019-02-19 10:50:57 -0500 |
---|---|---|
committer | Mathew Robinson <chasinglogic@gmail.com> | 2019-04-08 14:08:49 -0400 |
commit | 8dd6d4755734ed37c1b98dfdefce3ca6bc65f1f6 (patch) | |
tree | 69e936c4953cbead2e3bae2690157c5fe75e709d /buildscripts/linter | |
parent | c600aa9d7423eca8151daf626e2799d9a6c7b31c (diff) | |
download | mongo-8dd6d4755734ed37c1b98dfdefce3ca6bc65f1f6.tar.gz |
SERVER-32295 Support Python 3
Diffstat (limited to 'buildscripts/linter')
-rw-r--r-- | buildscripts/linter/base.py | 6 | ||||
-rw-r--r-- | buildscripts/linter/git.py | 4 | ||||
-rw-r--r-- | buildscripts/linter/mypy.py | 9 | ||||
-rw-r--r-- | buildscripts/linter/parallel.py | 8 | ||||
-rw-r--r-- | buildscripts/linter/pydocstyle.py | 2 | ||||
-rw-r--r-- | buildscripts/linter/pylint.py | 4 | ||||
-rw-r--r-- | buildscripts/linter/runner.py | 41 | ||||
-rw-r--r-- | buildscripts/linter/yapf.py | 4 |
8 files changed, 39 insertions, 39 deletions
diff --git a/buildscripts/linter/base.py b/buildscripts/linter/base.py index f22f59e4f01..bb22ea7cf80 100644 --- a/buildscripts/linter/base.py +++ b/buildscripts/linter/base.py @@ -1,16 +1,12 @@ """Base class and support functions for linters.""" -from __future__ import absolute_import -from __future__ import print_function from abc import ABCMeta, abstractmethod from typing import Dict, List, Optional -class LinterBase(object): +class LinterBase(object, metaclass=ABCMeta): """Base Class for all linters.""" - __metaclass__ = ABCMeta - def __init__(self, cmd_name, required_version, cmd_location=None): # type: (str, str, Optional[str]) -> None """ diff --git a/buildscripts/linter/git.py b/buildscripts/linter/git.py index 9b5f55481d4..4fa15f65907 100644 --- a/buildscripts/linter/git.py +++ b/buildscripts/linter/git.py @@ -1,6 +1,4 @@ """Git Utility functions.""" -from __future__ import absolute_import -from __future__ import print_function import itertools import os @@ -196,7 +194,7 @@ def get_files_to_check_from_patch(patches, filter_function): lines = [] # type: List[str] for patch in patches: - with open(patch, "rb") as infile: + with open(patch, "r") as infile: lines += infile.readlines() candidates = [check.match(line).group(1) for line in lines if check.match(line)] diff --git a/buildscripts/linter/mypy.py b/buildscripts/linter/mypy.py index c720ae8f870..1189093dd2c 100644 --- a/buildscripts/linter/mypy.py +++ b/buildscripts/linter/mypy.py @@ -1,6 +1,4 @@ """Mypy linter support module.""" -from __future__ import absolute_import -from __future__ import print_function import os from typing import List @@ -26,7 +24,12 @@ class MypyLinter(base.LinterBase): def get_lint_cmd_args(self, file_name): # type: (str) -> List[str] """Get the command to run a linter.""" - return [file_name] + # Only idl and linter should be type checked by mypy. Other + # files return errors under python 3 type checking. If we + # return an empty list the runner will skip this file. + if 'idl' in file_name or 'linter' in file_name: + return [file_name] + return [] def ignore_interpreter(self): # type: () -> bool diff --git a/buildscripts/linter/parallel.py b/buildscripts/linter/parallel.py index 0648bfb16e7..b80ec7f2c1b 100644 --- a/buildscripts/linter/parallel.py +++ b/buildscripts/linter/parallel.py @@ -1,8 +1,6 @@ """Utility code to execute code in parallel.""" -from __future__ import absolute_import -from __future__ import print_function -import Queue +import queue import threading import time from multiprocessing import cpu_count @@ -17,7 +15,7 @@ def parallel_process(items, func): except NotImplementedError: cpus = 1 - task_queue = Queue.Queue() # type: Queue.Queue + task_queue = queue.Queue() # type: Queue.Queue # Use a list so that worker function will capture this variable pp_event = threading.Event() @@ -30,7 +28,7 @@ def parallel_process(items, func): while not pp_event.is_set(): try: item = task_queue.get_nowait() - except Queue.Empty: + except queue.Empty: # if the queue is empty, exit the worker thread pp_event.set() return diff --git a/buildscripts/linter/pydocstyle.py b/buildscripts/linter/pydocstyle.py index b259becfd1c..8d6b7dde0c7 100644 --- a/buildscripts/linter/pydocstyle.py +++ b/buildscripts/linter/pydocstyle.py @@ -1,6 +1,4 @@ """PyDocStyle linter support module.""" -from __future__ import absolute_import -from __future__ import print_function from typing import List diff --git a/buildscripts/linter/pylint.py b/buildscripts/linter/pylint.py index 71a062f9076..58f452d09b6 100644 --- a/buildscripts/linter/pylint.py +++ b/buildscripts/linter/pylint.py @@ -1,6 +1,4 @@ """PyLint linter support module.""" -from __future__ import absolute_import -from __future__ import print_function import os from typing import List @@ -15,7 +13,7 @@ class PyLintLinter(base.LinterBase): def __init__(self): # type: () -> None """Create a pylint linter.""" - super(PyLintLinter, self).__init__("pylint", "pylint 1.9.3") + super(PyLintLinter, self).__init__("pylint", "pylint 2.3.1") def get_lint_version_cmd_args(self): # type: () -> List[str] diff --git a/buildscripts/linter/runner.py b/buildscripts/linter/runner.py index 83b58dd9884..af2a83de62d 100644 --- a/buildscripts/linter/runner.py +++ b/buildscripts/linter/runner.py @@ -1,6 +1,4 @@ """Class to support running various linters in a common framework.""" -from __future__ import absolute_import -from __future__ import print_function import difflib import logging @@ -23,19 +21,22 @@ def _check_version(linter, cmd_path, args): logging.info(str(cmd)) process_handle = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, stderr = process_handle.communicate() + output = output.decode('utf-8') if process_handle.returncode: - logging.info("Version check failed for [%s], return code '%d'." + - "Standard Output:\n%s\nStandard Error:\n%s", cmd, - process_handle.returncode, output, stderr) + logging.info( + "Version check failed for [%s], return code '%d'." + "Standard Output:\n%s\nStandard Error:\n%s", cmd, process_handle.returncode, output, + stderr) required_version = re.escape(linter.required_version) pattern = r"\b%s\b" % (required_version) if not re.search(pattern, output): - logging.info("Linter %s has wrong version for '%s'. Expected '%s'," + - "Standard Output:\n'%s'\nStandard Error:\n%s", linter.cmd_name, cmd, - required_version, output, stderr) + logging.info( + "Linter %s has wrong version for '%s'. Expected '%s'," + "Standard Output:\n'%s'\nStandard Error:\n%s", linter.cmd_name, cmd, + required_version, output, stderr) return False except OSError as os_error: @@ -126,7 +127,8 @@ def find_linters(linter_list, config_dict): for linter in linter_list: linter_instance = _find_linter(linter, config_dict) if not linter_instance: - logging.error("""\ + logging.error( + """\ Could not find the correct version of linter '%s', expected '%s'. Check your PATH environment variable or re-run with --verbose for more information. @@ -166,16 +168,25 @@ class LintRunner(object): # type: (base.LinterInstance, str) -> bool """Run the specified linter for the file.""" - cmd = linter.cmd_path + linter.linter.get_lint_cmd_args(file_name) + cmd = linter.cmd_path + cmd += linter.linter.get_lint_cmd_args(file_name) + if cmd == linter.cmd_path: + # If args is empty it means we didn't get a valid command + # to run and so should skip this file. + # + # For example the MyPy linter class will return empty args + # for non-idl files since they shouldn't be type checked. + return True + logging.debug(str(cmd)) try: if linter.linter.needs_file_diff(): # Need a file diff with open(file_name, 'rb') as original_text: - original_file = original_text.read() + original_file = original_text.read().decode('utf-8') - formatted_file = subprocess.check_output(cmd) + formatted_file = subprocess.check_output(cmd).decode('utf-8') if original_file != formatted_file: original_lines = original_file.splitlines() formatted_lines = formatted_file.splitlines() @@ -196,7 +207,7 @@ class LintRunner(object): return False else: - output = subprocess.check_output(cmd) + output = subprocess.check_output(cmd).decode('utf-8') # On Windows, mypy.bat returns 0 even if there are length failures so we need to # check if there was any output @@ -205,7 +216,7 @@ class LintRunner(object): return False except subprocess.CalledProcessError as cpe: - self._safe_print("CMD [%s] failed:\n%s" % (cmd, cpe.output)) + self._safe_print("CMD [%s] failed:\n%s" % (cmd, cpe.output.decode('utf-8'))) return False return True @@ -217,7 +228,7 @@ class LintRunner(object): logging.debug(str(cmd)) try: - subprocess.check_output(cmd) + subprocess.check_output(cmd).decode('utf-8') except subprocess.CalledProcessError as cpe: self._safe_print("CMD [%s] failed:\n%s" % (cmd, cpe.output)) return False diff --git a/buildscripts/linter/yapf.py b/buildscripts/linter/yapf.py index 1ea3da7bae3..d787810a0da 100644 --- a/buildscripts/linter/yapf.py +++ b/buildscripts/linter/yapf.py @@ -1,6 +1,4 @@ """YAPF linter support module.""" -from __future__ import absolute_import -from __future__ import print_function from typing import List @@ -13,7 +11,7 @@ class YapfLinter(base.LinterBase): def __init__(self): # type: () -> None """Create a yapf linter.""" - super(YapfLinter, self).__init__("yapf", "yapf 0.21.0") + super(YapfLinter, self).__init__("yapf", "yapf 0.26.0") def get_lint_version_cmd_args(self): # type: () -> List[str] |