summaryrefslogtreecommitdiff
path: root/buildscripts/pylinters.py
diff options
context:
space:
mode:
authorMark Benvenuto <mark.benvenuto@mongodb.com>2017-04-18 18:18:26 -0400
committerMark Benvenuto <mark.benvenuto@mongodb.com>2017-04-18 18:18:26 -0400
commitc776e095ac25d0426624f4a84c03f0094c3c661f (patch)
tree4ee2b13dd37a64194934abf672e23759c112ca0b /buildscripts/pylinters.py
parentb9f9195390142f4e1363dc83b986ead5dc8993b8 (diff)
downloadmongo-c776e095ac25d0426624f4a84c03f0094c3c661f.tar.gz
SERVER-28308 Integrate python linting for IDL into Evergreen
Diffstat (limited to 'buildscripts/pylinters.py')
-rwxr-xr-xbuildscripts/pylinters.py210
1 files changed, 210 insertions, 0 deletions
diff --git a/buildscripts/pylinters.py b/buildscripts/pylinters.py
new file mode 100755
index 00000000000..539979e7dfe
--- /dev/null
+++ b/buildscripts/pylinters.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python2
+"""Extensible script to run one or more Python Linters across a subset of files in parallel."""
+from __future__ import absolute_import
+from __future__ import print_function
+
+import argparse
+import logging
+import os
+import sys
+import threading
+from abc import ABCMeta, abstractmethod
+from typing import Any, Dict, List
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
+
+from buildscripts.linter import base # pylint: disable=wrong-import-position
+from buildscripts.linter import git # pylint: disable=wrong-import-position
+from buildscripts.linter import mypy # pylint: disable=wrong-import-position
+from buildscripts.linter import parallel # pylint: disable=wrong-import-position
+from buildscripts.linter import pydocstyle # pylint: disable=wrong-import-position
+from buildscripts.linter import pylint # pylint: disable=wrong-import-position
+from buildscripts.linter import runner # pylint: disable=wrong-import-position
+from buildscripts.linter import yapf # pylint: disable=wrong-import-position
+
+# List of supported linters
+_LINTERS = [
+ yapf.YapfLinter(),
+ pylint.PyLintLinter(),
+ pydocstyle.PyDocstyleLinter(),
+ mypy.MypyLinter(),
+]
+
+
+def get_py_linter(linter_filter):
+ # type: (str) -> List[base.LinterBase]
+ """
+ Get a list of linters to use.
+
+ 'all' or None - select all linters
+ 'a,b,c' - a comma delimited list is describes a list of linters to choose
+ """
+ if linter_filter is None or linter_filter == "all":
+ return _LINTERS
+
+ linter_list = linter_filter.split(",")
+
+ linter_candidates = [linter for linter in _LINTERS if linter.cmd_name in linter_list]
+
+ if len(linter_candidates) == 0:
+ raise ValueError("No linters found for filter '%s'" % (linter_filter))
+
+ return linter_candidates
+
+
+def is_interesting_file(file_name):
+ # type: (str) -> bool
+ """"Return true if this file should be checked."""
+ return file_name.endswith(".py") and (file_name.startswith("buildscripts/idl") or
+ file_name.startswith("buildscripts/linter") or
+ file_name.startswith("buildscripts/pylinters.py"))
+
+
+def _get_build_dir():
+ # type: () -> str
+ """Get the location of the scons' build directory in case we need to download clang-format."""
+ return os.path.join(git.get_base_dir(), "build")
+
+
+def _lint_files(linters, config_dict, file_names):
+ # type: (str, Dict[str, str], List[str]) -> None
+ """Lint a list of files with clang-format."""
+ linter_list = get_py_linter(linters)
+
+ lint_runner = runner.LintRunner()
+
+ linter_instances = runner.find_linters(linter_list, config_dict)
+ if not linter_instances:
+ sys.exit(1)
+
+ for linter in linter_instances:
+ run_fix = lambda param1: lint_runner.run_lint(linter, param1) # pylint: disable=cell-var-from-loop
+ lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_fix)
+
+ if not lint_clean:
+ print("ERROR: Code Style does not match coding style")
+ sys.exit(1)
+
+
+def lint_patch(linters, config_dict, file_name):
+ # type: (str, Dict[str, str], List[str]) -> None
+ """Lint patch command entry point."""
+ file_names = git.get_files_to_check_from_patch(file_name, is_interesting_file)
+
+ # Patch may have files that we do not want to check which is fine
+ if file_names:
+ _lint_files(linters, config_dict, file_names)
+
+
+def lint(linters, config_dict, file_names):
+ # type: (str, Dict[str, str], List[str]) -> None
+ """Lint files command entry point."""
+ all_file_names = git.get_files_to_check(file_names, is_interesting_file)
+
+ _lint_files(linters, config_dict, all_file_names)
+
+
+def lint_all(linters, config_dict, file_names):
+ # type: (str, Dict[str, str], List[str]) -> None
+ # pylint: disable=unused-argument
+ """Lint files command entry point based on working tree."""
+ all_file_names = git.get_files_to_check_working_tree(is_interesting_file)
+
+ _lint_files(linters, config_dict, all_file_names)
+
+
+def _fix_files(linters, config_dict, file_names):
+ # type: (str, Dict[str, str], List[str]) -> None
+ """Fix a list of files with linters if possible."""
+ linter_list = get_py_linter(linters)
+
+ # Get a list of linters which return a valid command for get_fix_cmd()
+ fix_list = [fixer for fixer in linter_list if fixer.get_fix_cmd_args("ignore")]
+
+ if len(fix_list) == 0:
+ raise ValueError("Cannot find any linters '%s' that support fixing." % (linters))
+
+ lint_runner = runner.LintRunner()
+
+ linter_instances = runner.find_linters(fix_list, config_dict)
+ if not linter_instances:
+ sys.exit(1)
+
+ for linter in linter_instances:
+ run_linter = lambda param1: lint_runner.run(linter.cmd_path + linter.linter.get_fix_cmd_args(param1)) # pylint: disable=cell-var-from-loop
+
+ lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_linter)
+
+ if not lint_clean:
+ print("ERROR: Code Style does not match coding style")
+ sys.exit(1)
+
+
+def fix_func(linters, config_dict, file_names):
+ # type: (str, Dict[str, str], List[str]) -> None
+ """Fix files command entry point."""
+ all_file_names = git.get_files_to_check(file_names, is_interesting_file)
+
+ _fix_files(linters, config_dict, all_file_names)
+
+
+def main():
+ # type: () -> None
+ """Main entry point."""
+
+ parser = argparse.ArgumentParser(description='PyLinter frontend.')
+
+ linters = get_py_linter(None)
+
+ dest_prefix = "linter_"
+ for linter1 in linters:
+ msg = 'Path to linter %s' % (linter1.cmd_name)
+ parser.add_argument(
+ '--' + linter1.cmd_name, type=str, help=msg, dest=dest_prefix + linter1.cmd_name)
+
+ parser.add_argument(
+ '--linters',
+ type=str,
+ help="Comma separated list of filters to use, defaults to 'all'",
+ default="all")
+
+ parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging")
+
+ sub = parser.add_subparsers(title="Linter subcommands", help="sub-command help")
+
+ parser_lint = sub.add_parser('lint', help='Lint only Git files')
+ parser_lint.add_argument("file_names", nargs="*", help="Globs of files to check")
+ parser_lint.set_defaults(func=lint)
+
+ parser_lint_all = sub.add_parser('lint-all', help='Lint All files')
+ parser_lint_all.add_argument("file_names", nargs="*", help="Globs of files to check")
+ parser_lint_all.set_defaults(func=lint_all)
+
+ parser_lint_patch = sub.add_parser('lint-patch', help='Lint the files in a patch')
+ parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
+ parser_lint_patch.set_defaults(func=lint_patch)
+
+ parser_fix = sub.add_parser('fix', help='Fix files if possible')
+ parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check")
+ parser_fix.set_defaults(func=fix_func)
+
+ args = parser.parse_args()
+
+ # Create a dictionary of linter locations if the user needs to override the location of a
+ # linter. This is common for mypy on Windows for instance.
+ config_dict = {}
+ for key in args.__dict__:
+ if key.startswith("linter_"):
+ name = key.replace(dest_prefix, "")
+ config_dict[name] = args.__dict__[key]
+
+ if args.verbose:
+ logging.basicConfig(level=logging.DEBUG)
+
+ args.func(args.linters, config_dict, args.file_names)
+
+
+if __name__ == "__main__":
+ main()