1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
|
#!/usr/bin/env python3
"""Extensible script to run one or more Python Linters across a subset of files in parallel."""
import argparse
import logging
import os
import sys
from typing import Dict, List
import structlog
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
# pylint: disable=wrong-import-position
from buildscripts.linter.filediff import gather_changed_files_for_lint
from buildscripts.linter import base
from buildscripts.linter import git
from buildscripts.linter import mypy
from buildscripts.linter import parallel
from buildscripts.linter import pydocstyle
from buildscripts.linter import pylint
from buildscripts.linter import runner
from buildscripts.linter import yapf
# pylint: enable=wrong-import-position
# List of supported linters
_LINTERS = [
yapf.YapfLinter(),
pylint.PyLintLinter(),
pydocstyle.PyDocstyleLinter(),
mypy.MypyLinter(),
]
def get_py_linter(linter_filter):
# type: (str) -> List[base.LinterBase]
"""
Get a list of linters to use.
'all' or None - select all linters
'a,b,c' - a comma delimited list is describes a list of linters to choose
"""
if linter_filter is None or linter_filter == "all":
return _LINTERS
linter_list = linter_filter.split(",")
linter_candidates = [linter for linter in _LINTERS if linter.cmd_name in linter_list]
if not linter_candidates:
raise ValueError("No linters found for filter '%s'" % (linter_filter))
return linter_candidates
def is_interesting_file(file_name):
# type: (str) -> bool
"""Return true if this file should be checked."""
file_blacklist = [] # type: List[str]
directory_blacklist = ["src/third_party"]
if file_name in file_blacklist or file_name.startswith(tuple(directory_blacklist)):
return False
directory_list = ["buildscripts", "pytests"]
return file_name.endswith(".py") and file_name.startswith(tuple(directory_list))
def _lint_files(linters, config_dict, file_names):
# type: (str, Dict[str, str], List[str]) -> None
"""Lint a list of files with clang-format."""
linter_list = get_py_linter(linters)
lint_runner = runner.LintRunner()
linter_instances = runner.find_linters(linter_list, config_dict)
if not linter_instances:
sys.exit(1)
failed_lint = False
for linter in linter_instances:
run_fix = lambda param1: lint_runner.run_lint(linter, param1) # pylint: disable=cell-var-from-loop
lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_fix)
if not lint_clean:
failed_lint = True
if failed_lint:
print("ERROR: Code Style does not match coding style")
sys.exit(1)
def lint_patch(linters, config_dict, file_name):
# type: (str, Dict[str, str], List[str]) -> None
"""Lint patch command entry point."""
file_names = git.get_files_to_check_from_patch(file_name, is_interesting_file)
# Patch may have files that we do not want to check which is fine
if file_names:
_lint_files(linters, config_dict, file_names)
def lint_git_diff(linters, config_dict, _):
# type: (str, Dict[str, str], List[str]) -> None
"""Lint git diff command entry point."""
file_names = gather_changed_files_for_lint(is_interesting_file)
# Patch may have files that we do not want to check which is fine
if file_names:
_lint_files(linters, config_dict, file_names)
def lint(linters, config_dict, file_names):
# type: (str, Dict[str, str], List[str]) -> None
"""Lint files command entry point."""
all_file_names = git.get_files_to_check(file_names, is_interesting_file)
_lint_files(linters, config_dict, all_file_names)
def lint_all(linters, config_dict, file_names):
# type: (str, Dict[str, str], List[str]) -> None
# pylint: disable=unused-argument
"""Lint files command entry point based on working tree."""
all_file_names = git.get_files_to_check_working_tree(is_interesting_file)
_lint_files(linters, config_dict, all_file_names)
def _fix_files(linters, config_dict, file_names):
# type: (str, Dict[str, str], List[str]) -> None
"""Fix a list of files with linters if possible."""
linter_list = get_py_linter(linters)
# Get a list of linters which return a valid command for get_fix_cmd()
fix_list = [fixer for fixer in linter_list if fixer.get_fix_cmd_args("ignore")]
if not fix_list:
raise ValueError("Cannot find any linters '%s' that support fixing." % (linters))
lint_runner = runner.LintRunner()
linter_instances = runner.find_linters(fix_list, config_dict)
if not linter_instances:
sys.exit(1)
for linter in linter_instances:
run_linter = lambda param1: lint_runner.run(linter.cmd_path + linter.linter. # pylint: disable=cell-var-from-loop
get_fix_cmd_args(param1)) # pylint: disable=cell-var-from-loop
lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_linter)
if not lint_clean:
print("ERROR: Code Style does not match coding style")
sys.exit(1)
def fix_func(linters, config_dict, file_names):
# type: (str, Dict[str, str], List[str]) -> None
"""Fix files command entry point."""
all_file_names = git.get_files_to_check(file_names, is_interesting_file)
_fix_files(linters, config_dict, all_file_names)
def main():
# type: () -> None
"""Execute Main entry point."""
parser = argparse.ArgumentParser(description='PyLinter frontend.')
linters = get_py_linter(None)
dest_prefix = "linter_"
for linter1 in linters:
msg = 'Path to linter %s' % (linter1.cmd_name)
parser.add_argument('--' + linter1.cmd_name, type=str, help=msg,
dest=dest_prefix + linter1.cmd_name)
parser.add_argument('--linters', type=str,
help="Comma separated list of filters to use, defaults to 'all'",
default="all")
parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging")
sub = parser.add_subparsers(title="Linter subcommands", help="sub-command help")
parser_lint = sub.add_parser('lint', help='Lint only Git files')
parser_lint.add_argument("file_names", nargs="*", help="Globs of files to check")
parser_lint.set_defaults(func=lint)
parser_lint_all = sub.add_parser('lint-all', help='Lint All files')
parser_lint_all.add_argument("file_names", nargs="*", help="Globs of files to check")
parser_lint_all.set_defaults(func=lint_all)
parser_lint_patch = sub.add_parser('lint-patch', help='Lint the files in a patch')
parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
parser_lint_patch.set_defaults(func=lint_patch)
parser_lint_patch = sub.add_parser('lint-git-diff',
help='Lint the files since the last git commit')
parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
parser_lint_patch.set_defaults(func=lint_git_diff)
parser_fix = sub.add_parser('fix', help='Fix files if possible')
parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check")
parser_fix.set_defaults(func=fix_func)
args = parser.parse_args()
# Create a dictionary of linter locations if the user needs to override the location of a
# linter. This is common for mypy on Windows for instance.
config_dict = {}
for key in args.__dict__:
if key.startswith("linter_"):
name = key.replace(dest_prefix, "")
config_dict[name] = args.__dict__[key]
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
args.func(args.linters, config_dict, args.file_names)
if __name__ == "__main__":
main()
|
