diff options
| -rw-r--r-- | SConstruct | 4 | ||||
| -rwxr-xr-x | buildscripts/clang_format.py | 28 | ||||
| -rwxr-xr-x | buildscripts/eslint.py | 37 | ||||
| -rw-r--r-- | buildscripts/linter/filediff.py | 56 | ||||
| -rwxr-xr-x | buildscripts/pylinters.py | 41 |
5 files changed, 117 insertions, 49 deletions
diff --git a/SConstruct b/SConstruct index aa9d050afe9..a50191bd63d 100644 --- a/SConstruct +++ b/SConstruct @@ -4221,7 +4221,7 @@ if get_option('lint-scope') == 'changed': "buildscripts/pylinters.py", patch_file, ], - action="$PYTHON ${SOURCES[0]} lint-patch ${SOURCES[1]}" + action="REVISION=$REVISION ENTERPRISE_REV=$ENTERPRISE_REV $PYTHON ${SOURCES[0]} lint-git-diff" ) clang_format = env.Command( @@ -4230,7 +4230,7 @@ if get_option('lint-scope') == 'changed': "buildscripts/clang_format.py", patch_file, ], - action="$PYTHON ${SOURCES[0]} lint-patch ${SOURCES[1]}" + action="REVISION=$REVISION ENTERPRISE_REV=$ENTERPRISE_REV $PYTHON ${SOURCES[0]} lint-git-diff" ) eslint = env.Command( diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py index 78662c4876c..0ec79106c31 100755 --- a/buildscripts/clang_format.py +++ b/buildscripts/clang_format.py @@ -10,7 +10,7 @@ import difflib import glob -from io import StringIO +import logging import os import re import shutil @@ -24,16 +24,17 @@ import urllib.error import urllib.parse import urllib.request -from distutils import spawn # pylint: disable=no-name-in-module from optparse import OptionParser -from multiprocessing import cpu_count +import structlog # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) -from buildscripts.linter import git # pylint: disable=wrong-import-position -from buildscripts.linter import parallel # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position +from buildscripts.linter.filediff import gather_changed_files_for_lint +from buildscripts.linter import git, parallel +# pylint: enable=wrong-import-position ############################################################################## # @@ -349,6 +350,18 @@ def lint_patch(clang_format, infile): _lint_files(clang_format, files) +def lint_git_diff(clang_format): + """ + Lint the files that have changes since the last git commit. + + :param clang_format: Path to clang_format command. + """ + files = gather_changed_files_for_lint(is_interesting_file) + + if files: + _lint_files(clang_format, files) + + def lint(clang_format): """Lint files command entry point.""" files = git.get_files_to_check([], is_interesting_file) @@ -548,6 +561,9 @@ def usage(): def main(): """Execute Main entry point.""" + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + structlog.configure(logger_factory=structlog.stdlib.LoggerFactory()) + parser = OptionParser() parser.add_option("-c", "--clang-format", type="string", dest="clang_format") @@ -562,6 +578,8 @@ def main(): lint_all(options.clang_format) elif command == "lint-patch": lint_patch(options.clang_format, args[2:]) + elif command == "lint-git-diff": + lint_git_diff(options.clang_format) elif command == "format": format_func(options.clang_format) elif command == "format-my": diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py index 220385a1e91..92a90b03ff3 100755 --- a/buildscripts/eslint.py +++ b/buildscripts/eslint.py @@ -19,23 +19,23 @@ import sys import tarfile import tempfile import threading -from typing import Optional, Dict, Tuple, List +from typing import Optional import urllib.error import urllib.parse import urllib.request from distutils import spawn # pylint: disable=no-name-in-module from optparse import OptionParser -from git import Repo import structlog # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) -from buildscripts.linter import git # pylint: disable=wrong-import-position -from buildscripts.linter import parallel # pylint: disable=wrong-import-position -from buildscripts.patch_builds.change_data import find_changed_files_in_repos # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position +from buildscripts.linter.filediff import gather_changed_files_for_lint +from buildscripts.linter import git, parallel +# pylint: enable=wrong-import-position ############################################################################## # @@ -228,26 +228,6 @@ def lint_patch(eslint, infile): return True -def get_revision_for_repo(path: str) -> str: - """ - Get the git revision for the given git repository. - - :param path: Path to git repository. - :return: Git revision to compare against for given repo. - """ - if "enterprise" in path: - return os.environ.get("ENTERPRISE_REV") - return os.environ.get("REVISION") - - -def get_repos_and_revisions() -> Tuple[List[Repo], Dict[str, str]]: - """Get the repo object and a map of revisions to compare against.""" - modules = git.get_module_paths() - repos = [Repo(path) for path in modules] - revision_map = {repo.git_dir: get_revision_for_repo(repo.git_dir) for repo in repos} - return repos, revision_map - - def lint_git_diff(eslint: Optional[str]) -> bool: """ Lint the files that have changes since the last git commit. @@ -255,12 +235,7 @@ def lint_git_diff(eslint: Optional[str]) -> bool: :param eslint: Path to eslint command. :return: True if lint was successful. """ - repos, revision_map = get_repos_and_revisions() - LOGGER.info("revisions", revision=revision_map) - candidate_files = find_changed_files_in_repos(repos, revision_map) - LOGGER.info("Found candidate_files", candidate_files=candidate_files) - files = [filename for filename in candidate_files if is_interesting_file(filename)] - LOGGER.info("Found files to lint", files=files) + files = gather_changed_files_for_lint(is_interesting_file) # Patch may have files that we do not want to check which is fine if files: diff --git a/buildscripts/linter/filediff.py b/buildscripts/linter/filediff.py new file mode 100644 index 00000000000..222e33df932 --- /dev/null +++ b/buildscripts/linter/filediff.py @@ -0,0 +1,56 @@ +"""Modules for find which files should be linted.""" +import os +import sys +from typing import Tuple, List, Dict, Callable + +from git import Repo +import structlog + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) + +# pylint: disable=wrong-import-position +from buildscripts.linter import git +from buildscripts.patch_builds.change_data import find_changed_files_in_repos +# pylint: enable=wrong-import-position + +LOGGER = structlog.get_logger(__name__) + + +def _get_revision_for_repo(path: str) -> str: + """ + Get the git revision for the given git repository. + + :param path: Path to git repository. + :return: Git revision to compare against for given repo. + """ + if "enterprise" in path: + return os.environ.get("ENTERPRISE_REV") + return os.environ.get("REVISION") + + +def _get_repos_and_revisions() -> Tuple[List[Repo], Dict[str, str]]: + """Get the repo object and a map of revisions to compare against.""" + modules = git.get_module_paths() + repos = [Repo(path) for path in modules] + revision_map = {repo.git_dir: _get_revision_for_repo(repo.git_dir) for repo in repos} + return repos, revision_map + + +def gather_changed_files_for_lint(is_interesting_file: Callable): + """ + Get the files that have changes since the last git commit. + + :param is_interesting_file: Filter for whether a file should be returned. + :return: List of files for linting. + """ + repos, revision_map = _get_repos_and_revisions() + LOGGER.info("revisions", revision=revision_map) + + candidate_files = find_changed_files_in_repos(repos, revision_map) + files = [filename for filename in candidate_files if is_interesting_file(filename)] + + LOGGER.info("Found files to lint", files=files) + + return files diff --git a/buildscripts/pylinters.py b/buildscripts/pylinters.py index 705c1c5f0e4..aab4378d80e 100755 --- a/buildscripts/pylinters.py +++ b/buildscripts/pylinters.py @@ -5,22 +5,25 @@ import argparse import logging import os import sys -import threading -from abc import ABCMeta, abstractmethod -from typing import Any, Dict, List +from typing import Dict, List + +import structlog # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) -from buildscripts.linter import base # pylint: disable=wrong-import-position -from buildscripts.linter import git # pylint: disable=wrong-import-position -from buildscripts.linter import mypy # pylint: disable=wrong-import-position -from buildscripts.linter import parallel # pylint: disable=wrong-import-position -from buildscripts.linter import pydocstyle # pylint: disable=wrong-import-position -from buildscripts.linter import pylint # pylint: disable=wrong-import-position -from buildscripts.linter import runner # pylint: disable=wrong-import-position -from buildscripts.linter import yapf # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position +from buildscripts.linter.filediff import gather_changed_files_for_lint +from buildscripts.linter import base +from buildscripts.linter import git +from buildscripts.linter import mypy +from buildscripts.linter import parallel +from buildscripts.linter import pydocstyle +from buildscripts.linter import pylint +from buildscripts.linter import runner +from buildscripts.linter import yapf +# pylint: enable=wrong-import-position # List of supported linters _LINTERS = [ @@ -98,6 +101,16 @@ def lint_patch(linters, config_dict, file_name): _lint_files(linters, config_dict, file_names) +def lint_git_diff(linters, config_dict, _): + # type: (str, Dict[str, str], List[str]) -> None + """Lint git diff command entry point.""" + file_names = gather_changed_files_for_lint(is_interesting_file) + + # Patch may have files that we do not want to check which is fine + if file_names: + _lint_files(linters, config_dict, file_names) + + def lint(linters, config_dict, file_names): # type: (str, Dict[str, str], List[str]) -> None """Lint files command entry point.""" @@ -185,6 +198,11 @@ def main(): parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check") parser_lint_patch.set_defaults(func=lint_patch) + parser_lint_patch = sub.add_parser('lint-git-diff', + help='Lint the files since the last git commit') + parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check") + parser_lint_patch.set_defaults(func=lint_git_diff) + parser_fix = sub.add_parser('fix', help='Fix files if possible') parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check") parser_fix.set_defaults(func=fix_func) @@ -201,6 +219,7 @@ def main(): if args.verbose: logging.basicConfig(level=logging.DEBUG) + structlog.configure(logger_factory=structlog.stdlib.LoggerFactory()) args.func(args.linters, config_dict, args.file_names) |