summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--SConstruct4
-rwxr-xr-xbuildscripts/clang_format.py28
-rwxr-xr-xbuildscripts/eslint.py37
-rw-r--r--buildscripts/linter/filediff.py56
-rwxr-xr-xbuildscripts/pylinters.py41
5 files changed, 117 insertions, 49 deletions
diff --git a/SConstruct b/SConstruct
index aa9d050afe9..a50191bd63d 100644
--- a/SConstruct
+++ b/SConstruct
@@ -4221,7 +4221,7 @@ if get_option('lint-scope') == 'changed':
"buildscripts/pylinters.py",
patch_file,
],
- action="$PYTHON ${SOURCES[0]} lint-patch ${SOURCES[1]}"
+ action="REVISION=$REVISION ENTERPRISE_REV=$ENTERPRISE_REV $PYTHON ${SOURCES[0]} lint-git-diff"
)
clang_format = env.Command(
@@ -4230,7 +4230,7 @@ if get_option('lint-scope') == 'changed':
"buildscripts/clang_format.py",
patch_file,
],
- action="$PYTHON ${SOURCES[0]} lint-patch ${SOURCES[1]}"
+ action="REVISION=$REVISION ENTERPRISE_REV=$ENTERPRISE_REV $PYTHON ${SOURCES[0]} lint-git-diff"
)
eslint = env.Command(
diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py
index 78662c4876c..0ec79106c31 100755
--- a/buildscripts/clang_format.py
+++ b/buildscripts/clang_format.py
@@ -10,7 +10,7 @@
import difflib
import glob
-from io import StringIO
+import logging
import os
import re
import shutil
@@ -24,16 +24,17 @@ import urllib.error
import urllib.parse
import urllib.request
-from distutils import spawn # pylint: disable=no-name-in-module
from optparse import OptionParser
-from multiprocessing import cpu_count
+import structlog
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
-from buildscripts.linter import git # pylint: disable=wrong-import-position
-from buildscripts.linter import parallel # pylint: disable=wrong-import-position
+# pylint: disable=wrong-import-position
+from buildscripts.linter.filediff import gather_changed_files_for_lint
+from buildscripts.linter import git, parallel
+# pylint: enable=wrong-import-position
##############################################################################
#
@@ -349,6 +350,18 @@ def lint_patch(clang_format, infile):
_lint_files(clang_format, files)
+def lint_git_diff(clang_format):
+ """
+ Lint the files that have changes since the last git commit.
+
+ :param clang_format: Path to clang_format command.
+ """
+ files = gather_changed_files_for_lint(is_interesting_file)
+
+ if files:
+ _lint_files(clang_format, files)
+
+
def lint(clang_format):
"""Lint files command entry point."""
files = git.get_files_to_check([], is_interesting_file)
@@ -548,6 +561,9 @@ def usage():
def main():
"""Execute Main entry point."""
+ logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+ structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
+
parser = OptionParser()
parser.add_option("-c", "--clang-format", type="string", dest="clang_format")
@@ -562,6 +578,8 @@ def main():
lint_all(options.clang_format)
elif command == "lint-patch":
lint_patch(options.clang_format, args[2:])
+ elif command == "lint-git-diff":
+ lint_git_diff(options.clang_format)
elif command == "format":
format_func(options.clang_format)
elif command == "format-my":
diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py
index 220385a1e91..92a90b03ff3 100755
--- a/buildscripts/eslint.py
+++ b/buildscripts/eslint.py
@@ -19,23 +19,23 @@ import sys
import tarfile
import tempfile
import threading
-from typing import Optional, Dict, Tuple, List
+from typing import Optional
import urllib.error
import urllib.parse
import urllib.request
from distutils import spawn # pylint: disable=no-name-in-module
from optparse import OptionParser
-from git import Repo
import structlog
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
-from buildscripts.linter import git # pylint: disable=wrong-import-position
-from buildscripts.linter import parallel # pylint: disable=wrong-import-position
-from buildscripts.patch_builds.change_data import find_changed_files_in_repos # pylint: disable=wrong-import-position
+# pylint: disable=wrong-import-position
+from buildscripts.linter.filediff import gather_changed_files_for_lint
+from buildscripts.linter import git, parallel
+# pylint: enable=wrong-import-position
##############################################################################
#
@@ -228,26 +228,6 @@ def lint_patch(eslint, infile):
return True
-def get_revision_for_repo(path: str) -> str:
- """
- Get the git revision for the given git repository.
-
- :param path: Path to git repository.
- :return: Git revision to compare against for given repo.
- """
- if "enterprise" in path:
- return os.environ.get("ENTERPRISE_REV")
- return os.environ.get("REVISION")
-
-
-def get_repos_and_revisions() -> Tuple[List[Repo], Dict[str, str]]:
- """Get the repo object and a map of revisions to compare against."""
- modules = git.get_module_paths()
- repos = [Repo(path) for path in modules]
- revision_map = {repo.git_dir: get_revision_for_repo(repo.git_dir) for repo in repos}
- return repos, revision_map
-
-
def lint_git_diff(eslint: Optional[str]) -> bool:
"""
Lint the files that have changes since the last git commit.
@@ -255,12 +235,7 @@ def lint_git_diff(eslint: Optional[str]) -> bool:
:param eslint: Path to eslint command.
:return: True if lint was successful.
"""
- repos, revision_map = get_repos_and_revisions()
- LOGGER.info("revisions", revision=revision_map)
- candidate_files = find_changed_files_in_repos(repos, revision_map)
- LOGGER.info("Found candidate_files", candidate_files=candidate_files)
- files = [filename for filename in candidate_files if is_interesting_file(filename)]
- LOGGER.info("Found files to lint", files=files)
+ files = gather_changed_files_for_lint(is_interesting_file)
# Patch may have files that we do not want to check which is fine
if files:
diff --git a/buildscripts/linter/filediff.py b/buildscripts/linter/filediff.py
new file mode 100644
index 00000000000..222e33df932
--- /dev/null
+++ b/buildscripts/linter/filediff.py
@@ -0,0 +1,56 @@
+"""Modules for find which files should be linted."""
+import os
+import sys
+from typing import Tuple, List, Dict, Callable
+
+from git import Repo
+import structlog
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
+
+# pylint: disable=wrong-import-position
+from buildscripts.linter import git
+from buildscripts.patch_builds.change_data import find_changed_files_in_repos
+# pylint: enable=wrong-import-position
+
+LOGGER = structlog.get_logger(__name__)
+
+
+def _get_revision_for_repo(path: str) -> str:
+ """
+ Get the git revision for the given git repository.
+
+ :param path: Path to git repository.
+ :return: Git revision to compare against for given repo.
+ """
+ if "enterprise" in path:
+ return os.environ.get("ENTERPRISE_REV")
+ return os.environ.get("REVISION")
+
+
+def _get_repos_and_revisions() -> Tuple[List[Repo], Dict[str, str]]:
+ """Get the repo object and a map of revisions to compare against."""
+ modules = git.get_module_paths()
+ repos = [Repo(path) for path in modules]
+ revision_map = {repo.git_dir: _get_revision_for_repo(repo.git_dir) for repo in repos}
+ return repos, revision_map
+
+
+def gather_changed_files_for_lint(is_interesting_file: Callable):
+ """
+ Get the files that have changes since the last git commit.
+
+ :param is_interesting_file: Filter for whether a file should be returned.
+ :return: List of files for linting.
+ """
+ repos, revision_map = _get_repos_and_revisions()
+ LOGGER.info("revisions", revision=revision_map)
+
+ candidate_files = find_changed_files_in_repos(repos, revision_map)
+ files = [filename for filename in candidate_files if is_interesting_file(filename)]
+
+ LOGGER.info("Found files to lint", files=files)
+
+ return files
diff --git a/buildscripts/pylinters.py b/buildscripts/pylinters.py
index 705c1c5f0e4..aab4378d80e 100755
--- a/buildscripts/pylinters.py
+++ b/buildscripts/pylinters.py
@@ -5,22 +5,25 @@ import argparse
import logging
import os
import sys
-import threading
-from abc import ABCMeta, abstractmethod
-from typing import Any, Dict, List
+from typing import Dict, List
+
+import structlog
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
-from buildscripts.linter import base # pylint: disable=wrong-import-position
-from buildscripts.linter import git # pylint: disable=wrong-import-position
-from buildscripts.linter import mypy # pylint: disable=wrong-import-position
-from buildscripts.linter import parallel # pylint: disable=wrong-import-position
-from buildscripts.linter import pydocstyle # pylint: disable=wrong-import-position
-from buildscripts.linter import pylint # pylint: disable=wrong-import-position
-from buildscripts.linter import runner # pylint: disable=wrong-import-position
-from buildscripts.linter import yapf # pylint: disable=wrong-import-position
+# pylint: disable=wrong-import-position
+from buildscripts.linter.filediff import gather_changed_files_for_lint
+from buildscripts.linter import base
+from buildscripts.linter import git
+from buildscripts.linter import mypy
+from buildscripts.linter import parallel
+from buildscripts.linter import pydocstyle
+from buildscripts.linter import pylint
+from buildscripts.linter import runner
+from buildscripts.linter import yapf
+# pylint: enable=wrong-import-position
# List of supported linters
_LINTERS = [
@@ -98,6 +101,16 @@ def lint_patch(linters, config_dict, file_name):
_lint_files(linters, config_dict, file_names)
+def lint_git_diff(linters, config_dict, _):
+ # type: (str, Dict[str, str], List[str]) -> None
+ """Lint git diff command entry point."""
+ file_names = gather_changed_files_for_lint(is_interesting_file)
+
+ # Patch may have files that we do not want to check which is fine
+ if file_names:
+ _lint_files(linters, config_dict, file_names)
+
+
def lint(linters, config_dict, file_names):
# type: (str, Dict[str, str], List[str]) -> None
"""Lint files command entry point."""
@@ -185,6 +198,11 @@ def main():
parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
parser_lint_patch.set_defaults(func=lint_patch)
+ parser_lint_patch = sub.add_parser('lint-git-diff',
+ help='Lint the files since the last git commit')
+ parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
+ parser_lint_patch.set_defaults(func=lint_git_diff)
+
parser_fix = sub.add_parser('fix', help='Fix files if possible')
parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check")
parser_fix.set_defaults(func=fix_func)
@@ -201,6 +219,7 @@ def main():
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
+ structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
args.func(args.linters, config_dict, args.file_names)