SERVER-47796: Ensure linters run against enterprise modules

5 files changed, 117 insertions, 49 deletions

diff --git a/SConstruct b/SConstruct
index aa9d050afe9..a50191bd63d 100644
--- a/SConstruct
+++ b/SConstruct
@@ -4221,7 +4221,7 @@ if get_option('lint-scope') == 'changed':
             "buildscripts/pylinters.py",
             patch_file,
         ],
-        action="$PYTHON ${SOURCES[0]} lint-patch ${SOURCES[1]}"
+        action="REVISION=$REVISION ENTERPRISE_REV=$ENTERPRISE_REV $PYTHON ${SOURCES[0]} lint-git-diff"
     )
 
     clang_format = env.Command(
@@ -4230,7 +4230,7 @@ if get_option('lint-scope') == 'changed':
             "buildscripts/clang_format.py",
             patch_file,
         ],
-        action="$PYTHON ${SOURCES[0]} lint-patch ${SOURCES[1]}"
+        action="REVISION=$REVISION ENTERPRISE_REV=$ENTERPRISE_REV $PYTHON ${SOURCES[0]} lint-git-diff"
     )
 
     eslint = env.Command(
diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py
index 78662c4876c..0ec79106c31 100755
--- a/buildscripts/clang_format.py
+++ b/buildscripts/clang_format.py
@@ -10,7 +10,7 @@
 
 import difflib
 import glob
-from io import StringIO
+import logging
 import os
 import re
 import shutil
@@ -24,16 +24,17 @@ import urllib.error
 import urllib.parse
 import urllib.request
 
-from distutils import spawn  # pylint: disable=no-name-in-module
 from optparse import OptionParser
-from multiprocessing import cpu_count
+import structlog
 
 # Get relative imports to work when the package is not installed on the PYTHONPATH.
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
 
-from buildscripts.linter import git  # pylint: disable=wrong-import-position
-from buildscripts.linter import parallel  # pylint: disable=wrong-import-position
+# pylint: disable=wrong-import-position
+from buildscripts.linter.filediff import gather_changed_files_for_lint
+from buildscripts.linter import git, parallel
+# pylint: enable=wrong-import-position
 
 ##############################################################################
 #
@@ -349,6 +350,18 @@ def lint_patch(clang_format, infile):
         _lint_files(clang_format, files)
 
 
+def lint_git_diff(clang_format):
+    """
+    Lint the files that have changes since the last git commit.
+
+    :param clang_format: Path to clang_format command.
+    """
+    files = gather_changed_files_for_lint(is_interesting_file)
+
+    if files:
+        _lint_files(clang_format, files)
+
+
 def lint(clang_format):
     """Lint files command entry point."""
     files = git.get_files_to_check([], is_interesting_file)
@@ -548,6 +561,9 @@ def usage():
 
 def main():
     """Execute Main entry point."""
+    logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+    structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
+
     parser = OptionParser()
     parser.add_option("-c", "--clang-format", type="string", dest="clang_format")
 
@@ -562,6 +578,8 @@ def main():
             lint_all(options.clang_format)
         elif command == "lint-patch":
             lint_patch(options.clang_format, args[2:])
+        elif command == "lint-git-diff":
+            lint_git_diff(options.clang_format)
         elif command == "format":
             format_func(options.clang_format)
         elif command == "format-my":
diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py
index 220385a1e91..92a90b03ff3 100755
--- a/buildscripts/eslint.py
+++ b/buildscripts/eslint.py
@@ -19,23 +19,23 @@ import sys
 import tarfile
 import tempfile
 import threading
-from typing import Optional, Dict, Tuple, List
+from typing import Optional
 import urllib.error
 import urllib.parse
 import urllib.request
 
 from distutils import spawn  # pylint: disable=no-name-in-module
 from optparse import OptionParser
-from git import Repo
 import structlog
 
 # Get relative imports to work when the package is not installed on the PYTHONPATH.
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
 
-from buildscripts.linter import git  # pylint: disable=wrong-import-position
-from buildscripts.linter import parallel  # pylint: disable=wrong-import-position
-from buildscripts.patch_builds.change_data import find_changed_files_in_repos  # pylint: disable=wrong-import-position
+# pylint: disable=wrong-import-position
+from buildscripts.linter.filediff import gather_changed_files_for_lint
+from buildscripts.linter import git, parallel
+# pylint: enable=wrong-import-position
 
 ##############################################################################
 #
@@ -228,26 +228,6 @@ def lint_patch(eslint, infile):
     return True
 
 
-def get_revision_for_repo(path: str) -> str:
-    """
-    Get the git revision for the given git repository.
-
-    :param path: Path to git repository.
-    :return: Git revision to compare against for given repo.
-    """
-    if "enterprise" in path:
-        return os.environ.get("ENTERPRISE_REV")
-    return os.environ.get("REVISION")
-
-
-def get_repos_and_revisions() -> Tuple[List[Repo], Dict[str, str]]:
-    """Get the repo object and a map of revisions to compare against."""
-    modules = git.get_module_paths()
-    repos = [Repo(path) for path in modules]
-    revision_map = {repo.git_dir: get_revision_for_repo(repo.git_dir) for repo in repos}
-    return repos, revision_map
-
-
 def lint_git_diff(eslint: Optional[str]) -> bool:
     """
     Lint the files that have changes since the last git commit.
@@ -255,12 +235,7 @@ def lint_git_diff(eslint: Optional[str]) -> bool:
     :param eslint: Path to eslint command.
     :return: True if lint was successful.
     """
-    repos, revision_map = get_repos_and_revisions()
-    LOGGER.info("revisions", revision=revision_map)
-    candidate_files = find_changed_files_in_repos(repos, revision_map)
-    LOGGER.info("Found candidate_files", candidate_files=candidate_files)
-    files = [filename for filename in candidate_files if is_interesting_file(filename)]
-    LOGGER.info("Found files to lint", files=files)
+    files = gather_changed_files_for_lint(is_interesting_file)
 
     # Patch may have files that we do not want to check which is fine
     if files:
diff --git a/buildscripts/linter/filediff.py b/buildscripts/linter/filediff.py
new file mode 100644
index 00000000000..222e33df932
--- /dev/null
+++ b/buildscripts/linter/filediff.py
@@ -0,0 +1,56 @@
+"""Modules for find which files should be linted."""
+import os
+import sys
+from typing import Tuple, List, Dict, Callable
+
+from git import Repo
+import structlog
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
+
+# pylint: disable=wrong-import-position
+from buildscripts.linter import git
+from buildscripts.patch_builds.change_data import find_changed_files_in_repos
+# pylint: enable=wrong-import-position
+
+LOGGER = structlog.get_logger(__name__)
+
+
+def _get_revision_for_repo(path: str) -> str:
+    """
+    Get the git revision for the given git repository.
+
+    :param path: Path to git repository.
+    :return: Git revision to compare against for given repo.
+    """
+    if "enterprise" in path:
+        return os.environ.get("ENTERPRISE_REV")
+    return os.environ.get("REVISION")
+
+
+def _get_repos_and_revisions() -> Tuple[List[Repo], Dict[str, str]]:
+    """Get the repo object and a map of revisions to compare against."""
+    modules = git.get_module_paths()
+    repos = [Repo(path) for path in modules]
+    revision_map = {repo.git_dir: _get_revision_for_repo(repo.git_dir) for repo in repos}
+    return repos, revision_map
+
+
+def gather_changed_files_for_lint(is_interesting_file: Callable):
+    """
+    Get the files that have changes since the last git commit.
+
+    :param is_interesting_file: Filter for whether a file should be returned.
+    :return: List of files for linting.
+    """
+    repos, revision_map = _get_repos_and_revisions()
+    LOGGER.info("revisions", revision=revision_map)
+
+    candidate_files = find_changed_files_in_repos(repos, revision_map)
+    files = [filename for filename in candidate_files if is_interesting_file(filename)]
+
+    LOGGER.info("Found files to lint", files=files)
+
+    return files
diff --git a/buildscripts/pylinters.py b/buildscripts/pylinters.py
index 705c1c5f0e4..aab4378d80e 100755
--- a/buildscripts/pylinters.py
+++ b/buildscripts/pylinters.py
@@ -5,22 +5,25 @@ import argparse
 import logging
 import os
 import sys
-import threading
-from abc import ABCMeta, abstractmethod
-from typing import Any, Dict, List
+from typing import Dict, List
+
+import structlog
 
 # Get relative imports to work when the package is not installed on the PYTHONPATH.
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
 
-from buildscripts.linter import base  # pylint: disable=wrong-import-position
-from buildscripts.linter import git  # pylint: disable=wrong-import-position
-from buildscripts.linter import mypy  # pylint: disable=wrong-import-position
-from buildscripts.linter import parallel  # pylint: disable=wrong-import-position
-from buildscripts.linter import pydocstyle  # pylint: disable=wrong-import-position
-from buildscripts.linter import pylint  # pylint: disable=wrong-import-position
-from buildscripts.linter import runner  # pylint: disable=wrong-import-position
-from buildscripts.linter import yapf  # pylint: disable=wrong-import-position
+# pylint: disable=wrong-import-position
+from buildscripts.linter.filediff import gather_changed_files_for_lint
+from buildscripts.linter import base
+from buildscripts.linter import git
+from buildscripts.linter import mypy
+from buildscripts.linter import parallel
+from buildscripts.linter import pydocstyle
+from buildscripts.linter import pylint
+from buildscripts.linter import runner
+from buildscripts.linter import yapf
+# pylint: enable=wrong-import-position
 
 # List of supported linters
 _LINTERS = [
@@ -98,6 +101,16 @@ def lint_patch(linters, config_dict, file_name):
         _lint_files(linters, config_dict, file_names)
 
 
+def lint_git_diff(linters, config_dict, _):
+    # type: (str, Dict[str, str], List[str]) -> None
+    """Lint git diff command entry point."""
+    file_names = gather_changed_files_for_lint(is_interesting_file)
+
+    # Patch may have files that we do not want to check which is fine
+    if file_names:
+        _lint_files(linters, config_dict, file_names)
+
+
 def lint(linters, config_dict, file_names):
     # type: (str, Dict[str, str], List[str]) -> None
     """Lint files command entry point."""
@@ -185,6 +198,11 @@ def main():
     parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
     parser_lint_patch.set_defaults(func=lint_patch)
 
+    parser_lint_patch = sub.add_parser('lint-git-diff',
+                                       help='Lint the files since the last git commit')
+    parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
+    parser_lint_patch.set_defaults(func=lint_git_diff)
+
     parser_fix = sub.add_parser('fix', help='Fix files if possible')
     parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check")
     parser_fix.set_defaults(func=fix_func)
@@ -201,6 +219,7 @@ def main():
 
     if args.verbose:
         logging.basicConfig(level=logging.DEBUG)
+    structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
 
     args.func(args.linters, config_dict, args.file_names)