SERVER-28308 Integrate python linting for IDL into Evergreen

1 files changed, 121 insertions, 82 deletions

diff --git a/buildscripts/linter/git.py b/buildscripts/linter/git.py
index c8f95149750..45b54bbd960 100644
--- a/buildscripts/linter/git.py
+++ b/buildscripts/linter/git.py
@@ -1,3 +1,4 @@
+"""Git Utility functions."""
 from __future__ import absolute_import
 from __future__ import print_function
 
@@ -5,6 +6,7 @@ import itertools
 import os
 import re
 import subprocess
+from typing import Any, Callable, List, Tuple
 
 from buildscripts import moduleconfig
 from buildscripts.resmokelib.utils import globstar
@@ -13,26 +15,31 @@ from buildscripts.resmokelib.utils import globstar
 # Has to match the string in SConstruct
 MODULE_DIR = "src/mongo/db/modules"
 
+
 def get_base_dir():
-    """Get the base directory for mongo repo.
-        This script assumes that it is running in buildscripts/, and uses
-        that to find the base directory.
+    # type: () -> str
+    """
+    Get the base directory for mongo repo.
+
+    This script assumes that it is running in buildscripts/, and uses
+    that to find the base directory.
     """
     try:
         return subprocess.check_output(['git', 'rev-parse', '--show-toplevel']).rstrip()
-    except:
+    except subprocess.CalledProcessError:
         # We are not in a valid git directory. Use the script path instead.
         return os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
+
 def get_repos():
-    """Get a list of Repos to check clang-format for
-    """
+    # type: () -> List[Repo]
+    """Get a list of Repos to check linters for."""
     base_dir = get_base_dir()
 
     # Get a list of modules
     # TODO: how do we filter rocks, does it matter?
     mongo_modules = moduleconfig.discover_module_directories(
-                        os.path.join(base_dir, MODULE_DIR), None)
+        os.path.join(base_dir, MODULE_DIR), None)
 
     paths = [os.path.join(base_dir, MODULE_DIR, m) for m in mongo_modules]
 
@@ -40,41 +47,50 @@ def get_repos():
 
     return [Repo(p) for p in paths]
 
+
 class Repo(object):
-    """Class encapsulates all knowledge about a git repository, and its metadata
-        to run clang-format.
-    """
+    """Class encapsulates all knowledge about a git repository, and its metadata to run linters."""
+
     def __init__(self, path):
+        # type: (str) -> None
+        """Construct a repo object."""
         self.path = path
 
     def _callgito(self, args):
-        """Call git for this repository, and return the captured output
-        """
+        # type: (List[str]) -> str
+        """Call git for this repository, and return the captured output."""
         # These two flags are the equivalent of -C in newer versions of Git
         # but we use these to support versions pre 1.8.5 but it depends on the command
         # and what the current directory is
         if "ls-files" in args:
             # This command depends on the current directory and works better if not run with
             # work-tree
-            return subprocess.check_output(['git', '--git-dir', os.path.join(self.path, ".git")]
-                                            + args)
+            return subprocess.check_output(['git', '--git-dir', os.path.join(self.path, ".git")] +
+                                           args)
         else:
-            return subprocess.check_output(['git', '--git-dir', os.path.join(self.path, ".git"),
-                                '--work-tree', self.path] + args)
+            return subprocess.check_output([
+                'git', '--git-dir', os.path.join(self.path, ".git"), '--work-tree', self.path
+            ] + args)
 
     def _callgit(self, args):
-        """Call git for this repository without capturing output
+        # type: (List[str]) -> int
+        """
+        Call git for this repository without capturing output.
+
         This is designed to be used when git returns non-zero exit codes.
         """
         # These two flags are the equivalent of -C in newer versions of Git
         # but we use these to support versions pre 1.8.5 but it depends on the command
         # and what the current directory is
-        return subprocess.call(['git', '--git-dir', os.path.join(self.path, ".git"),
-                                '--work-tree', self.path] + args)
+        return subprocess.call([
+            'git',
+            '--git-dir',
+            os.path.join(self.path, ".git"),
+        ] + args)
 
     def _get_local_dir(self, path):
-        """Get a directory path relative to the git root directory
-        """
+        # type: (str) -> str
+        """Get a directory path relative to the git root directory."""
         if os.path.isabs(path):
             path = os.path.relpath(path, self.path)
 
@@ -84,13 +100,16 @@ class Repo(object):
         return path
 
     def get_candidates(self, candidates, filter_function):
-        """Get the set of candidate files to check by querying the repository
+        # type: (List[str], Callable[[str], bool]) -> List[str]
+        """
+        Get the set of candidate files to check by querying the repository.
 
         Returns the full path to the file for clang-format to consume.
         """
         if candidates is not None and len(candidates) > 0:
             candidates = [self._get_local_dir(f) for f in candidates]
-            valid_files = list(set(candidates).intersection(self.get_candidate_files(filter_function)))
+            valid_files = list(
+                set(candidates).intersection(self.get_candidate_files(filter_function)))
         else:
             valid_files = list(self.get_candidate_files(filter_function))
 
@@ -100,29 +119,31 @@ class Repo(object):
         return valid_files
 
     def _git_ls_files(self, cmd, filter_function):
-        """Run git-ls-files and filter the list of files to a valid candidate list
-        """
+        # type: (List[str], Callable[[str], bool]) -> List[str]
+        """Run git-ls-files and filter the list of files to a valid candidate list."""
         gito = self._callgito(cmd)
 
         # This allows us to pick all the interesting files
         # in the mongo and mongo-enterprise repos
-        file_list = [line.rstrip()
-                for line in gito.splitlines() if filter_function(line.rstrip())]
+        file_list = [line.rstrip() for line in gito.splitlines() if filter_function(line.rstrip())]
 
         return file_list
 
     def get_candidate_files(self, filter_function):
-        """Query git to get a list of all files in the repo to consider for analysis
-        """
+        # type: (Callable[[str], bool]) -> List[str]
+        """Query git to get a list of all files in the repo to consider for analysis."""
         return self._git_ls_files(["ls-files", "--cached"], filter_function)
 
     def get_working_tree_candidate_files(self, filter_function):
-        """Query git to get a list of all files in the working tree to consider for analysis
-        """
+        # type: (Callable[[str], bool]) -> List[str]
+        # pylint: disable=invalid-name
+        """Query git to get a list of all files in the working tree to consider for analysis."""
         return self._git_ls_files(["ls-files", "--cached", "--others"], filter_function)
 
     def get_working_tree_candidates(self, filter_function):
-        """Get the set of candidate files to check by querying the repository
+        # type: (Callable[[str], bool]) -> List[str]
+        """
+        Get the set of candidate files to check by querying the repository.
 
         Returns the full path to the file for clang-format to consume.
         """
@@ -137,44 +158,47 @@ class Repo(object):
         return valid_files
 
     def is_detached(self):
-        """Is the current working tree in a detached HEAD state?
-        """
+        # type: () -> bool
+        """Return true if the current working tree in a detached HEAD state."""
         # symbolic-ref returns 1 if the repo is in a detached HEAD state
-        return self._callgit(["symbolic-ref", "--quiet", "HEAD"])
+        return self._callgit(["symbolic-ref", "--quiet", "HEAD"]) == 1
 
     def is_ancestor(self, parent, child):
-        """Is the specified parent hash an ancestor of child hash?
-        """
+        # type: (str, str) -> bool
+        """Return true if the specified parent hash an ancestor of child hash."""
         # merge base returns 0 if parent is an ancestor of child
         return not self._callgit(["merge-base", "--is-ancestor", parent, child])
 
     def is_commit(self, sha1):
-        """Is the specified hash a valid git commit?
-        """
+        # type: (str) -> bool
+        """Return true if the specified hash is a valid git commit."""
         # cat-file -e returns 0 if it is a valid hash
         return not self._callgit(["cat-file", "-e", "%s^{commit}" % sha1])
 
     def is_working_tree_dirty(self):
-        """Does the current working tree have changes?
-        """
+        # type: () -> bool
+        """Return true the current working tree have changes."""
         # diff returns 1 if the working tree has local changes
-        return self._callgit(["diff", "--quiet"])
+        return self._callgit(["diff", "--quiet"]) == 1
 
     def does_branch_exist(self, branch):
-        """Does the branch exist?
-        """
+        # type: (str) -> bool
+        """Return true if the branch exists."""
         # rev-parse returns 0 if the branch exists
         return not self._callgit(["rev-parse", "--verify", branch])
 
     def get_merge_base(self, commit):
-        """Get the merge base between 'commit' and HEAD
-        """
+        # type: (str) -> str
+        """Get the merge base between 'commit' and HEAD."""
         return self._callgito(["merge-base", "HEAD", commit]).rstrip()
 
     def get_branch_name(self):
-        """Get the current branch name, short form
-           This returns "master", not "refs/head/master"
-           Will not work if the current branch is detached
+        # type: () -> str
+        """
+        Get the current branch name, short form.
+
+        This returns "master", not "refs/head/master".
+        Will not work if the current branch is detached.
         """
         branch = self.rev_parse(["--abbrev-ref", "HEAD"])
         if branch == "HEAD":
@@ -183,87 +207,100 @@ class Repo(object):
         return branch
 
     def add(self, command):
-        """git add wrapper
-        """
+        # type: (List[str]) -> str
+        """Git add wrapper."""
         return self._callgito(["add"] + command)
 
     def checkout(self, command):
-        """git checkout wrapper
-        """
+        # type: (List[str]) -> str
+        """Git checkout wrapper."""
         return self._callgito(["checkout"] + command)
 
     def commit(self, command):
-        """git commit wrapper
-        """
+        # type: (List[str]) -> str
+        """Git commit wrapper."""
         return self._callgito(["commit"] + command)
 
     def diff(self, command):
-        """git diff wrapper
-        """
+        # type: (List[str]) -> str
+        """Git diff wrapper."""
         return self._callgito(["diff"] + command)
 
     def log(self, command):
-        """git log wrapper
-        """
+        # type: (List[str]) -> str
+        """Git log wrapper."""
         return self._callgito(["log"] + command)
 
     def rev_parse(self, command):
-        """git rev-parse wrapper
-        """
+        # type: (List[str]) -> str
+        """Git rev-parse wrapper."""
         return self._callgito(["rev-parse"] + command).rstrip()
 
     def rm(self, command):
-        """git rm wrapper
-        """
+        # type: (List[str]) -> str
+        # pylint: disable=invalid-name
+        """Git rm wrapper."""
         return self._callgito(["rm"] + command)
 
     def show(self, command):
-        """git show wrapper
-        """
+        # type: (List[str]) -> str
+        """Git show wrapper."""
         return self._callgito(["show"] + command)
 
 
 def expand_file_string(glob_pattern):
-    """Expand a string that represents a set of files
-    """
+    # type: (str) -> List[str]
+    """Expand a string that represents a set of files."""
     return [os.path.abspath(f) for f in globstar.iglob(glob_pattern)]
 
+
 def get_files_to_check_working_tree(filter_function):
-    """Get a list of files to check from the working tree.
-       This will pick up files not managed by git.
+    # type: (Callable[[str], bool]) -> List[str]
+    """
+    Get a list of files to check from the working tree.
+
+    This will pick up files not managed by git.
     """
     repos = get_repos()
 
-    valid_files = list(itertools.chain.from_iterable([r.get_working_tree_candidates(filter_function) for r in repos]))
+    valid_files = list(
+        itertools.chain.from_iterable(
+            [r.get_working_tree_candidates(filter_function) for r in repos]))
 
     return valid_files
 
+
 def get_files_to_check(files, filter_function):
-    """Get a list of files that need to be checked
-       based on which files are managed by git.
-    """
+    # type: (List[str], Callable[[str], bool]) -> List[str]
+    """Get a list of files that need to be checked based on which files are managed by git."""
     # Get a list of candidate_files
-    candidates = [expand_file_string(f) for f in files]
-    candidates = list(itertools.chain.from_iterable(candidates))
+    candidates_nested = [expand_file_string(f) for f in files]
+    candidates = list(itertools.chain.from_iterable(candidates_nested))
 
     if len(files) > 0 and len(candidates) == 0:
-        raise ValueError("Globs '%s' did not find any files." % (files))
+        raise ValueError("Globs '%s' did not find any files with glob." % (files))
 
     repos = get_repos()
 
-    valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates, filter_function) for r in repos]))
+    valid_files = list(
+        itertools.chain.from_iterable(
+            [r.get_candidates(candidates, filter_function) for r in repos]))
+
+    if len(files) > 0 and len(valid_files) == 0:
+        raise ValueError("Globs '%s' did not find any files with glob in git." % (files))
 
     return valid_files
 
+
 def get_files_to_check_from_patch(patches, filter_function):
-    """Take a patch file generated by git diff, and scan the patch for a list of files to check.
-    """
-    candidates = []
+    # type: (List[str], Callable[[str], bool]) -> List[str]
+    """Take a patch file generated by git diff, and scan the patch for a list of files to check."""
+    candidates = []  # type: List[str]
 
     # Get a list of candidate_files
     check = re.compile(r"^diff --git a\/([\w\/\.\-]+) b\/[\w\/\.\-]+")
 
-    lines = []
+    lines = []  # type: List[str]
     for patch in patches:
         with open(patch, "rb") as infile:
             lines += infile.readlines()
@@ -272,6 +309,8 @@ def get_files_to_check_from_patch(patches, filter_function):
 
     repos = get_repos()
 
-    valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates, filter_function) for r in repos]))
+    valid_files = list(
+        itertools.chain.from_iterable(
+            [r.get_candidates(candidates, filter_function) for r in repos]))
 
     return valid_files