SERVER-19986 clang_format.py lint should enumerate all files for scons integration

(cherry picked from commit 2700f16594bfb76eb6af5c194bd9b764ad00b586)

1 files changed, 57 insertions, 49 deletions

diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py
index a756ff58ae5..3e6f96104ed 100755
--- a/buildscripts/clang_format.py
+++ b/buildscripts/clang_format.py
@@ -31,7 +31,6 @@ from multiprocessing import cpu_count
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
 
-from buildscripts.resmokelib.utils import globstar
 from buildscripts import moduleconfig
 
 
@@ -183,7 +182,6 @@ def get_clang_format_from_linux_cache(dest_file):
     # Destination Path
     shutil.move("llvm/Release/bin/clang-format", dest_file)
 
-
 class ClangFormat(object):
     """Class encapsulates finding a suitable copy of clang-format,
     and linting/formating an individual file
@@ -318,8 +316,6 @@ def parallel_process(items, func):
     except NotImplementedError:
         cpus = 1
 
-    # print("Running across %d cpus" % (cpus))
-
     task_queue = Queue.Queue()
 
     # Use a list so that worker function will capture this variable
@@ -412,42 +408,28 @@ class Repo(object):
     def __init__(self, path):
         self.path = path
 
-       # Get candidate files
-        self.candidate_files = self._get_candidate_files()
-
         self.root = self._get_root()
 
     def _callgito(self, args):
         """Call git for this repository
         """
         # These two flags are the equivalent of -C in newer versions of Git
-        # but we use these to support versions back to ~1.8
+        # but we use these to support versions pre 1.8.5 but it depends on the command
+        # and what the current directory is
         return callo(['git', '--git-dir', os.path.join(self.path, ".git"),
-                        '--work-tree', self.path] + args)
-
-    def _get_local_dir(self, path):
-        """Get a directory path relative to the git root directory
-        """
-        if os.path.isabs(path):
-            return os.path.relpath(path, self.root)
-        return path
+                            '--work-tree', self.path] + args)
 
-    def get_candidates(self, candidates):
-        """Get the set of candidate files to check by doing an intersection
-        between the input list, and the list of candidates in the repository
+    def get_candidates(self):
+        """Get the set of candidate files to check by querying the repository
 
         Returns the full path to the file for clang-format to consume.
         """
         # NOTE: Files may have an absolute root (i.e. leading /)
-
-        if candidates is not None and len(candidates) > 0:
-            candidates = [self._get_local_dir(f) for f in candidates]
-            valid_files = list(set(candidates).intersection(self.get_candidate_files()))
-        else:
-            valid_files = list(self.get_candidate_files())
+        valid_files = list(self.get_candidate_files())
 
         # Get the full file name here
         valid_files = [os.path.normpath(os.path.join(self.root, f)) for f in valid_files]
+
         return valid_files
 
     def get_root(self):
@@ -462,15 +444,10 @@ class Repo(object):
 
         return gito.rstrip()
 
-    def get_candidate_files(self):
-        """Get a list of candidate files
-        """
-        return self._get_candidate_files()
-
-    def _get_candidate_files(self):
-        """Query git to get a list of all files in the repo to consider for analysis
+    def _git_ls_files(self, cmd):
+        """Run git-ls-files and filter the list of files to a valid candidate list
         """
-        gito = self._callgito(["ls-files"])
+        gito = self._callgito(cmd)
 
         # This allows us to pick all the interesting files
         # in the mongo and mongo-enterprise repos
@@ -485,24 +462,45 @@ class Repo(object):
 
         return file_list
 
+    def get_candidate_files(self):
+        """Query git to get a list of all files in the repo to consider for analysis
+        """
+        return self._git_ls_files(["ls-files", "--cached"])
+
+    def get_working_tree_candidate_files(self):
+        """Query git to get a list of all files in the working tree to consider for analysis
+        """
+        return self._git_ls_files(["ls-files", "--cached", "--others"])
+
+    def get_working_tree_candidates(self):
+        """Get the set of candidate files to check by querying the repository
 
-def expand_file_string(glob_pattern):
-    """Expand a string that represents a set of files
+        Returns the full path to the file for clang-format to consume.
+        """
+        valid_files = list(self.get_working_tree_candidate_files())
+
+        # Get the full file name here
+        valid_files = [os.path.normpath(os.path.join(self.root, f)) for f in valid_files]
+
+        return valid_files
+
+def get_files_to_check_working_tree():
+    """Get a list of files to check form the working tree.
+       This will pick up files not managed by git.
     """
-    return [os.path.abspath(f) for f in globstar.iglob(glob_pattern)]
+    repos = get_repos()
 
-def get_files_to_check(files):
-    """Filter the specified list of files to check down to the actual
-        list of files that need to be checked."""
-    candidates = []
+    valid_files = list(itertools.chain.from_iterable([r.get_working_tree_candidates() for r in repos]))
 
-    # Get a list of candidate_files
-    candidates = [expand_file_string(f) for f in files]
-    candidates = list(itertools.chain.from_iterable(candidates))
+    return valid_files
 
+def get_files_to_check():
+    """Get a list of files that need to be checked
+       based on which files are managed by git.
+    """
     repos = get_repos()
 
-    valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates) for r in repos]))
+    valid_files = list(itertools.chain.from_iterable([r.get_candidates() for r in repos]))
 
     return valid_files
 
@@ -552,10 +550,19 @@ def lint_patch(clang_format, infile):
     if files:
         _lint_files(clang_format, files)
 
-def lint(clang_format, glob):
+def lint(clang_format):
     """Lint files command entry point
     """
-    files = get_files_to_check(glob)
+    files = get_files_to_check()
+
+    _lint_files(clang_format, files)
+
+    return True
+
+def lint_all(clang_format):
+    """Lint files command entry point based on working tree
+    """
+    files = get_files_to_check_working_tree()
 
     _lint_files(clang_format, files)
 
@@ -582,8 +589,7 @@ def format_func(clang_format, glob):
 def usage():
     """Print usage
     """
-    print("clang-format.py supports 3 commands [ lint, lint-patch, format ]. Run "
-            " <command> -? for more information")
+    print("clang-format.py supports 4 commands [ lint, lint-all, lint-patch, format ].")
 
 def main():
     """Main entry point
@@ -597,7 +603,9 @@ def main():
         command = args[1]
 
         if command == "lint":
-            lint(options.clang_format, args[2:])
+            lint(options.clang_format)
+        elif command == "lint-all":
+            lint_all(options.clang_format)
         elif command == "lint-patch":
             lint_patch(options.clang_format, args[2:])
         elif command == "format":