SERVER-67651 replace clang tidy with clang tidy condensed-follow up PR

5 files changed, 188 insertions, 219 deletions

diff --git a/buildscripts/clang_tidy.py b/buildscripts/clang_tidy.py
new file mode 100755
index 00000000000..60631b5e9c7
--- /dev/null
+++ b/buildscripts/clang_tidy.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""Runs clang-tidy in parallel and combines the the results for easier viewing."""
+
+import argparse
+import datetime
+import json
+import os
+import subprocess
+import sys
+import locale
+import time
+from typing import Any, Dict, List, Optional, Tuple
+import multiprocessing
+from pathlib import Path
+from concurrent import futures
+from simple_report import Result, Report, put_report, try_combine_reports, make_report
+import yaml
+
+
+def _clang_tidy_executor(clang_tidy_filename: str, clang_tidy_binary: str,
+                         clang_tidy_cfg: Dict[str, Any], output_dir: str,
+                         show_stdout: bool) -> Tuple[str, Optional[str]]:
+
+    clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent
+    os.makedirs(clang_tidy_parent_dir, exist_ok=True)
+
+    output_filename_base = clang_tidy_parent_dir / clang_tidy_filename.name
+    output_filename_fixes = output_filename_base.with_suffix(".yml")
+    clang_tidy_command = [
+        clang_tidy_binary, clang_tidy_filename, f"-export-fixes={output_filename_fixes}",
+        f"-config={json.dumps(clang_tidy_cfg)}"
+    ]
+    proc = subprocess.run(clang_tidy_command, capture_output=True, check=False)
+    files_to_parse = None
+    if proc.returncode != 0:
+        output_filename_out = output_filename_base.with_suffix(".fail")
+        files_to_parse = output_filename_fixes
+        if not show_stdout:
+            print(
+                f"Running clang-tidy on {clang_tidy_filename} had errors see {output_filename_out}")
+        else:
+            print(f"Running clang-tidy on {clang_tidy_filename}")
+            print(f"{proc.stderr.decode(locale.getpreferredencoding())}")
+            print(f"{proc.stdout.decode(locale.getpreferredencoding())}")
+    else:
+        output_filename_out = output_filename_base.with_suffix(".pass")
+        if not show_stdout:
+            print(f"Running clang-tidy on {clang_tidy_filename} had no errors")
+
+    with open(output_filename_out, 'wb') as output:
+        output.write(proc.stderr)
+        output.write(proc.stdout)
+    return proc.stdout.decode(locale.getpreferredencoding()), files_to_parse
+
+
+def _combine_errors(fixes_filename: str, files_to_parse: List[str]) -> int:
+    failed_files = 0
+    all_fixes = {}
+
+    #loop files_to_parse and count the number of failed_files
+    for item in files_to_parse:
+        if item is None:
+            continue
+        failed_files += 1
+
+        # Read the yaml fixes for the file to combine them with the other suggested fixes
+        with open(item) as input_yml:
+            fixes = yaml.safe_load(input_yml)
+        for fix in fixes['Diagnostics']:
+            fix_data = all_fixes.setdefault(fix["DiagnosticName"], {}).setdefault(
+                fix["FilePath"], {}).setdefault(
+                    fix["FileOffset"], {
+                        "replacements": fix["Replacements"], "message": fix["Message"], "count": 0,
+                        "source_files": []
+                    })
+            fix_data["count"] += 1
+            fix_data["source_files"].append(fixes['MainSourceFile'])
+    with open(fixes_filename, "w") as files_file:
+        json.dump(all_fixes, files_file, indent=4, sort_keys=True)
+
+    return failed_files
+
+
+def __dedup_errors(clang_tidy_errors_threads: List[str]) -> str:
+    #use dict as an 'ordered set'(in python 3.6+), set value to dummy value(true here)
+    error_to_dummy_value = dict()
+    for errs in clang_tidy_errors_threads:
+        if errs:
+            for val in errs.splitlines():
+                error_to_dummy_value[val] = True
+    return os.linesep.join(error_to_dummy_value.keys())
+
+
+def main():
+    """Execute Main entry point."""
+
+    parser = argparse.ArgumentParser(description='Run multithreaded clang-tidy')
+
+    parser.add_argument('-t', "--threads", type=int, default=multiprocessing.cpu_count(),
+                        help="Run with a specific number of threads")
+    parser.add_argument("-d", "--output-dir", type=str, default="clang_tidy_fixes",
+                        help="Directory to write all clang-tidy output to")
+    parser.add_argument("-o", "--fixes-file", type=str, default="clang_tidy_fixes.json",
+                        help="Report json file to write combined fixes to")
+    parser.add_argument("-c", "--compile-commands", type=str, default="compile_commands.json",
+                        help="compile_commands.json file to use to find the files to tidy")
+    parser.add_argument("-q", "--show-stdout", type=bool, default=True,
+                        help="Log errors to console")
+    parser.add_argument("-l", "--log-file", type=str, default="clang_tidy",
+                        help="clang tidy log from evergreen")
+    # TODO: Is there someway to get this without hardcoding this much
+    parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v3")
+    parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=".clang-tidy")
+    args = parser.parse_args()
+
+    clang_tidy_binary = f'/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy'
+
+    with open(args.compile_commands) as compile_commands:
+        compile_commands = json.load(compile_commands)
+
+    with open(args.clang_tidy_cfg) as clang_tidy_cfg:
+        clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg)
+    files_to_tidy = list()
+    files_to_parse = list()
+    for file_doc in compile_commands:
+        # A few special cases of files to ignore
+        if not "src/mongo" in file_doc["file"]:
+            continue
+        # TODO SERVER-49884 Remove this when we no longer check in generated Bison.
+        if "parser_gen.cpp" in file_doc["file"]:
+            continue
+        files_to_tidy.append(Path(file_doc["file"]))
+
+    total_jobs = len(files_to_tidy)
+    workers = args.threads
+
+    clang_tidy_errors_futures: List[str] = []
+    clang_tidy_executor_futures: List[futures.ThreadPoolExecutor.submit] = []
+
+    # total completed tasks
+    tasks_completed = 0
+
+    with futures.ThreadPoolExecutor(max_workers=workers) as executor:
+        start_time = time.time()
+
+        # submit all futures
+        for clang_tidy_filename in files_to_tidy:
+            clang_tidy_executor_futures.append(
+                executor.submit(_clang_tidy_executor, clang_tidy_filename, clang_tidy_binary,
+                                clang_tidy_cfg, args.output_dir, args.show_stdout))
+
+        for future in futures.as_completed(clang_tidy_executor_futures):
+            clang_tidy_errors_futures.append(future.result()[0])
+            files_to_parse.append(future.result()[1])
+            tasks_completed += 1
+            pretty_time_duration = str(datetime.timedelta(seconds=time.time() - start_time))
+            print(
+                f" The number of jobs completed is {tasks_completed}/{total_jobs}. Duration {pretty_time_duration}"
+            )
+
+    failed_files = _combine_errors(Path(args.output_dir, args.fixes_file), files_to_parse)
+
+    # Zip up all the files for upload
+    subprocess.run(["tar", "-czvf", args.output_dir + ".tgz", args.output_dir], check=False)
+
+    # create report and dump to report.json
+    error_file_contents = __dedup_errors(clang_tidy_errors_futures)
+    report = make_report(args.log_file, error_file_contents, 1 if failed_files > 0 else 0)
+    try_combine_reports(report)
+    put_report(report)
+
+    return failed_files
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/buildscripts/clang_tidy.sh b/buildscripts/clang_tidy.sh
deleted file mode 100755
index 28f37143775..00000000000
--- a/buildscripts/clang_tidy.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-set -o errexit
-set -o verbose
-
-CLANG_TIDY_TOOLCHAIN_VERSION="${1:-v3}"
-CLANG_TIDY_FIX_MODE="${2:-scan}"
-
-# check the version the user request matches the compile_commands
-TEST_COMMAND="$(jq -r '.[] | .command' compile_commands.json | head -n 1)"
-if [[ "$CLANG_TIDY_TOOLCHAIN_VERSION" != *"-force" ]] && [[ $TEST_COMMAND != "/opt/mongodbtoolchain/$CLANG_TIDY_TOOLCHAIN_VERSION"* ]]; then
-  echo "ERROR: compile commands generated with different toolchain version than $CLANG_TIDY_TOOLCHAIN_VERSION"
-  echo "Run with $CLANG_TIDY_TOOLCHAIN_VERSION-force to run clang-tidy anyways."
-  exit 1
-fi
-
-# if they forced it, extract the raw toolchain version
-if [[ "$CLANG_TIDY_TOOLCHAIN_VERSION" == *"-force" ]]; then
-  # the ?????? here strips off the "-force" but character counting
-  CLANG_TIDY_TOOLCHAIN_VERSION=${CLANG_TIDY_TOOLCHAIN_VERSION%??????}
-fi
-
-if [ "$CLANG_TIDY_FIX_MODE" == "fix" ]; then
-  CLANG_TIDY_MAX_ARGS=1
-  CLANG_TIDY_MAX_PROCESSES=1
-  CLANG_TIDY_FIX_MODE="--fix-errors"
-else
-  CLANG_TIDY_MAX_ARGS=32
-  CLANG_TIDY_MAX_PROCESSES=$(grep -c ^processor /proc/cpuinfo)
-  CLANG_TIDY_FIX_MODE=""
-fi
-
-# TODO SERVER-49884 Remove this when we no longer check in generated Bison.
-BISON_GENERATED_PATTERN=parser_gen\.cpp
-
-# Here we use the -header-filter option to instruct clang-tidy to scan our header files. The
-# regex instructs clang-tidy to scan headers in our source directory with the mongo/* regex, and
-# the build directory to analyze generated headers with the build/* regex
-jq -r '.[] | .file' compile_commands.json \
-  | grep src/mongo \
-  | grep -v $BISON_GENERATED_PATTERN \
-  | xargs -n $CLANG_TIDY_MAX_ARGS -P $CLANG_TIDY_MAX_PROCESSES -t \
-    /opt/mongodbtoolchain/$CLANG_TIDY_TOOLCHAIN_VERSION/bin/clang-tidy \
-    $CLANG_TIDY_FIX_MODE -p ./compile_commands.json
diff --git a/buildscripts/clang_tidy_condensed.py b/buildscripts/clang_tidy_condensed.py
deleted file mode 100755
index 995ae5f3292..00000000000
--- a/buildscripts/clang_tidy_condensed.py
+++ /dev/null
@@ -1,165 +0,0 @@
-#!/usr/bin/env python3
-"""Runs clang-tidy in parallel and combines the the results for easier viewing."""
-
-import argparse
-import datetime
-import json
-import os
-import subprocess
-import sys
-import threading
-import queue
-import time
-from typing import Any, Dict, List, Optional
-import multiprocessing
-from pathlib import Path
-
-import yaml
-
-files_to_tidy = queue.SimpleQueue()
-files_to_parse = queue.SimpleQueue()
-
-
-def _clang_tidy_executor(clang_tidy_binary: str, clang_tidy_cfg: Dict[str, Any], output_dir: str):
-    while True:
-        clang_tidy_filename: Optional[Path] = files_to_tidy.get()
-        if clang_tidy_filename is None:
-            files_to_parse.put(None)
-            files_to_tidy.put(None)
-            break
-
-        print(f"Running clang-tidy on {clang_tidy_filename}")
-        clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent
-        os.makedirs(clang_tidy_parent_dir, exist_ok=True)
-
-        output_filename_base = clang_tidy_parent_dir / clang_tidy_filename.name
-        output_filename_fixes = output_filename_base.with_suffix(".yml")
-        clang_tidy_command = [
-            clang_tidy_binary, clang_tidy_filename, f"-export-fixes={output_filename_fixes}",
-            f"-config={json.dumps(clang_tidy_cfg)}"
-        ]
-        proc = subprocess.run(clang_tidy_command, capture_output=True, check=False)
-        if proc.returncode != 0:
-            output_filename_out = output_filename_base.with_suffix(".fail")
-            files_to_parse.put(output_filename_fixes)
-            print(
-                f"Running clang-tidy on {clang_tidy_filename} had errors see {output_filename_out}")
-        else:
-            output_filename_out = output_filename_base.with_suffix(".pass")
-            print(f"Running clang-tidy on {clang_tidy_filename} had no errors")
-
-        with open(output_filename_out, 'wb') as output:
-            output.write(proc.stderr)
-            output.write(proc.stdout)
-
-
-def _combine_errors(clang_tidy_executors: int, fixes_filename: str) -> int:
-    failed_files = 0
-    all_fixes = {}
-    while clang_tidy_executors > 0:
-        item = files_to_parse.get()
-
-        # Once all running threads say they are done we want to exit
-        if item is None:
-            clang_tidy_executors -= 1
-            continue
-
-        failed_files += 1
-
-        # Read the yaml fixes for the file to combine them with the other suggested fixes
-        with open(item) as input_yml:
-            fixes = yaml.safe_load(input_yml)
-        for fix in fixes['Diagnostics']:
-            fix_data = all_fixes.setdefault(fix["DiagnosticName"], {}).setdefault(
-                fix["FilePath"], {}).setdefault(
-                    fix["FileOffset"], {
-                        "replacements": fix["Replacements"], "message": fix["Message"], "count": 0,
-                        "source_files": []
-                    })
-            fix_data["count"] += 1
-            fix_data["source_files"].append(fixes['MainSourceFile'])
-    with open(fixes_filename, "w") as files_file:
-        json.dump(all_fixes, files_file, indent=4, sort_keys=True)
-
-    return failed_files
-
-
-def _report_status(total_jobs: int, clang_tidy_executor_threads: List[threading.Thread]):
-    start_time = time.time()
-    running_jobs = 1
-    while running_jobs > 0:
-        time.sleep(5)
-        pretty_time_duration = str(datetime.timedelta(seconds=time.time() - start_time))
-        running_jobs = sum(
-            [1 for t in clang_tidy_executor_threads if t.is_alive()])  # Count threads running a job
-        # files_to_tidy contains a None which can be ignored
-        print(
-            f"There are {running_jobs} active jobs. The number of jobs queued is {files_to_tidy.qsize()-1}/{total_jobs}. Duration {pretty_time_duration}."
-        )
-
-
-def main():
-    """Execute Main entry point."""
-
-    parser = argparse.ArgumentParser(description='Run multithreaded clang-tidy')
-
-    parser.add_argument('-t', "--threads", type=int, default=multiprocessing.cpu_count(),
-                        help="Run with a specific number of threads")
-    parser.add_argument("-d", "--output-dir", type=str, default="clang_tidy_fixes",
-                        help="Directory to write all clang-tidy output to")
-    parser.add_argument("-o", "--fixes-file", type=str, default="clang_tidy_fixes.json",
-                        help="Report json file to write combined fixes to")
-    parser.add_argument("-c", "--compile-commands", type=str, default="compile_commands.json",
-                        help="compile_commands.json file to use to find the files to tidy")
-    # TODO: Is there someway to get this without hardcoding this much
-    parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v3")
-    parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=".clang-tidy")
-    args = parser.parse_args()
-
-    clang_tidy_binary = f'/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy'
-
-    with open(args.compile_commands) as compile_commands:
-        compile_commands = json.load(compile_commands)
-
-    with open(args.clang_tidy_cfg) as clang_tidy_cfg:
-        clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg)
-
-    for file_doc in compile_commands:
-        # A few special cases of files to ignore
-        if not "src/mongo" in file_doc["file"]:
-            continue
-        # TODO SERVER-49884 Remove this when we no longer check in generated Bison.
-        if "parser_gen.cpp" in file_doc["file"]:
-            continue
-        files_to_tidy.put(Path(file_doc["file"]))
-
-    total_jobs = files_to_tidy.qsize()
-    files_to_tidy.put(None)
-    workers = args.threads
-
-    clang_tidy_executor_threads: List[threading.Thread] = []
-    for _ in range(workers):
-        clang_tidy_executor_threads.append(
-            threading.Thread(target=_clang_tidy_executor, args=(clang_tidy_binary, clang_tidy_cfg,
-                                                                args.output_dir)))
-        clang_tidy_executor_threads[-1].start()
-
-    report_status_thread = threading.Thread(target=_report_status,
-                                            args=(total_jobs, clang_tidy_executor_threads))
-    report_status_thread.start()
-
-    failed_files = _combine_errors(workers, Path(args.output_dir, args.fixes_file))
-
-    # Join all threads
-    report_status_thread.join()
-    for thread in clang_tidy_executor_threads:
-        thread.join()
-
-    # Zip up all the files for upload
-    subprocess.run(["tar", "-czvf", args.output_dir + ".tgz", args.output_dir], check=False)
-
-    return failed_files
-
-
-if __name__ == "__main__":
-    sys.exit(main())
diff --git a/buildscripts/simple_report.py b/buildscripts/simple_report.py
index 4364530eed8..cdfe0cf5427 100644
--- a/buildscripts/simple_report.py
+++ b/buildscripts/simple_report.py
@@ -51,7 +51,7 @@ def _clean_log_file(log_file: pathlib.Path, dedup_lines: bool) -> str:
     return os.linesep.join(lines)
 
 
-def _make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report:
+def make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report:
     status = "pass" if exit_code == 0 else "fail"
     return Report({
         'failures':
@@ -64,7 +64,7 @@ def _make_report(test_name: str, log_file_contents: str, exit_code: int) -> Repo
     })
 
 
-def _try_combine_reports(out: Report):
+def try_combine_reports(out: Report):
     try:
         with open("report.json") as fh:
             report = json.load(fh)
@@ -80,7 +80,7 @@ def _dedup_lines(lines: List[str]) -> List[str]:
     return list(set(lines))
 
 
-def _put_report(out: Report):
+def put_report(out: Report):
     with open("report.json", "w") as fh:
         json.dump(out, fh)
 
@@ -93,9 +93,9 @@ def _put_report(out: Report):
 def main(test_name: str, log_file: pathlib.Path, exit_code: int, dedup_lines: bool):
     """Given a test name, path to log file and exit code, generate/append an Evergreen report.json."""
     log_file_contents = _clean_log_file(log_file, dedup_lines)
-    report = _make_report(test_name, log_file_contents, exit_code)
-    _try_combine_reports(report)
-    _put_report(report)
+    report = make_report(test_name, log_file_contents, exit_code)
+    try_combine_reports(report)
+    put_report(report)
 
 
 if __name__ == "__main__":
diff --git a/buildscripts/tests/test_simple_report.py b/buildscripts/tests/test_simple_report.py
index ea49718671b..3ea6ed41887 100644
--- a/buildscripts/tests/test_simple_report.py
+++ b/buildscripts/tests/test_simple_report.py
@@ -33,9 +33,9 @@ class TestSimpleReport(unittest.TestCase):
         for _ in range(0, 100):
             self._test_trivial_report()  # pylint: disable=no-value-for-parameter
 
-    @patch(ns("_try_combine_reports"))
+    @patch(ns("try_combine_reports"))
     @patch(ns("_clean_log_file"))
-    @patch(ns("_put_report"))
+    @patch(ns("put_report"))
     def _test_trivial_report(self, mock_put_report, mock_clean_log_file, _mock_try_combine_reports):
         exit_code = self.rng.randint(0, 254)
         print(f"Trying exit code: {exit_code}")
@@ -55,8 +55,8 @@ class TestSimpleReport(unittest.TestCase):
             self.assertEqual(report["failures"], 1)
         self.assertEqual(result.exit_code, 0)
 
-    @patch(ns("_try_combine_reports"))
-    @patch(ns("_put_report"))
+    @patch(ns("try_combine_reports"))
+    @patch(ns("put_report"))
     def test_truncate_scons(self, mock_put_report, _mock_try_combine_reports):
         exit_code = 0
         data = fix_newlines(
@@ -82,8 +82,8 @@ interesting part"""))
         self.assertEqual(report["failures"], 0)
         self.assertEqual(result.exit_code, 0)
 
-    @patch(ns("_try_combine_reports"))
-    @patch(ns("_put_report"))
+    @patch(ns("try_combine_reports"))
+    @patch(ns("put_report"))
     def test_non_scons_log(self, mock_put_report, _mock_try_combine_reports):
         exit_code = 0
         data = fix_newlines(