diff options
Diffstat (limited to 'buildscripts/clang_tidy_condensed.py')
| -rwxr-xr-x | buildscripts/clang_tidy_condensed.py | 165 |
1 files changed, 0 insertions, 165 deletions
diff --git a/buildscripts/clang_tidy_condensed.py b/buildscripts/clang_tidy_condensed.py deleted file mode 100755 index 995ae5f3292..00000000000 --- a/buildscripts/clang_tidy_condensed.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 -"""Runs clang-tidy in parallel and combines the the results for easier viewing.""" - -import argparse -import datetime -import json -import os -import subprocess -import sys -import threading -import queue -import time -from typing import Any, Dict, List, Optional -import multiprocessing -from pathlib import Path - -import yaml - -files_to_tidy = queue.SimpleQueue() -files_to_parse = queue.SimpleQueue() - - -def _clang_tidy_executor(clang_tidy_binary: str, clang_tidy_cfg: Dict[str, Any], output_dir: str): - while True: - clang_tidy_filename: Optional[Path] = files_to_tidy.get() - if clang_tidy_filename is None: - files_to_parse.put(None) - files_to_tidy.put(None) - break - - print(f"Running clang-tidy on {clang_tidy_filename}") - clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent - os.makedirs(clang_tidy_parent_dir, exist_ok=True) - - output_filename_base = clang_tidy_parent_dir / clang_tidy_filename.name - output_filename_fixes = output_filename_base.with_suffix(".yml") - clang_tidy_command = [ - clang_tidy_binary, clang_tidy_filename, f"-export-fixes={output_filename_fixes}", - f"-config={json.dumps(clang_tidy_cfg)}" - ] - proc = subprocess.run(clang_tidy_command, capture_output=True, check=False) - if proc.returncode != 0: - output_filename_out = output_filename_base.with_suffix(".fail") - files_to_parse.put(output_filename_fixes) - print( - f"Running clang-tidy on {clang_tidy_filename} had errors see {output_filename_out}") - else: - output_filename_out = output_filename_base.with_suffix(".pass") - print(f"Running clang-tidy on {clang_tidy_filename} had no errors") - - with open(output_filename_out, 'wb') as output: - output.write(proc.stderr) - output.write(proc.stdout) - - -def _combine_errors(clang_tidy_executors: int, fixes_filename: str) -> int: - failed_files = 0 - all_fixes = {} - while clang_tidy_executors > 0: - item = files_to_parse.get() - - # Once all running threads say they are done we want to exit - if item is None: - clang_tidy_executors -= 1 - continue - - failed_files += 1 - - # Read the yaml fixes for the file to combine them with the other suggested fixes - with open(item) as input_yml: - fixes = yaml.safe_load(input_yml) - for fix in fixes['Diagnostics']: - fix_data = all_fixes.setdefault(fix["DiagnosticName"], {}).setdefault( - fix["FilePath"], {}).setdefault( - fix["FileOffset"], { - "replacements": fix["Replacements"], "message": fix["Message"], "count": 0, - "source_files": [] - }) - fix_data["count"] += 1 - fix_data["source_files"].append(fixes['MainSourceFile']) - with open(fixes_filename, "w") as files_file: - json.dump(all_fixes, files_file, indent=4, sort_keys=True) - - return failed_files - - -def _report_status(total_jobs: int, clang_tidy_executor_threads: List[threading.Thread]): - start_time = time.time() - running_jobs = 1 - while running_jobs > 0: - time.sleep(5) - pretty_time_duration = str(datetime.timedelta(seconds=time.time() - start_time)) - running_jobs = sum( - [1 for t in clang_tidy_executor_threads if t.is_alive()]) # Count threads running a job - # files_to_tidy contains a None which can be ignored - print( - f"There are {running_jobs} active jobs. The number of jobs queued is {files_to_tidy.qsize()-1}/{total_jobs}. Duration {pretty_time_duration}." - ) - - -def main(): - """Execute Main entry point.""" - - parser = argparse.ArgumentParser(description='Run multithreaded clang-tidy') - - parser.add_argument('-t', "--threads", type=int, default=multiprocessing.cpu_count(), - help="Run with a specific number of threads") - parser.add_argument("-d", "--output-dir", type=str, default="clang_tidy_fixes", - help="Directory to write all clang-tidy output to") - parser.add_argument("-o", "--fixes-file", type=str, default="clang_tidy_fixes.json", - help="Report json file to write combined fixes to") - parser.add_argument("-c", "--compile-commands", type=str, default="compile_commands.json", - help="compile_commands.json file to use to find the files to tidy") - # TODO: Is there someway to get this without hardcoding this much - parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v3") - parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=".clang-tidy") - args = parser.parse_args() - - clang_tidy_binary = f'/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy' - - with open(args.compile_commands) as compile_commands: - compile_commands = json.load(compile_commands) - - with open(args.clang_tidy_cfg) as clang_tidy_cfg: - clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg) - - for file_doc in compile_commands: - # A few special cases of files to ignore - if not "src/mongo" in file_doc["file"]: - continue - # TODO SERVER-49884 Remove this when we no longer check in generated Bison. - if "parser_gen.cpp" in file_doc["file"]: - continue - files_to_tidy.put(Path(file_doc["file"])) - - total_jobs = files_to_tidy.qsize() - files_to_tidy.put(None) - workers = args.threads - - clang_tidy_executor_threads: List[threading.Thread] = [] - for _ in range(workers): - clang_tidy_executor_threads.append( - threading.Thread(target=_clang_tidy_executor, args=(clang_tidy_binary, clang_tidy_cfg, - args.output_dir))) - clang_tidy_executor_threads[-1].start() - - report_status_thread = threading.Thread(target=_report_status, - args=(total_jobs, clang_tidy_executor_threads)) - report_status_thread.start() - - failed_files = _combine_errors(workers, Path(args.output_dir, args.fixes_file)) - - # Join all threads - report_status_thread.join() - for thread in clang_tidy_executor_threads: - thread.join() - - # Zip up all the files for upload - subprocess.run(["tar", "-czvf", args.output_dir + ".tgz", args.output_dir], check=False) - - return failed_files - - -if __name__ == "__main__": - sys.exit(main()) |