From 425eb62e4440d2de191ce336ae1557424beda449 Mon Sep 17 00:00:00 2001 From: Juan Gu Date: Mon, 26 Sep 2022 18:51:38 +0000 Subject: SERVER-67651 replace clang tidy with clang tidy condensed-follow up PR --- .clang-tidy-extra-checks | 523 --------------------------- buildscripts/clang_tidy.py | 176 +++++++++ buildscripts/clang_tidy.sh | 42 --- buildscripts/clang_tidy_condensed.py | 165 --------- buildscripts/simple_report.py | 12 +- buildscripts/tests/test_simple_report.py | 12 +- etc/evergreen.yml | 17 +- etc/evergreen_yml_components/definitions.yml | 57 +-- evergreen/run_clang_tidy.sh | 7 +- evergreen/run_clang_tidy_condensed.sh | 10 - 10 files changed, 197 insertions(+), 824 deletions(-) delete mode 100644 .clang-tidy-extra-checks create mode 100755 buildscripts/clang_tidy.py delete mode 100755 buildscripts/clang_tidy.sh delete mode 100755 buildscripts/clang_tidy_condensed.py delete mode 100755 evergreen/run_clang_tidy_condensed.sh diff --git a/.clang-tidy-extra-checks b/.clang-tidy-extra-checks deleted file mode 100644 index 6872ab2bfe7..00000000000 --- a/.clang-tidy-extra-checks +++ /dev/null @@ -1,523 +0,0 @@ ---- -# Generated with /opt/mongodbtoolchain/v3/bin/clang-tidy --dump-config and /opt/mongodbtoolchain/v3/bin/clang-tidy --list-checks --checks=* -# We want to be explicit about what checks we enable (rather than bugprone-*) to forwards compatibility with clang-tidy. -# Forwards compatibility is useful to upgrade clang tidy AND for code hinting tools that might use a newer version fo clang tidy - -# Here is an explanation for why some of the checks are disabled: -# -# -bugprone-throw-keyword-missing: based on having exception in the name -# -bugprone-virtual-near-miss: based on naming this can be avoid by using the virtual, override, final paradigm -# -bugprone-easily-swappable-parameters: too many false positives -# -/+cert*: see details in https://releases.llvm.org/7.0.0/tools/clang/tools/extra/docs/clang-tidy/checks/list.html -# +cert-err34-c: warn use of atoi vs strtol (the latter has error reporting) -# +cert-err52-cpp: do not use longjmp or setjmp -# +cert-err60-cpp: exception objects must be no-throw copy ctor -# +cert-flp30-c: do not use floating point values in for loops -# -cppcoreguidelines-pro-type-vararg: warns about c style va_args, these are use sparingly and mostly for printfs -# -cppcoreguidelines-pro-type-reinterpret-cast: warns about reinterpret_cast, used frequently in our codebase -# -cppcoreguidelines-pro-type-const-cast: warns of using const_cast, this is a well known anti-pattern so almost all cases of this will just be ignored -# -cppcoreguidelines-pro-bounds-array-to-pointer-decay: warns when decaying and array to pointer. This can be fixed by adding an explicit cast. -# -cppcoreguidelines-owning-memory (requires gsl::owner): superseded by using unique_ptr -# -cppcoreguidelines-narrowing-conversions (int i = 0; i += 0.1): Too many false positives - TODO enabled -# -cppcoreguidelines-pro-bounds-constant-array-index: requires gsl -# -cppcoreguidelines-pro-bounds-pointer-arithmetic: too many false positives (will flag char* data = ...; data[i] = 4;) -# -cppcoreguidelines-pro-type-union-access: We already have a lot of union code and this check will flag every instance of usiong a union -# -cppcoreguidelines-*: Most checks here are style problems rather than real bugs -# -hicpp-no-assembler: we use inline assembly and if we don't we should not warn about using it -# -hicpp-special-member-functions: alias for cppcoreguidelines-special-member-functions -# -google-runtime-references: disallows non-cost references as function args -# -google-readability-todo: todos should follow a specific format -# -readability-identifier-naming: like function size this is super opinionated. I want to avoid this for now and revist when we can write out own rules. -# -readability-function-size: I am not sure how we would want to use this. I also cannot be the only one deciding on what makes a function too complex. -# -readability-named-parameter: Not helpful -# -modernize-return-braced-init-list: I think this hurts readability -# -modernize-redundant-void-arg: I think this hurts readability -# - -Checks: > - clang-diagnostic-*, - -clang-analyzer-*, - -abseil-string-find-startswith - -android-cloexec-accept, - -android-cloexec-accept4, - -android-cloexec-creat, - -android-cloexec-dup, - -android-cloexec-epoll-create, - -android-cloexec-epoll-create1, - -android-cloexec-fopen, - -android-cloexec-inotify-init, - -android-cloexec-inotify-init1, - -android-cloexec-memfd-create, - -android-cloexec-open, - -android-cloexec-socket, - boost-use-to-string, - bugprone-argument-comment, - bugprone-assert-side-effect, - bugprone-bool-pointer-implicit-conversion, - bugprone-copy-constructor-init, - bugprone-dangling-handle, - bugprone-exception-escape, - bugprone-fold-init-type, - bugprone-forward-declaration-namespace, - bugprone-forwarding-reference-overload, - bugprone-inaccurate-erase, - bugprone-incorrect-roundings, - bugprone-integer-division, - bugprone-lambda-function-name, - bugprone-macro-parentheses, - bugprone-macro-repeated-side-effects, - bugprone-misplaced-operator-in-strlen-in-alloc, - bugprone-misplaced-widening-cast, - bugprone-move-forwarding-reference, - bugprone-multiple-statement-macro, - bugprone-narrowing-conversions, - bugprone-parent-virtual-call, - bugprone-sizeof-container, - bugprone-sizeof-expression, - bugprone-string-constructor, - bugprone-string-integer-assignment, - bugprone-string-literal-with-embedded-nul, - bugprone-suspicious-enum-usage, - bugprone-suspicious-memset-usage, - bugprone-suspicious-missing-comma, - bugprone-suspicious-semicolon, - bugprone-suspicious-string-compare, - bugprone-swapped-arguments, - bugprone-terminating-continue, - -bugprone-throw-keyword-missing, - bugprone-undefined-memory-manipulation, - bugprone-undelegated-constructor, - bugprone-unused-raii, - bugprone-unused-return-value, - bugprone-use-after-move, - -bugprone-virtual-near-miss, - -cert-dcl03-c, - -cert-dcl21-cpp, - -cert-dcl50-cpp, - -cert-dcl54-cpp, - -cert-dcl58-cpp, - -cert-dcl59-cpp, - -cert-env33-c, - -cert-err09-cpp, - cert-err34-c, - cert-err52-cpp, - -cert-err58-cpp, - cert-err60-cpp, - -cert-err61-cpp, - -cert-fio38-c, - cert-flp30-c, - -cert-msc30-c, - -cert-msc32-c, - -cert-msc50-cpp, - -cert-msc51-cpp, - -cert-oop11-cpp, - cppcoreguidelines-avoid-goto, - cppcoreguidelines-c-copy-assignment-signature, - cppcoreguidelines-interfaces-global-init, - cppcoreguidelines-narrowing-conversions, - -cppcoreguidelines-no-malloc, - -cppcoreguidelines-owning-memory, - -cppcoreguidelines-pro-bounds-array-to-pointer-decay, - -cppcoreguidelines-pro-bounds-constant-array-index, - -cppcoreguidelines-pro-bounds-pointer-arithmetic, - -cppcoreguidelines-pro-type-const-cast, - -cppcoreguidelines-pro-type-cstyle-cast, - -cppcoreguidelines-pro-type-member-init, - -cppcoreguidelines-pro-type-reinterpret-cast, - -cppcoreguidelines-pro-type-static-cast-downcast, - -cppcoreguidelines-pro-type-union-access, - -cppcoreguidelines-pro-type-vararg, - -cppcoreguidelines-slicing, - -cppcoreguidelines-special-member-functions, - -fuchsia-default-arguments, - -fuchsia-header-anon-namespaces, - -fuchsia-multiple-inheritance, - -fuchsia-overloaded-operator, - -fuchsia-restrict-system-includes, - -fuchsia-statically-constructed-objects, - -fuchsia-trailing-return, - -fuchsia-virtual-inheritance, - google-build-explicit-make-pair, - google-build-namespaces, - -google-build-using-namespace, - -google-default-arguments, - -google-explicit-constructor, - google-global-names-in-headers, - google-objc-avoid-throwing-exception, - google-objc-global-variable-declaration, - -google-readability-braces-around-statements, - -google-readability-casting, - google-readability-function-size, - google-readability-namespace-comments, - -google-readability-todo, - -google-runtime-int, - google-runtime-operator, - -google-runtime-references, - hicpp-avoid-goto, - -hicpp-braces-around-statements, - -hicpp-deprecated-headers, - -hicpp-exception-baseclass, - -hicpp-explicit-conversions, - hicpp-function-size, - hicpp-invalid-access-moved, - -hicpp-member-init, - hicpp-move-const-arg, - -hicpp-multiway-paths-covered, - hicpp-named-parameter, - -hicpp-new-delete-operators, - -hicpp-no-array-decay, - -hicpp-no-assembler, - -hicpp-no-malloc, - -hicpp-noexcept-move, - -hicpp-signed-bitwise, - -hicpp-special-member-functions, - hicpp-static-assert, - hicpp-undelegated-constructor, - -hicpp-use-auto, - -hicpp-use-emplace, - hicpp-use-equals-default, - hicpp-use-equals-delete, - hicpp-use-noexcept, - -hicpp-use-nullptr, - -hicpp-use-override, - hicpp-vararg, - -llvm-header-guard, - -llvm-include-order, - -llvm-namespace-comment, - -llvm-twine-local, - misc-definitions-in-headers, - misc-misplaced-const, - -misc-new-delete-overloads, - misc-non-copyable-objects, - misc-redundant-expression, - misc-static-assert, - misc-throw-by-value-catch-by-reference, - misc-unconventional-assign-operator, - misc-uniqueptr-reset-release, - misc-unused-alias-decls, - -misc-unused-parameters, - -misc-unused-using-decls, - modernize-avoid-bind, - -modernize-deprecated-headers, - -modernize-loop-convert, - modernize-make-shared, - -modernize-make-unique, - -modernize-pass-by-value, - -modernize-raw-string-literal, - -modernize-redundant-void-arg, - modernize-replace-auto-ptr, - modernize-replace-random-shuffle, - -modernize-return-braced-init-list, - modernize-shrink-to-fit, - modernize-unary-static-assert, - -modernize-use-auto, - modernize-use-bool-literals, - -modernize-use-default-member-init, - modernize-use-emplace, - -modernize-use-equals-default, - -modernize-use-equals-delete, - modernize-use-noexcept, - -modernize-use-nullptr, - -modernize-use-override, - -modernize-use-transparent-functors, - modernize-use-uncaught-exceptions, - -modernize-use-using, - -mpi-buffer-deref, - -mpi-type-mismatch, - -objc-avoid-nserror-init, - -objc-avoid-spinlock, - -objc-forbidden-subclassing, - -objc-property-declaration, - performance-faster-string-find, - performance-for-range-copy, - performance-implicit-conversion-in-loop, - performance-inefficient-algorithm, - performance-inefficient-string-concatenation, - performance-inefficient-vector-operation, - -performance-move-const-arg, - performance-move-constructor-init, - -performance-noexcept-move-constructor, - performance-type-promotion-in-math-fn, - -performance-unnecessary-copy-initialization, - performance-unnecessary-value-param, - -portability-simd-intrinsics, - readability-avoid-const-params-in-decls, - -readability-braces-around-statements, - -readability-container-size-empty, - readability-delete-null-pointer, - readability-deleted-default, - -readability-else-after-return, - -readability-function-size, - -readability-identifier-naming, - -readability-implicit-bool-conversion, - readability-inconsistent-declaration-parameter-name, - -readability-misleading-indentation, - readability-misplaced-array-index, - -readability-named-parameter, - readability-non-const-parameter, - readability-redundant-control-flow, - -readability-redundant-declaration, - readability-redundant-function-ptr-dereference, - -readability-redundant-member-init, - -readability-redundant-smartptr-get, - -readability-redundant-string-cstr, - -readability-redundant-string-init, - -readability-simplify-boolean-expr, - readability-simplify-subscript-expr, - readability-static-accessed-through-instance, - -readability-static-definition-in-anonymous-namespace, - readability-string-compare, - readability-uniqueptr-delete-release, - -zircon-temporary-objects -WarningsAsErrors: '*' -HeaderFilterRegex: '(mongo/.*|build/.*)' -AnalyzeTemporaryDtors: false -FormatStyle: none -User: ubuntu -CheckOptions: - - key: bugprone-argument-comment.StrictMode - value: '0' - - key: bugprone-assert-side-effect.AssertMacros - value: assert - - key: bugprone-assert-side-effect.CheckFunctionCalls - value: '0' - - key: bugprone-dangling-handle.HandleClasses - value: 'std::basic_string_view;std::experimental::basic_string_view' - - key: bugprone-exception-escape.FunctionsThatShouldNotThrow - value: '' - - key: bugprone-exception-escape.IgnoredExceptions - value: '' - - key: bugprone-misplaced-widening-cast.CheckImplicitCasts - value: '0' - - key: bugprone-sizeof-expression.WarnOnSizeOfCompareToConstant - value: '1' - - key: bugprone-sizeof-expression.WarnOnSizeOfConstant - value: '1' - - key: bugprone-sizeof-expression.WarnOnSizeOfIntegerExpression - value: '0' - - key: bugprone-sizeof-expression.WarnOnSizeOfThis - value: '1' - - key: bugprone-string-constructor.LargeLengthThreshold - value: '8388608' - - key: bugprone-string-constructor.WarnOnLargeLength - value: '1' - - key: bugprone-suspicious-enum-usage.StrictMode - value: '0' - - key: bugprone-suspicious-missing-comma.MaxConcatenatedTokens - value: '5' - - key: bugprone-suspicious-missing-comma.RatioThreshold - value: '0.200000' - - key: bugprone-suspicious-missing-comma.SizeThreshold - value: '5' - - key: bugprone-suspicious-string-compare.StringCompareLikeFunctions - value: '' - - key: bugprone-suspicious-string-compare.WarnOnImplicitComparison - value: '1' - - key: bugprone-suspicious-string-compare.WarnOnLogicalNotComparison - value: '0' - - key: bugprone-unused-return-value.CheckedFunctions - value: '::std::async;::std::launder;::std::remove;::std::remove_if;::std::unique;::std::unique_ptr::release;::std::basic_string::empty;::std::vector::empty' - - key: cppcoreguidelines-no-malloc.Allocations - value: '::malloc;::calloc' - - key: cppcoreguidelines-no-malloc.Deallocations - value: '::free' - - key: cppcoreguidelines-no-malloc.Reallocations - value: '::realloc' - - key: cppcoreguidelines-pro-bounds-constant-array-index.GslHeader - value: '' - - key: cppcoreguidelines-pro-bounds-constant-array-index.IncludeStyle - value: '0' - - key: cppcoreguidelines-pro-type-member-init.IgnoreArrays - value: '0' - - key: cppcoreguidelines-special-member-functions.AllowMissingMoveFunctions - value: '0' - - key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor - value: '1' - - key: google-build-namespaces.HeaderFileExtensions - value: ',h,hh,hpp,hxx' - - key: google-global-names-in-headers.HeaderFileExtensions - value: ',h,hh,hpp,hxx' - - key: google-readability-braces-around-statements.ShortStatementLines - value: '1' - - key: google-readability-function-size.BranchThreshold - value: '4294967295' - - key: google-readability-function-size.LineThreshold - value: '4294967295' - - key: google-readability-function-size.NestingThreshold - value: '4294967295' - - key: google-readability-function-size.ParameterThreshold - value: '4294967295' - - key: google-readability-function-size.StatementThreshold - value: '800' - - key: google-readability-function-size.VariableThreshold - value: '4294967295' - - key: google-readability-namespace-comments.ShortNamespaceLines - value: '10' - - key: google-readability-namespace-comments.SpacesBeforeComments - value: '2' - - key: google-runtime-int.SignedTypePrefix - value: int - - key: google-runtime-int.TypeSuffix - value: '' - - key: google-runtime-int.UnsignedTypePrefix - value: uint - - key: hicpp-braces-around-statements.ShortStatementLines - value: '0' - - key: hicpp-function-size.BranchThreshold - value: '4294967295' - - key: hicpp-function-size.LineThreshold - value: '4294967295' - - key: hicpp-function-size.NestingThreshold - value: '4294967295' - - key: hicpp-function-size.ParameterThreshold - value: '4294967295' - - key: hicpp-function-size.StatementThreshold - value: '800' - - key: hicpp-function-size.VariableThreshold - value: '4294967295' - - key: hicpp-member-init.IgnoreArrays - value: '0' - - key: hicpp-move-const-arg.CheckTriviallyCopyableMove - value: '1' - - key: hicpp-multiway-paths-covered.WarnOnMissingElse - value: '0' - - key: hicpp-named-parameter.IgnoreFailedSplit - value: '0' - - key: hicpp-no-malloc.Allocations - value: '::malloc;::calloc' - - key: hicpp-no-malloc.Deallocations - value: '::free' - - key: hicpp-no-malloc.Reallocations - value: '::realloc' - - key: hicpp-special-member-functions.AllowMissingMoveFunctions - value: '0' - - key: hicpp-special-member-functions.AllowSoleDefaultDtor - value: '0' - - key: hicpp-use-auto.MinTypeNameLength - value: '5' - - key: hicpp-use-auto.RemoveStars - value: '0' - - key: hicpp-use-emplace.ContainersWithPushBack - value: '::std::vector;::std::list;::std::deque' - - key: hicpp-use-emplace.SmartPointers - value: '::std::shared_ptr;::std::unique_ptr;::std::auto_ptr;::std::weak_ptr' - - key: hicpp-use-emplace.TupleMakeFunctions - value: '::std::make_pair;::std::make_tuple' - - key: hicpp-use-emplace.TupleTypes - value: '::std::pair;::std::tuple' - - key: hicpp-use-equals-default.IgnoreMacros - value: '1' - - key: hicpp-use-noexcept.ReplacementString - value: '' - - key: hicpp-use-noexcept.UseNoexceptFalse - value: '1' - - key: hicpp-use-nullptr.NullMacros - value: '' - - key: llvm-namespace-comment.ShortNamespaceLines - value: '1' - - key: llvm-namespace-comment.SpacesBeforeComments - value: '1' - - key: misc-definitions-in-headers.HeaderFileExtensions - value: ',h,hh,hpp,hxx' - - key: misc-definitions-in-headers.UseHeaderFileExtension - value: '1' - - key: misc-throw-by-value-catch-by-reference.CheckThrowTemporaries - value: '1' - - key: misc-unused-parameters.StrictMode - value: '0' - - key: modernize-loop-convert.MaxCopySize - value: '16' - - key: modernize-loop-convert.MinConfidence - value: reasonable - - key: modernize-loop-convert.NamingStyle - value: CamelCase - - key: modernize-make-shared.IgnoreMacros - value: '1' - - key: modernize-make-shared.IncludeStyle - value: '0' - - key: modernize-make-shared.MakeSmartPtrFunction - value: 'std::make_shared' - - key: modernize-make-shared.MakeSmartPtrFunctionHeader - value: memory - - key: modernize-make-unique.IgnoreMacros - value: '1' - - key: modernize-make-unique.IncludeStyle - value: '0' - - key: modernize-make-unique.MakeSmartPtrFunction - value: 'std::make_unique' - - key: modernize-make-unique.MakeSmartPtrFunctionHeader - value: memory - - key: modernize-pass-by-value.IncludeStyle - value: llvm - - key: modernize-pass-by-value.ValuesOnly - value: '0' - - key: modernize-raw-string-literal.ReplaceShorterLiterals - value: '0' - - key: modernize-replace-auto-ptr.IncludeStyle - value: llvm - - key: modernize-replace-random-shuffle.IncludeStyle - value: llvm - - key: modernize-use-auto.MinTypeNameLength - value: '5' - - key: modernize-use-auto.RemoveStars - value: '0' - - key: modernize-use-default-member-init.IgnoreMacros - value: '1' - - key: modernize-use-default-member-init.UseAssignment - value: '0' - - key: modernize-use-emplace.ContainersWithPushBack - value: '::std::vector;::std::list;::std::deque' - - key: modernize-use-emplace.SmartPointers - value: '::std::shared_ptr;::std::unique_ptr;::std::auto_ptr;::std::weak_ptr' - - key: modernize-use-emplace.TupleMakeFunctions - value: '::std::make_pair;::std::make_tuple' - - key: modernize-use-emplace.TupleTypes - value: '::std::pair;::std::tuple' - - key: modernize-use-equals-default.IgnoreMacros - value: '1' - - key: modernize-use-noexcept.ReplacementString - value: '' - - key: modernize-use-noexcept.UseNoexceptFalse - value: '1' - - key: modernize-use-nullptr.NullMacros - value: 'NULL' - - key: modernize-use-transparent-functors.SafeMode - value: '0' - - key: modernize-use-using.IgnoreMacros - value: '1' - - key: performance-faster-string-find.StringLikeClasses - value: 'std::basic_string' - - key: performance-for-range-copy.WarnOnAllAutoCopies - value: '0' - - key: performance-inefficient-string-concatenation.StrictMode - value: '0' - - key: performance-inefficient-vector-operation.VectorLikeClasses - value: '::std::vector' - - key: performance-move-const-arg.CheckTriviallyCopyableMove - value: '1' - - key: performance-move-constructor-init.IncludeStyle - value: llvm - - key: performance-type-promotion-in-math-fn.IncludeStyle - value: llvm - - key: performance-unnecessary-value-param.IncludeStyle - value: llvm - - key: readability-braces-around-statements.ShortStatementLines - value: '0' - - key: readability-implicit-bool-conversion.AllowIntegerConditions - value: '0' - - key: readability-implicit-bool-conversion.AllowPointerConditions - value: '0' - - key: readability-inconsistent-declaration-parameter-name.IgnoreMacros - value: '1' - - key: readability-inconsistent-declaration-parameter-name.Strict - value: '0' - - key: readability-simplify-boolean-expr.ChainedConditionalAssignment - value: '0' - - key: readability-simplify-boolean-expr.ChainedConditionalReturn - value: '0' - - key: readability-simplify-subscript-expr.Types - value: '::std::basic_string;::std::basic_string_view;::std::vector;::std::array' - - key: readability-static-accessed-through-instance.NameSpecifierNestingThreshold - value: '3' -... - diff --git a/buildscripts/clang_tidy.py b/buildscripts/clang_tidy.py new file mode 100755 index 00000000000..60631b5e9c7 --- /dev/null +++ b/buildscripts/clang_tidy.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Runs clang-tidy in parallel and combines the the results for easier viewing.""" + +import argparse +import datetime +import json +import os +import subprocess +import sys +import locale +import time +from typing import Any, Dict, List, Optional, Tuple +import multiprocessing +from pathlib import Path +from concurrent import futures +from simple_report import Result, Report, put_report, try_combine_reports, make_report +import yaml + + +def _clang_tidy_executor(clang_tidy_filename: str, clang_tidy_binary: str, + clang_tidy_cfg: Dict[str, Any], output_dir: str, + show_stdout: bool) -> Tuple[str, Optional[str]]: + + clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent + os.makedirs(clang_tidy_parent_dir, exist_ok=True) + + output_filename_base = clang_tidy_parent_dir / clang_tidy_filename.name + output_filename_fixes = output_filename_base.with_suffix(".yml") + clang_tidy_command = [ + clang_tidy_binary, clang_tidy_filename, f"-export-fixes={output_filename_fixes}", + f"-config={json.dumps(clang_tidy_cfg)}" + ] + proc = subprocess.run(clang_tidy_command, capture_output=True, check=False) + files_to_parse = None + if proc.returncode != 0: + output_filename_out = output_filename_base.with_suffix(".fail") + files_to_parse = output_filename_fixes + if not show_stdout: + print( + f"Running clang-tidy on {clang_tidy_filename} had errors see {output_filename_out}") + else: + print(f"Running clang-tidy on {clang_tidy_filename}") + print(f"{proc.stderr.decode(locale.getpreferredencoding())}") + print(f"{proc.stdout.decode(locale.getpreferredencoding())}") + else: + output_filename_out = output_filename_base.with_suffix(".pass") + if not show_stdout: + print(f"Running clang-tidy on {clang_tidy_filename} had no errors") + + with open(output_filename_out, 'wb') as output: + output.write(proc.stderr) + output.write(proc.stdout) + return proc.stdout.decode(locale.getpreferredencoding()), files_to_parse + + +def _combine_errors(fixes_filename: str, files_to_parse: List[str]) -> int: + failed_files = 0 + all_fixes = {} + + #loop files_to_parse and count the number of failed_files + for item in files_to_parse: + if item is None: + continue + failed_files += 1 + + # Read the yaml fixes for the file to combine them with the other suggested fixes + with open(item) as input_yml: + fixes = yaml.safe_load(input_yml) + for fix in fixes['Diagnostics']: + fix_data = all_fixes.setdefault(fix["DiagnosticName"], {}).setdefault( + fix["FilePath"], {}).setdefault( + fix["FileOffset"], { + "replacements": fix["Replacements"], "message": fix["Message"], "count": 0, + "source_files": [] + }) + fix_data["count"] += 1 + fix_data["source_files"].append(fixes['MainSourceFile']) + with open(fixes_filename, "w") as files_file: + json.dump(all_fixes, files_file, indent=4, sort_keys=True) + + return failed_files + + +def __dedup_errors(clang_tidy_errors_threads: List[str]) -> str: + #use dict as an 'ordered set'(in python 3.6+), set value to dummy value(true here) + error_to_dummy_value = dict() + for errs in clang_tidy_errors_threads: + if errs: + for val in errs.splitlines(): + error_to_dummy_value[val] = True + return os.linesep.join(error_to_dummy_value.keys()) + + +def main(): + """Execute Main entry point.""" + + parser = argparse.ArgumentParser(description='Run multithreaded clang-tidy') + + parser.add_argument('-t', "--threads", type=int, default=multiprocessing.cpu_count(), + help="Run with a specific number of threads") + parser.add_argument("-d", "--output-dir", type=str, default="clang_tidy_fixes", + help="Directory to write all clang-tidy output to") + parser.add_argument("-o", "--fixes-file", type=str, default="clang_tidy_fixes.json", + help="Report json file to write combined fixes to") + parser.add_argument("-c", "--compile-commands", type=str, default="compile_commands.json", + help="compile_commands.json file to use to find the files to tidy") + parser.add_argument("-q", "--show-stdout", type=bool, default=True, + help="Log errors to console") + parser.add_argument("-l", "--log-file", type=str, default="clang_tidy", + help="clang tidy log from evergreen") + # TODO: Is there someway to get this without hardcoding this much + parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v3") + parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=".clang-tidy") + args = parser.parse_args() + + clang_tidy_binary = f'/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy' + + with open(args.compile_commands) as compile_commands: + compile_commands = json.load(compile_commands) + + with open(args.clang_tidy_cfg) as clang_tidy_cfg: + clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg) + files_to_tidy = list() + files_to_parse = list() + for file_doc in compile_commands: + # A few special cases of files to ignore + if not "src/mongo" in file_doc["file"]: + continue + # TODO SERVER-49884 Remove this when we no longer check in generated Bison. + if "parser_gen.cpp" in file_doc["file"]: + continue + files_to_tidy.append(Path(file_doc["file"])) + + total_jobs = len(files_to_tidy) + workers = args.threads + + clang_tidy_errors_futures: List[str] = [] + clang_tidy_executor_futures: List[futures.ThreadPoolExecutor.submit] = [] + + # total completed tasks + tasks_completed = 0 + + with futures.ThreadPoolExecutor(max_workers=workers) as executor: + start_time = time.time() + + # submit all futures + for clang_tidy_filename in files_to_tidy: + clang_tidy_executor_futures.append( + executor.submit(_clang_tidy_executor, clang_tidy_filename, clang_tidy_binary, + clang_tidy_cfg, args.output_dir, args.show_stdout)) + + for future in futures.as_completed(clang_tidy_executor_futures): + clang_tidy_errors_futures.append(future.result()[0]) + files_to_parse.append(future.result()[1]) + tasks_completed += 1 + pretty_time_duration = str(datetime.timedelta(seconds=time.time() - start_time)) + print( + f" The number of jobs completed is {tasks_completed}/{total_jobs}. Duration {pretty_time_duration}" + ) + + failed_files = _combine_errors(Path(args.output_dir, args.fixes_file), files_to_parse) + + # Zip up all the files for upload + subprocess.run(["tar", "-czvf", args.output_dir + ".tgz", args.output_dir], check=False) + + # create report and dump to report.json + error_file_contents = __dedup_errors(clang_tidy_errors_futures) + report = make_report(args.log_file, error_file_contents, 1 if failed_files > 0 else 0) + try_combine_reports(report) + put_report(report) + + return failed_files + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/buildscripts/clang_tidy.sh b/buildscripts/clang_tidy.sh deleted file mode 100755 index 28f37143775..00000000000 --- a/buildscripts/clang_tidy.sh +++ /dev/null @@ -1,42 +0,0 @@ -set -o errexit -set -o verbose - -CLANG_TIDY_TOOLCHAIN_VERSION="${1:-v3}" -CLANG_TIDY_FIX_MODE="${2:-scan}" - -# check the version the user request matches the compile_commands -TEST_COMMAND="$(jq -r '.[] | .command' compile_commands.json | head -n 1)" -if [[ "$CLANG_TIDY_TOOLCHAIN_VERSION" != *"-force" ]] && [[ $TEST_COMMAND != "/opt/mongodbtoolchain/$CLANG_TIDY_TOOLCHAIN_VERSION"* ]]; then - echo "ERROR: compile commands generated with different toolchain version than $CLANG_TIDY_TOOLCHAIN_VERSION" - echo "Run with $CLANG_TIDY_TOOLCHAIN_VERSION-force to run clang-tidy anyways." - exit 1 -fi - -# if they forced it, extract the raw toolchain version -if [[ "$CLANG_TIDY_TOOLCHAIN_VERSION" == *"-force" ]]; then - # the ?????? here strips off the "-force" but character counting - CLANG_TIDY_TOOLCHAIN_VERSION=${CLANG_TIDY_TOOLCHAIN_VERSION%??????} -fi - -if [ "$CLANG_TIDY_FIX_MODE" == "fix" ]; then - CLANG_TIDY_MAX_ARGS=1 - CLANG_TIDY_MAX_PROCESSES=1 - CLANG_TIDY_FIX_MODE="--fix-errors" -else - CLANG_TIDY_MAX_ARGS=32 - CLANG_TIDY_MAX_PROCESSES=$(grep -c ^processor /proc/cpuinfo) - CLANG_TIDY_FIX_MODE="" -fi - -# TODO SERVER-49884 Remove this when we no longer check in generated Bison. -BISON_GENERATED_PATTERN=parser_gen\.cpp - -# Here we use the -header-filter option to instruct clang-tidy to scan our header files. The -# regex instructs clang-tidy to scan headers in our source directory with the mongo/* regex, and -# the build directory to analyze generated headers with the build/* regex -jq -r '.[] | .file' compile_commands.json \ - | grep src/mongo \ - | grep -v $BISON_GENERATED_PATTERN \ - | xargs -n $CLANG_TIDY_MAX_ARGS -P $CLANG_TIDY_MAX_PROCESSES -t \ - /opt/mongodbtoolchain/$CLANG_TIDY_TOOLCHAIN_VERSION/bin/clang-tidy \ - $CLANG_TIDY_FIX_MODE -p ./compile_commands.json diff --git a/buildscripts/clang_tidy_condensed.py b/buildscripts/clang_tidy_condensed.py deleted file mode 100755 index 995ae5f3292..00000000000 --- a/buildscripts/clang_tidy_condensed.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 -"""Runs clang-tidy in parallel and combines the the results for easier viewing.""" - -import argparse -import datetime -import json -import os -import subprocess -import sys -import threading -import queue -import time -from typing import Any, Dict, List, Optional -import multiprocessing -from pathlib import Path - -import yaml - -files_to_tidy = queue.SimpleQueue() -files_to_parse = queue.SimpleQueue() - - -def _clang_tidy_executor(clang_tidy_binary: str, clang_tidy_cfg: Dict[str, Any], output_dir: str): - while True: - clang_tidy_filename: Optional[Path] = files_to_tidy.get() - if clang_tidy_filename is None: - files_to_parse.put(None) - files_to_tidy.put(None) - break - - print(f"Running clang-tidy on {clang_tidy_filename}") - clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent - os.makedirs(clang_tidy_parent_dir, exist_ok=True) - - output_filename_base = clang_tidy_parent_dir / clang_tidy_filename.name - output_filename_fixes = output_filename_base.with_suffix(".yml") - clang_tidy_command = [ - clang_tidy_binary, clang_tidy_filename, f"-export-fixes={output_filename_fixes}", - f"-config={json.dumps(clang_tidy_cfg)}" - ] - proc = subprocess.run(clang_tidy_command, capture_output=True, check=False) - if proc.returncode != 0: - output_filename_out = output_filename_base.with_suffix(".fail") - files_to_parse.put(output_filename_fixes) - print( - f"Running clang-tidy on {clang_tidy_filename} had errors see {output_filename_out}") - else: - output_filename_out = output_filename_base.with_suffix(".pass") - print(f"Running clang-tidy on {clang_tidy_filename} had no errors") - - with open(output_filename_out, 'wb') as output: - output.write(proc.stderr) - output.write(proc.stdout) - - -def _combine_errors(clang_tidy_executors: int, fixes_filename: str) -> int: - failed_files = 0 - all_fixes = {} - while clang_tidy_executors > 0: - item = files_to_parse.get() - - # Once all running threads say they are done we want to exit - if item is None: - clang_tidy_executors -= 1 - continue - - failed_files += 1 - - # Read the yaml fixes for the file to combine them with the other suggested fixes - with open(item) as input_yml: - fixes = yaml.safe_load(input_yml) - for fix in fixes['Diagnostics']: - fix_data = all_fixes.setdefault(fix["DiagnosticName"], {}).setdefault( - fix["FilePath"], {}).setdefault( - fix["FileOffset"], { - "replacements": fix["Replacements"], "message": fix["Message"], "count": 0, - "source_files": [] - }) - fix_data["count"] += 1 - fix_data["source_files"].append(fixes['MainSourceFile']) - with open(fixes_filename, "w") as files_file: - json.dump(all_fixes, files_file, indent=4, sort_keys=True) - - return failed_files - - -def _report_status(total_jobs: int, clang_tidy_executor_threads: List[threading.Thread]): - start_time = time.time() - running_jobs = 1 - while running_jobs > 0: - time.sleep(5) - pretty_time_duration = str(datetime.timedelta(seconds=time.time() - start_time)) - running_jobs = sum( - [1 for t in clang_tidy_executor_threads if t.is_alive()]) # Count threads running a job - # files_to_tidy contains a None which can be ignored - print( - f"There are {running_jobs} active jobs. The number of jobs queued is {files_to_tidy.qsize()-1}/{total_jobs}. Duration {pretty_time_duration}." - ) - - -def main(): - """Execute Main entry point.""" - - parser = argparse.ArgumentParser(description='Run multithreaded clang-tidy') - - parser.add_argument('-t', "--threads", type=int, default=multiprocessing.cpu_count(), - help="Run with a specific number of threads") - parser.add_argument("-d", "--output-dir", type=str, default="clang_tidy_fixes", - help="Directory to write all clang-tidy output to") - parser.add_argument("-o", "--fixes-file", type=str, default="clang_tidy_fixes.json", - help="Report json file to write combined fixes to") - parser.add_argument("-c", "--compile-commands", type=str, default="compile_commands.json", - help="compile_commands.json file to use to find the files to tidy") - # TODO: Is there someway to get this without hardcoding this much - parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v3") - parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=".clang-tidy") - args = parser.parse_args() - - clang_tidy_binary = f'/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy' - - with open(args.compile_commands) as compile_commands: - compile_commands = json.load(compile_commands) - - with open(args.clang_tidy_cfg) as clang_tidy_cfg: - clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg) - - for file_doc in compile_commands: - # A few special cases of files to ignore - if not "src/mongo" in file_doc["file"]: - continue - # TODO SERVER-49884 Remove this when we no longer check in generated Bison. - if "parser_gen.cpp" in file_doc["file"]: - continue - files_to_tidy.put(Path(file_doc["file"])) - - total_jobs = files_to_tidy.qsize() - files_to_tidy.put(None) - workers = args.threads - - clang_tidy_executor_threads: List[threading.Thread] = [] - for _ in range(workers): - clang_tidy_executor_threads.append( - threading.Thread(target=_clang_tidy_executor, args=(clang_tidy_binary, clang_tidy_cfg, - args.output_dir))) - clang_tidy_executor_threads[-1].start() - - report_status_thread = threading.Thread(target=_report_status, - args=(total_jobs, clang_tidy_executor_threads)) - report_status_thread.start() - - failed_files = _combine_errors(workers, Path(args.output_dir, args.fixes_file)) - - # Join all threads - report_status_thread.join() - for thread in clang_tidy_executor_threads: - thread.join() - - # Zip up all the files for upload - subprocess.run(["tar", "-czvf", args.output_dir + ".tgz", args.output_dir], check=False) - - return failed_files - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/buildscripts/simple_report.py b/buildscripts/simple_report.py index 4364530eed8..cdfe0cf5427 100644 --- a/buildscripts/simple_report.py +++ b/buildscripts/simple_report.py @@ -51,7 +51,7 @@ def _clean_log_file(log_file: pathlib.Path, dedup_lines: bool) -> str: return os.linesep.join(lines) -def _make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report: +def make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report: status = "pass" if exit_code == 0 else "fail" return Report({ 'failures': @@ -64,7 +64,7 @@ def _make_report(test_name: str, log_file_contents: str, exit_code: int) -> Repo }) -def _try_combine_reports(out: Report): +def try_combine_reports(out: Report): try: with open("report.json") as fh: report = json.load(fh) @@ -80,7 +80,7 @@ def _dedup_lines(lines: List[str]) -> List[str]: return list(set(lines)) -def _put_report(out: Report): +def put_report(out: Report): with open("report.json", "w") as fh: json.dump(out, fh) @@ -93,9 +93,9 @@ def _put_report(out: Report): def main(test_name: str, log_file: pathlib.Path, exit_code: int, dedup_lines: bool): """Given a test name, path to log file and exit code, generate/append an Evergreen report.json.""" log_file_contents = _clean_log_file(log_file, dedup_lines) - report = _make_report(test_name, log_file_contents, exit_code) - _try_combine_reports(report) - _put_report(report) + report = make_report(test_name, log_file_contents, exit_code) + try_combine_reports(report) + put_report(report) if __name__ == "__main__": diff --git a/buildscripts/tests/test_simple_report.py b/buildscripts/tests/test_simple_report.py index ea49718671b..3ea6ed41887 100644 --- a/buildscripts/tests/test_simple_report.py +++ b/buildscripts/tests/test_simple_report.py @@ -33,9 +33,9 @@ class TestSimpleReport(unittest.TestCase): for _ in range(0, 100): self._test_trivial_report() # pylint: disable=no-value-for-parameter - @patch(ns("_try_combine_reports")) + @patch(ns("try_combine_reports")) @patch(ns("_clean_log_file")) - @patch(ns("_put_report")) + @patch(ns("put_report")) def _test_trivial_report(self, mock_put_report, mock_clean_log_file, _mock_try_combine_reports): exit_code = self.rng.randint(0, 254) print(f"Trying exit code: {exit_code}") @@ -55,8 +55,8 @@ class TestSimpleReport(unittest.TestCase): self.assertEqual(report["failures"], 1) self.assertEqual(result.exit_code, 0) - @patch(ns("_try_combine_reports")) - @patch(ns("_put_report")) + @patch(ns("try_combine_reports")) + @patch(ns("put_report")) def test_truncate_scons(self, mock_put_report, _mock_try_combine_reports): exit_code = 0 data = fix_newlines( @@ -82,8 +82,8 @@ interesting part""")) self.assertEqual(report["failures"], 0) self.assertEqual(result.exit_code, 0) - @patch(ns("_try_combine_reports")) - @patch(ns("_put_report")) + @patch(ns("try_combine_reports")) + @patch(ns("put_report")) def test_non_scons_log(self, mock_put_report, _mock_try_combine_reports): exit_code = 0 data = fix_newlines( diff --git a/etc/evergreen.yml b/etc/evergreen.yml index e54110ea7e7..782667be1ad 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -2353,9 +2353,9 @@ buildvariants: - rhel72-zseries-build - name: generate_buildid_to_debug_symbols_mapping -- &enterprise-rhel-80-64-dynamic-clang-tidy-condensed - name: enterprise-rhel80-dynamic-clang-tidy-condensed - display_name: "Enterprise Clang Tidy With Extra Checks" +- &enterprise-rhel-80-64-dynamic-clang-tidy + name: enterprise-rhel80-dynamic-clang-tidy-required + display_name: "! Enterprise Clang Tidy" cron: "0 4 * * *" modules: - enterprise @@ -2371,18 +2371,11 @@ buildvariants: scons_cache_mode: all show_scons_timings: false clang_tidy_toolchain: v3 - clang_tidy_file: .clang-tidy-extra-checks - tasks: - - name: clang_tidy_condensed_TG - -- <<: *enterprise-rhel-80-64-dynamic-clang-tidy-condensed - name: enterprise-rhel80-dynamic-clang-tidy-required - display_name: "! Enterprise Clang Tidy" - cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter + clang_tidy_file: .clang-tidy tasks: - name: clang_tidy_TG -- <<: *enterprise-rhel-80-64-dynamic-clang-tidy-condensed +- <<: *enterprise-rhel-80-64-dynamic-clang-tidy name: enterprise-rhel80-dynamic-clang-tidy-v4 display_name: "Enterprise Clang Tidy v4 Toolchain" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index c7b1a63f055..a9cbd84dfbd 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -2630,26 +2630,6 @@ tasks: ## clang_tidy - run clang_tidy - name: clang_tidy - tags: [] - exec_timeout_secs: 3600 # 1 hour timeout for the task overall - depends_on: - - name: version_expansions_gen - variant: generate-tasks-for-version - commands: - - func: "scons compile" - vars: - targets: compiledb - compiling_for_test: true - - command: subprocess.exec - type: test - timeout_secs: 3600 # 1 hour timeout for no output - params: - binary: bash - args: - - "./src/evergreen/run_clang_tidy.sh" - -## clang_tidy_condensed - run clang_tidy with python making the results clearer -- name: clang_tidy_condensed tags: [] exec_timeout_secs: 7200 # 1 hour timeout for the task overall depends_on: @@ -2666,7 +2646,7 @@ tasks: params: binary: bash args: - - "./src/evergreen/run_clang_tidy_condensed.sh" + - "./src/evergreen/run_clang_tidy.sh" ## compile_unittests ## - &compile_unittests @@ -7988,39 +7968,8 @@ task_groups: tasks: - build_metrics_tasks_gen -- name: clang_tidy_TG - setup_group_can_fail_task: true - setup_group: - - command: manifest.load - - func: "git get project and add git tag" - - func: "set task expansion macros" - - func: "f_expansions_write" - - func: "kill processes" - - func: "cleanup environment" - - func: "set up venv" - - func: "upload pip requirements" - - func: "configure evergreen api credentials" - - func: "get buildnumber" - - func: "f_expansions_write" - - func: "set up credentials" - - func: "set up win mount script" - - func: "generate compile expansions" - teardown_group: - - func: "f_expansions_write" - - func: "attach report" - - func: "umount shared scons directory" - - func: "cleanup environment" - setup_task: - - func: "apply compile expansions" - - func: "f_expansions_write" - - func: "set task expansion macros" - - func: "f_expansions_write" - teardown_task: - tasks: - - clang_tidy -# This is a copy of clang_tidy_TG and will hopefully replace it -- name: clang_tidy_condensed_TG +- name: clang_tidy_TG setup_group_can_fail_task: true setup_group: - command: manifest.load @@ -8050,7 +7999,7 @@ task_groups: - func: "f_expansions_write" teardown_task: tasks: - - clang_tidy_condensed + - clang_tidy - name: visibility_test_TG setup_group_can_fail_task: true diff --git a/evergreen/run_clang_tidy.sh b/evergreen/run_clang_tidy.sh index b1de18e730f..2c983470f48 100755 --- a/evergreen/run_clang_tidy.sh +++ b/evergreen/run_clang_tidy.sh @@ -5,10 +5,5 @@ set -o pipefail set -o verbose cd src -bash buildscripts/clang_tidy.sh ${clang_tidy_toolchain} | tee clang-tidy.log -exit_code=$? - activate_venv -$python ./buildscripts/simple_report.py --test-name clang_tidy --log-file clang-tidy.log --exit-code $exit_code --dedup-lines -echo $? -exit $exit_code +$python buildscripts/clang_tidy.py --clang-tidy-cfg ${clang_tidy_file} --clang-tidy-toolchain ${clang_tidy_toolchain} diff --git a/evergreen/run_clang_tidy_condensed.sh b/evergreen/run_clang_tidy_condensed.sh deleted file mode 100755 index 477a4c66680..00000000000 --- a/evergreen/run_clang_tidy_condensed.sh +++ /dev/null @@ -1,10 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -set -o pipefail -set -o verbose - -cd src -activate_venv -$python buildscripts/clang_tidy_condensed.py --clang-tidy-cfg ${clang_tidy_file} --clang-tidy-toolchain ${clang_tidy_toolchain} -exit $? -- cgit v1.2.1