diff options
author | auto-revert-processor <dev-prod-dag@mongodb.com> | 2022-09-24 03:59:14 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-09-24 04:28:59 +0000 |
commit | b463bf5448847019639ff76a70e658dd6837bda7 (patch) | |
tree | 05cc3e5ca383c9499364283a004e57893b94a937 | |
parent | 05cda545eb5d1911753cbd3c8004aac9308d890c (diff) | |
download | mongo-b463bf5448847019639ff76a70e658dd6837bda7.tar.gz |
Revert "SERVER-67651 replace clang tidy with clang tidy condensed"
This reverts commit 43e94464c25d98bb5ffb8c83af83bc959e343d29.
-rw-r--r-- | .clang-tidy-extra-checks | 523 | ||||
-rwxr-xr-x | buildscripts/clang_tidy.py | 176 | ||||
-rwxr-xr-x | buildscripts/clang_tidy.sh | 42 | ||||
-rwxr-xr-x | buildscripts/clang_tidy_condensed.py | 165 | ||||
-rw-r--r-- | buildscripts/simple_report.py | 12 | ||||
-rw-r--r-- | etc/evergreen.yml | 17 | ||||
-rw-r--r-- | etc/evergreen_yml_components/definitions.yml | 57 | ||||
-rwxr-xr-x | evergreen/run_clang_tidy.sh | 7 | ||||
-rwxr-xr-x | evergreen/run_clang_tidy_condensed.sh | 10 |
9 files changed, 818 insertions, 191 deletions
diff --git a/.clang-tidy-extra-checks b/.clang-tidy-extra-checks new file mode 100644 index 00000000000..6872ab2bfe7 --- /dev/null +++ b/.clang-tidy-extra-checks @@ -0,0 +1,523 @@ +--- +# Generated with /opt/mongodbtoolchain/v3/bin/clang-tidy --dump-config and /opt/mongodbtoolchain/v3/bin/clang-tidy --list-checks --checks=* +# We want to be explicit about what checks we enable (rather than bugprone-*) to forwards compatibility with clang-tidy. +# Forwards compatibility is useful to upgrade clang tidy AND for code hinting tools that might use a newer version fo clang tidy + +# Here is an explanation for why some of the checks are disabled: +# +# -bugprone-throw-keyword-missing: based on having exception in the name +# -bugprone-virtual-near-miss: based on naming this can be avoid by using the virtual, override, final paradigm +# -bugprone-easily-swappable-parameters: too many false positives +# -/+cert*: see details in https://releases.llvm.org/7.0.0/tools/clang/tools/extra/docs/clang-tidy/checks/list.html +# +cert-err34-c: warn use of atoi vs strtol (the latter has error reporting) +# +cert-err52-cpp: do not use longjmp or setjmp +# +cert-err60-cpp: exception objects must be no-throw copy ctor +# +cert-flp30-c: do not use floating point values in for loops +# -cppcoreguidelines-pro-type-vararg: warns about c style va_args, these are use sparingly and mostly for printfs +# -cppcoreguidelines-pro-type-reinterpret-cast: warns about reinterpret_cast, used frequently in our codebase +# -cppcoreguidelines-pro-type-const-cast: warns of using const_cast, this is a well known anti-pattern so almost all cases of this will just be ignored +# -cppcoreguidelines-pro-bounds-array-to-pointer-decay: warns when decaying and array to pointer. This can be fixed by adding an explicit cast. +# -cppcoreguidelines-owning-memory (requires gsl::owner<int*>): superseded by using unique_ptr +# -cppcoreguidelines-narrowing-conversions (int i = 0; i += 0.1): Too many false positives - TODO enabled +# -cppcoreguidelines-pro-bounds-constant-array-index: requires gsl +# -cppcoreguidelines-pro-bounds-pointer-arithmetic: too many false positives (will flag char* data = ...; data[i] = 4;) +# -cppcoreguidelines-pro-type-union-access: We already have a lot of union code and this check will flag every instance of usiong a union +# -cppcoreguidelines-*: Most checks here are style problems rather than real bugs +# -hicpp-no-assembler: we use inline assembly and if we don't we should not warn about using it +# -hicpp-special-member-functions: alias for cppcoreguidelines-special-member-functions +# -google-runtime-references: disallows non-cost references as function args +# -google-readability-todo: todos should follow a specific format +# -readability-identifier-naming: like function size this is super opinionated. I want to avoid this for now and revist when we can write out own rules. +# -readability-function-size: I am not sure how we would want to use this. I also cannot be the only one deciding on what makes a function too complex. +# -readability-named-parameter: Not helpful +# -modernize-return-braced-init-list: I think this hurts readability +# -modernize-redundant-void-arg: I think this hurts readability +# + +Checks: > + clang-diagnostic-*, + -clang-analyzer-*, + -abseil-string-find-startswith + -android-cloexec-accept, + -android-cloexec-accept4, + -android-cloexec-creat, + -android-cloexec-dup, + -android-cloexec-epoll-create, + -android-cloexec-epoll-create1, + -android-cloexec-fopen, + -android-cloexec-inotify-init, + -android-cloexec-inotify-init1, + -android-cloexec-memfd-create, + -android-cloexec-open, + -android-cloexec-socket, + boost-use-to-string, + bugprone-argument-comment, + bugprone-assert-side-effect, + bugprone-bool-pointer-implicit-conversion, + bugprone-copy-constructor-init, + bugprone-dangling-handle, + bugprone-exception-escape, + bugprone-fold-init-type, + bugprone-forward-declaration-namespace, + bugprone-forwarding-reference-overload, + bugprone-inaccurate-erase, + bugprone-incorrect-roundings, + bugprone-integer-division, + bugprone-lambda-function-name, + bugprone-macro-parentheses, + bugprone-macro-repeated-side-effects, + bugprone-misplaced-operator-in-strlen-in-alloc, + bugprone-misplaced-widening-cast, + bugprone-move-forwarding-reference, + bugprone-multiple-statement-macro, + bugprone-narrowing-conversions, + bugprone-parent-virtual-call, + bugprone-sizeof-container, + bugprone-sizeof-expression, + bugprone-string-constructor, + bugprone-string-integer-assignment, + bugprone-string-literal-with-embedded-nul, + bugprone-suspicious-enum-usage, + bugprone-suspicious-memset-usage, + bugprone-suspicious-missing-comma, + bugprone-suspicious-semicolon, + bugprone-suspicious-string-compare, + bugprone-swapped-arguments, + bugprone-terminating-continue, + -bugprone-throw-keyword-missing, + bugprone-undefined-memory-manipulation, + bugprone-undelegated-constructor, + bugprone-unused-raii, + bugprone-unused-return-value, + bugprone-use-after-move, + -bugprone-virtual-near-miss, + -cert-dcl03-c, + -cert-dcl21-cpp, + -cert-dcl50-cpp, + -cert-dcl54-cpp, + -cert-dcl58-cpp, + -cert-dcl59-cpp, + -cert-env33-c, + -cert-err09-cpp, + cert-err34-c, + cert-err52-cpp, + -cert-err58-cpp, + cert-err60-cpp, + -cert-err61-cpp, + -cert-fio38-c, + cert-flp30-c, + -cert-msc30-c, + -cert-msc32-c, + -cert-msc50-cpp, + -cert-msc51-cpp, + -cert-oop11-cpp, + cppcoreguidelines-avoid-goto, + cppcoreguidelines-c-copy-assignment-signature, + cppcoreguidelines-interfaces-global-init, + cppcoreguidelines-narrowing-conversions, + -cppcoreguidelines-no-malloc, + -cppcoreguidelines-owning-memory, + -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -cppcoreguidelines-pro-bounds-constant-array-index, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-const-cast, + -cppcoreguidelines-pro-type-cstyle-cast, + -cppcoreguidelines-pro-type-member-init, + -cppcoreguidelines-pro-type-reinterpret-cast, + -cppcoreguidelines-pro-type-static-cast-downcast, + -cppcoreguidelines-pro-type-union-access, + -cppcoreguidelines-pro-type-vararg, + -cppcoreguidelines-slicing, + -cppcoreguidelines-special-member-functions, + -fuchsia-default-arguments, + -fuchsia-header-anon-namespaces, + -fuchsia-multiple-inheritance, + -fuchsia-overloaded-operator, + -fuchsia-restrict-system-includes, + -fuchsia-statically-constructed-objects, + -fuchsia-trailing-return, + -fuchsia-virtual-inheritance, + google-build-explicit-make-pair, + google-build-namespaces, + -google-build-using-namespace, + -google-default-arguments, + -google-explicit-constructor, + google-global-names-in-headers, + google-objc-avoid-throwing-exception, + google-objc-global-variable-declaration, + -google-readability-braces-around-statements, + -google-readability-casting, + google-readability-function-size, + google-readability-namespace-comments, + -google-readability-todo, + -google-runtime-int, + google-runtime-operator, + -google-runtime-references, + hicpp-avoid-goto, + -hicpp-braces-around-statements, + -hicpp-deprecated-headers, + -hicpp-exception-baseclass, + -hicpp-explicit-conversions, + hicpp-function-size, + hicpp-invalid-access-moved, + -hicpp-member-init, + hicpp-move-const-arg, + -hicpp-multiway-paths-covered, + hicpp-named-parameter, + -hicpp-new-delete-operators, + -hicpp-no-array-decay, + -hicpp-no-assembler, + -hicpp-no-malloc, + -hicpp-noexcept-move, + -hicpp-signed-bitwise, + -hicpp-special-member-functions, + hicpp-static-assert, + hicpp-undelegated-constructor, + -hicpp-use-auto, + -hicpp-use-emplace, + hicpp-use-equals-default, + hicpp-use-equals-delete, + hicpp-use-noexcept, + -hicpp-use-nullptr, + -hicpp-use-override, + hicpp-vararg, + -llvm-header-guard, + -llvm-include-order, + -llvm-namespace-comment, + -llvm-twine-local, + misc-definitions-in-headers, + misc-misplaced-const, + -misc-new-delete-overloads, + misc-non-copyable-objects, + misc-redundant-expression, + misc-static-assert, + misc-throw-by-value-catch-by-reference, + misc-unconventional-assign-operator, + misc-uniqueptr-reset-release, + misc-unused-alias-decls, + -misc-unused-parameters, + -misc-unused-using-decls, + modernize-avoid-bind, + -modernize-deprecated-headers, + -modernize-loop-convert, + modernize-make-shared, + -modernize-make-unique, + -modernize-pass-by-value, + -modernize-raw-string-literal, + -modernize-redundant-void-arg, + modernize-replace-auto-ptr, + modernize-replace-random-shuffle, + -modernize-return-braced-init-list, + modernize-shrink-to-fit, + modernize-unary-static-assert, + -modernize-use-auto, + modernize-use-bool-literals, + -modernize-use-default-member-init, + modernize-use-emplace, + -modernize-use-equals-default, + -modernize-use-equals-delete, + modernize-use-noexcept, + -modernize-use-nullptr, + -modernize-use-override, + -modernize-use-transparent-functors, + modernize-use-uncaught-exceptions, + -modernize-use-using, + -mpi-buffer-deref, + -mpi-type-mismatch, + -objc-avoid-nserror-init, + -objc-avoid-spinlock, + -objc-forbidden-subclassing, + -objc-property-declaration, + performance-faster-string-find, + performance-for-range-copy, + performance-implicit-conversion-in-loop, + performance-inefficient-algorithm, + performance-inefficient-string-concatenation, + performance-inefficient-vector-operation, + -performance-move-const-arg, + performance-move-constructor-init, + -performance-noexcept-move-constructor, + performance-type-promotion-in-math-fn, + -performance-unnecessary-copy-initialization, + performance-unnecessary-value-param, + -portability-simd-intrinsics, + readability-avoid-const-params-in-decls, + -readability-braces-around-statements, + -readability-container-size-empty, + readability-delete-null-pointer, + readability-deleted-default, + -readability-else-after-return, + -readability-function-size, + -readability-identifier-naming, + -readability-implicit-bool-conversion, + readability-inconsistent-declaration-parameter-name, + -readability-misleading-indentation, + readability-misplaced-array-index, + -readability-named-parameter, + readability-non-const-parameter, + readability-redundant-control-flow, + -readability-redundant-declaration, + readability-redundant-function-ptr-dereference, + -readability-redundant-member-init, + -readability-redundant-smartptr-get, + -readability-redundant-string-cstr, + -readability-redundant-string-init, + -readability-simplify-boolean-expr, + readability-simplify-subscript-expr, + readability-static-accessed-through-instance, + -readability-static-definition-in-anonymous-namespace, + readability-string-compare, + readability-uniqueptr-delete-release, + -zircon-temporary-objects +WarningsAsErrors: '*' +HeaderFilterRegex: '(mongo/.*|build/.*)' +AnalyzeTemporaryDtors: false +FormatStyle: none +User: ubuntu +CheckOptions: + - key: bugprone-argument-comment.StrictMode + value: '0' + - key: bugprone-assert-side-effect.AssertMacros + value: assert + - key: bugprone-assert-side-effect.CheckFunctionCalls + value: '0' + - key: bugprone-dangling-handle.HandleClasses + value: 'std::basic_string_view;std::experimental::basic_string_view' + - key: bugprone-exception-escape.FunctionsThatShouldNotThrow + value: '' + - key: bugprone-exception-escape.IgnoredExceptions + value: '' + - key: bugprone-misplaced-widening-cast.CheckImplicitCasts + value: '0' + - key: bugprone-sizeof-expression.WarnOnSizeOfCompareToConstant + value: '1' + - key: bugprone-sizeof-expression.WarnOnSizeOfConstant + value: '1' + - key: bugprone-sizeof-expression.WarnOnSizeOfIntegerExpression + value: '0' + - key: bugprone-sizeof-expression.WarnOnSizeOfThis + value: '1' + - key: bugprone-string-constructor.LargeLengthThreshold + value: '8388608' + - key: bugprone-string-constructor.WarnOnLargeLength + value: '1' + - key: bugprone-suspicious-enum-usage.StrictMode + value: '0' + - key: bugprone-suspicious-missing-comma.MaxConcatenatedTokens + value: '5' + - key: bugprone-suspicious-missing-comma.RatioThreshold + value: '0.200000' + - key: bugprone-suspicious-missing-comma.SizeThreshold + value: '5' + - key: bugprone-suspicious-string-compare.StringCompareLikeFunctions + value: '' + - key: bugprone-suspicious-string-compare.WarnOnImplicitComparison + value: '1' + - key: bugprone-suspicious-string-compare.WarnOnLogicalNotComparison + value: '0' + - key: bugprone-unused-return-value.CheckedFunctions + value: '::std::async;::std::launder;::std::remove;::std::remove_if;::std::unique;::std::unique_ptr::release;::std::basic_string::empty;::std::vector::empty' + - key: cppcoreguidelines-no-malloc.Allocations + value: '::malloc;::calloc' + - key: cppcoreguidelines-no-malloc.Deallocations + value: '::free' + - key: cppcoreguidelines-no-malloc.Reallocations + value: '::realloc' + - key: cppcoreguidelines-pro-bounds-constant-array-index.GslHeader + value: '' + - key: cppcoreguidelines-pro-bounds-constant-array-index.IncludeStyle + value: '0' + - key: cppcoreguidelines-pro-type-member-init.IgnoreArrays + value: '0' + - key: cppcoreguidelines-special-member-functions.AllowMissingMoveFunctions + value: '0' + - key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor + value: '1' + - key: google-build-namespaces.HeaderFileExtensions + value: ',h,hh,hpp,hxx' + - key: google-global-names-in-headers.HeaderFileExtensions + value: ',h,hh,hpp,hxx' + - key: google-readability-braces-around-statements.ShortStatementLines + value: '1' + - key: google-readability-function-size.BranchThreshold + value: '4294967295' + - key: google-readability-function-size.LineThreshold + value: '4294967295' + - key: google-readability-function-size.NestingThreshold + value: '4294967295' + - key: google-readability-function-size.ParameterThreshold + value: '4294967295' + - key: google-readability-function-size.StatementThreshold + value: '800' + - key: google-readability-function-size.VariableThreshold + value: '4294967295' + - key: google-readability-namespace-comments.ShortNamespaceLines + value: '10' + - key: google-readability-namespace-comments.SpacesBeforeComments + value: '2' + - key: google-runtime-int.SignedTypePrefix + value: int + - key: google-runtime-int.TypeSuffix + value: '' + - key: google-runtime-int.UnsignedTypePrefix + value: uint + - key: hicpp-braces-around-statements.ShortStatementLines + value: '0' + - key: hicpp-function-size.BranchThreshold + value: '4294967295' + - key: hicpp-function-size.LineThreshold + value: '4294967295' + - key: hicpp-function-size.NestingThreshold + value: '4294967295' + - key: hicpp-function-size.ParameterThreshold + value: '4294967295' + - key: hicpp-function-size.StatementThreshold + value: '800' + - key: hicpp-function-size.VariableThreshold + value: '4294967295' + - key: hicpp-member-init.IgnoreArrays + value: '0' + - key: hicpp-move-const-arg.CheckTriviallyCopyableMove + value: '1' + - key: hicpp-multiway-paths-covered.WarnOnMissingElse + value: '0' + - key: hicpp-named-parameter.IgnoreFailedSplit + value: '0' + - key: hicpp-no-malloc.Allocations + value: '::malloc;::calloc' + - key: hicpp-no-malloc.Deallocations + value: '::free' + - key: hicpp-no-malloc.Reallocations + value: '::realloc' + - key: hicpp-special-member-functions.AllowMissingMoveFunctions + value: '0' + - key: hicpp-special-member-functions.AllowSoleDefaultDtor + value: '0' + - key: hicpp-use-auto.MinTypeNameLength + value: '5' + - key: hicpp-use-auto.RemoveStars + value: '0' + - key: hicpp-use-emplace.ContainersWithPushBack + value: '::std::vector;::std::list;::std::deque' + - key: hicpp-use-emplace.SmartPointers + value: '::std::shared_ptr;::std::unique_ptr;::std::auto_ptr;::std::weak_ptr' + - key: hicpp-use-emplace.TupleMakeFunctions + value: '::std::make_pair;::std::make_tuple' + - key: hicpp-use-emplace.TupleTypes + value: '::std::pair;::std::tuple' + - key: hicpp-use-equals-default.IgnoreMacros + value: '1' + - key: hicpp-use-noexcept.ReplacementString + value: '' + - key: hicpp-use-noexcept.UseNoexceptFalse + value: '1' + - key: hicpp-use-nullptr.NullMacros + value: '' + - key: llvm-namespace-comment.ShortNamespaceLines + value: '1' + - key: llvm-namespace-comment.SpacesBeforeComments + value: '1' + - key: misc-definitions-in-headers.HeaderFileExtensions + value: ',h,hh,hpp,hxx' + - key: misc-definitions-in-headers.UseHeaderFileExtension + value: '1' + - key: misc-throw-by-value-catch-by-reference.CheckThrowTemporaries + value: '1' + - key: misc-unused-parameters.StrictMode + value: '0' + - key: modernize-loop-convert.MaxCopySize + value: '16' + - key: modernize-loop-convert.MinConfidence + value: reasonable + - key: modernize-loop-convert.NamingStyle + value: CamelCase + - key: modernize-make-shared.IgnoreMacros + value: '1' + - key: modernize-make-shared.IncludeStyle + value: '0' + - key: modernize-make-shared.MakeSmartPtrFunction + value: 'std::make_shared' + - key: modernize-make-shared.MakeSmartPtrFunctionHeader + value: memory + - key: modernize-make-unique.IgnoreMacros + value: '1' + - key: modernize-make-unique.IncludeStyle + value: '0' + - key: modernize-make-unique.MakeSmartPtrFunction + value: 'std::make_unique' + - key: modernize-make-unique.MakeSmartPtrFunctionHeader + value: memory + - key: modernize-pass-by-value.IncludeStyle + value: llvm + - key: modernize-pass-by-value.ValuesOnly + value: '0' + - key: modernize-raw-string-literal.ReplaceShorterLiterals + value: '0' + - key: modernize-replace-auto-ptr.IncludeStyle + value: llvm + - key: modernize-replace-random-shuffle.IncludeStyle + value: llvm + - key: modernize-use-auto.MinTypeNameLength + value: '5' + - key: modernize-use-auto.RemoveStars + value: '0' + - key: modernize-use-default-member-init.IgnoreMacros + value: '1' + - key: modernize-use-default-member-init.UseAssignment + value: '0' + - key: modernize-use-emplace.ContainersWithPushBack + value: '::std::vector;::std::list;::std::deque' + - key: modernize-use-emplace.SmartPointers + value: '::std::shared_ptr;::std::unique_ptr;::std::auto_ptr;::std::weak_ptr' + - key: modernize-use-emplace.TupleMakeFunctions + value: '::std::make_pair;::std::make_tuple' + - key: modernize-use-emplace.TupleTypes + value: '::std::pair;::std::tuple' + - key: modernize-use-equals-default.IgnoreMacros + value: '1' + - key: modernize-use-noexcept.ReplacementString + value: '' + - key: modernize-use-noexcept.UseNoexceptFalse + value: '1' + - key: modernize-use-nullptr.NullMacros + value: 'NULL' + - key: modernize-use-transparent-functors.SafeMode + value: '0' + - key: modernize-use-using.IgnoreMacros + value: '1' + - key: performance-faster-string-find.StringLikeClasses + value: 'std::basic_string' + - key: performance-for-range-copy.WarnOnAllAutoCopies + value: '0' + - key: performance-inefficient-string-concatenation.StrictMode + value: '0' + - key: performance-inefficient-vector-operation.VectorLikeClasses + value: '::std::vector' + - key: performance-move-const-arg.CheckTriviallyCopyableMove + value: '1' + - key: performance-move-constructor-init.IncludeStyle + value: llvm + - key: performance-type-promotion-in-math-fn.IncludeStyle + value: llvm + - key: performance-unnecessary-value-param.IncludeStyle + value: llvm + - key: readability-braces-around-statements.ShortStatementLines + value: '0' + - key: readability-implicit-bool-conversion.AllowIntegerConditions + value: '0' + - key: readability-implicit-bool-conversion.AllowPointerConditions + value: '0' + - key: readability-inconsistent-declaration-parameter-name.IgnoreMacros + value: '1' + - key: readability-inconsistent-declaration-parameter-name.Strict + value: '0' + - key: readability-simplify-boolean-expr.ChainedConditionalAssignment + value: '0' + - key: readability-simplify-boolean-expr.ChainedConditionalReturn + value: '0' + - key: readability-simplify-subscript-expr.Types + value: '::std::basic_string;::std::basic_string_view;::std::vector;::std::array' + - key: readability-static-accessed-through-instance.NameSpecifierNestingThreshold + value: '3' +... + diff --git a/buildscripts/clang_tidy.py b/buildscripts/clang_tidy.py deleted file mode 100755 index 60631b5e9c7..00000000000 --- a/buildscripts/clang_tidy.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python3 -"""Runs clang-tidy in parallel and combines the the results for easier viewing.""" - -import argparse -import datetime -import json -import os -import subprocess -import sys -import locale -import time -from typing import Any, Dict, List, Optional, Tuple -import multiprocessing -from pathlib import Path -from concurrent import futures -from simple_report import Result, Report, put_report, try_combine_reports, make_report -import yaml - - -def _clang_tidy_executor(clang_tidy_filename: str, clang_tidy_binary: str, - clang_tidy_cfg: Dict[str, Any], output_dir: str, - show_stdout: bool) -> Tuple[str, Optional[str]]: - - clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent - os.makedirs(clang_tidy_parent_dir, exist_ok=True) - - output_filename_base = clang_tidy_parent_dir / clang_tidy_filename.name - output_filename_fixes = output_filename_base.with_suffix(".yml") - clang_tidy_command = [ - clang_tidy_binary, clang_tidy_filename, f"-export-fixes={output_filename_fixes}", - f"-config={json.dumps(clang_tidy_cfg)}" - ] - proc = subprocess.run(clang_tidy_command, capture_output=True, check=False) - files_to_parse = None - if proc.returncode != 0: - output_filename_out = output_filename_base.with_suffix(".fail") - files_to_parse = output_filename_fixes - if not show_stdout: - print( - f"Running clang-tidy on {clang_tidy_filename} had errors see {output_filename_out}") - else: - print(f"Running clang-tidy on {clang_tidy_filename}") - print(f"{proc.stderr.decode(locale.getpreferredencoding())}") - print(f"{proc.stdout.decode(locale.getpreferredencoding())}") - else: - output_filename_out = output_filename_base.with_suffix(".pass") - if not show_stdout: - print(f"Running clang-tidy on {clang_tidy_filename} had no errors") - - with open(output_filename_out, 'wb') as output: - output.write(proc.stderr) - output.write(proc.stdout) - return proc.stdout.decode(locale.getpreferredencoding()), files_to_parse - - -def _combine_errors(fixes_filename: str, files_to_parse: List[str]) -> int: - failed_files = 0 - all_fixes = {} - - #loop files_to_parse and count the number of failed_files - for item in files_to_parse: - if item is None: - continue - failed_files += 1 - - # Read the yaml fixes for the file to combine them with the other suggested fixes - with open(item) as input_yml: - fixes = yaml.safe_load(input_yml) - for fix in fixes['Diagnostics']: - fix_data = all_fixes.setdefault(fix["DiagnosticName"], {}).setdefault( - fix["FilePath"], {}).setdefault( - fix["FileOffset"], { - "replacements": fix["Replacements"], "message": fix["Message"], "count": 0, - "source_files": [] - }) - fix_data["count"] += 1 - fix_data["source_files"].append(fixes['MainSourceFile']) - with open(fixes_filename, "w") as files_file: - json.dump(all_fixes, files_file, indent=4, sort_keys=True) - - return failed_files - - -def __dedup_errors(clang_tidy_errors_threads: List[str]) -> str: - #use dict as an 'ordered set'(in python 3.6+), set value to dummy value(true here) - error_to_dummy_value = dict() - for errs in clang_tidy_errors_threads: - if errs: - for val in errs.splitlines(): - error_to_dummy_value[val] = True - return os.linesep.join(error_to_dummy_value.keys()) - - -def main(): - """Execute Main entry point.""" - - parser = argparse.ArgumentParser(description='Run multithreaded clang-tidy') - - parser.add_argument('-t', "--threads", type=int, default=multiprocessing.cpu_count(), - help="Run with a specific number of threads") - parser.add_argument("-d", "--output-dir", type=str, default="clang_tidy_fixes", - help="Directory to write all clang-tidy output to") - parser.add_argument("-o", "--fixes-file", type=str, default="clang_tidy_fixes.json", - help="Report json file to write combined fixes to") - parser.add_argument("-c", "--compile-commands", type=str, default="compile_commands.json", - help="compile_commands.json file to use to find the files to tidy") - parser.add_argument("-q", "--show-stdout", type=bool, default=True, - help="Log errors to console") - parser.add_argument("-l", "--log-file", type=str, default="clang_tidy", - help="clang tidy log from evergreen") - # TODO: Is there someway to get this without hardcoding this much - parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v3") - parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=".clang-tidy") - args = parser.parse_args() - - clang_tidy_binary = f'/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy' - - with open(args.compile_commands) as compile_commands: - compile_commands = json.load(compile_commands) - - with open(args.clang_tidy_cfg) as clang_tidy_cfg: - clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg) - files_to_tidy = list() - files_to_parse = list() - for file_doc in compile_commands: - # A few special cases of files to ignore - if not "src/mongo" in file_doc["file"]: - continue - # TODO SERVER-49884 Remove this when we no longer check in generated Bison. - if "parser_gen.cpp" in file_doc["file"]: - continue - files_to_tidy.append(Path(file_doc["file"])) - - total_jobs = len(files_to_tidy) - workers = args.threads - - clang_tidy_errors_futures: List[str] = [] - clang_tidy_executor_futures: List[futures.ThreadPoolExecutor.submit] = [] - - # total completed tasks - tasks_completed = 0 - - with futures.ThreadPoolExecutor(max_workers=workers) as executor: - start_time = time.time() - - # submit all futures - for clang_tidy_filename in files_to_tidy: - clang_tidy_executor_futures.append( - executor.submit(_clang_tidy_executor, clang_tidy_filename, clang_tidy_binary, - clang_tidy_cfg, args.output_dir, args.show_stdout)) - - for future in futures.as_completed(clang_tidy_executor_futures): - clang_tidy_errors_futures.append(future.result()[0]) - files_to_parse.append(future.result()[1]) - tasks_completed += 1 - pretty_time_duration = str(datetime.timedelta(seconds=time.time() - start_time)) - print( - f" The number of jobs completed is {tasks_completed}/{total_jobs}. Duration {pretty_time_duration}" - ) - - failed_files = _combine_errors(Path(args.output_dir, args.fixes_file), files_to_parse) - - # Zip up all the files for upload - subprocess.run(["tar", "-czvf", args.output_dir + ".tgz", args.output_dir], check=False) - - # create report and dump to report.json - error_file_contents = __dedup_errors(clang_tidy_errors_futures) - report = make_report(args.log_file, error_file_contents, 1 if failed_files > 0 else 0) - try_combine_reports(report) - put_report(report) - - return failed_files - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/buildscripts/clang_tidy.sh b/buildscripts/clang_tidy.sh new file mode 100755 index 00000000000..28f37143775 --- /dev/null +++ b/buildscripts/clang_tidy.sh @@ -0,0 +1,42 @@ +set -o errexit +set -o verbose + +CLANG_TIDY_TOOLCHAIN_VERSION="${1:-v3}" +CLANG_TIDY_FIX_MODE="${2:-scan}" + +# check the version the user request matches the compile_commands +TEST_COMMAND="$(jq -r '.[] | .command' compile_commands.json | head -n 1)" +if [[ "$CLANG_TIDY_TOOLCHAIN_VERSION" != *"-force" ]] && [[ $TEST_COMMAND != "/opt/mongodbtoolchain/$CLANG_TIDY_TOOLCHAIN_VERSION"* ]]; then + echo "ERROR: compile commands generated with different toolchain version than $CLANG_TIDY_TOOLCHAIN_VERSION" + echo "Run with $CLANG_TIDY_TOOLCHAIN_VERSION-force to run clang-tidy anyways." + exit 1 +fi + +# if they forced it, extract the raw toolchain version +if [[ "$CLANG_TIDY_TOOLCHAIN_VERSION" == *"-force" ]]; then + # the ?????? here strips off the "-force" but character counting + CLANG_TIDY_TOOLCHAIN_VERSION=${CLANG_TIDY_TOOLCHAIN_VERSION%??????} +fi + +if [ "$CLANG_TIDY_FIX_MODE" == "fix" ]; then + CLANG_TIDY_MAX_ARGS=1 + CLANG_TIDY_MAX_PROCESSES=1 + CLANG_TIDY_FIX_MODE="--fix-errors" +else + CLANG_TIDY_MAX_ARGS=32 + CLANG_TIDY_MAX_PROCESSES=$(grep -c ^processor /proc/cpuinfo) + CLANG_TIDY_FIX_MODE="" +fi + +# TODO SERVER-49884 Remove this when we no longer check in generated Bison. +BISON_GENERATED_PATTERN=parser_gen\.cpp + +# Here we use the -header-filter option to instruct clang-tidy to scan our header files. The +# regex instructs clang-tidy to scan headers in our source directory with the mongo/* regex, and +# the build directory to analyze generated headers with the build/* regex +jq -r '.[] | .file' compile_commands.json \ + | grep src/mongo \ + | grep -v $BISON_GENERATED_PATTERN \ + | xargs -n $CLANG_TIDY_MAX_ARGS -P $CLANG_TIDY_MAX_PROCESSES -t \ + /opt/mongodbtoolchain/$CLANG_TIDY_TOOLCHAIN_VERSION/bin/clang-tidy \ + $CLANG_TIDY_FIX_MODE -p ./compile_commands.json diff --git a/buildscripts/clang_tidy_condensed.py b/buildscripts/clang_tidy_condensed.py new file mode 100755 index 00000000000..995ae5f3292 --- /dev/null +++ b/buildscripts/clang_tidy_condensed.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Runs clang-tidy in parallel and combines the the results for easier viewing.""" + +import argparse +import datetime +import json +import os +import subprocess +import sys +import threading +import queue +import time +from typing import Any, Dict, List, Optional +import multiprocessing +from pathlib import Path + +import yaml + +files_to_tidy = queue.SimpleQueue() +files_to_parse = queue.SimpleQueue() + + +def _clang_tidy_executor(clang_tidy_binary: str, clang_tidy_cfg: Dict[str, Any], output_dir: str): + while True: + clang_tidy_filename: Optional[Path] = files_to_tidy.get() + if clang_tidy_filename is None: + files_to_parse.put(None) + files_to_tidy.put(None) + break + + print(f"Running clang-tidy on {clang_tidy_filename}") + clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent + os.makedirs(clang_tidy_parent_dir, exist_ok=True) + + output_filename_base = clang_tidy_parent_dir / clang_tidy_filename.name + output_filename_fixes = output_filename_base.with_suffix(".yml") + clang_tidy_command = [ + clang_tidy_binary, clang_tidy_filename, f"-export-fixes={output_filename_fixes}", + f"-config={json.dumps(clang_tidy_cfg)}" + ] + proc = subprocess.run(clang_tidy_command, capture_output=True, check=False) + if proc.returncode != 0: + output_filename_out = output_filename_base.with_suffix(".fail") + files_to_parse.put(output_filename_fixes) + print( + f"Running clang-tidy on {clang_tidy_filename} had errors see {output_filename_out}") + else: + output_filename_out = output_filename_base.with_suffix(".pass") + print(f"Running clang-tidy on {clang_tidy_filename} had no errors") + + with open(output_filename_out, 'wb') as output: + output.write(proc.stderr) + output.write(proc.stdout) + + +def _combine_errors(clang_tidy_executors: int, fixes_filename: str) -> int: + failed_files = 0 + all_fixes = {} + while clang_tidy_executors > 0: + item = files_to_parse.get() + + # Once all running threads say they are done we want to exit + if item is None: + clang_tidy_executors -= 1 + continue + + failed_files += 1 + + # Read the yaml fixes for the file to combine them with the other suggested fixes + with open(item) as input_yml: + fixes = yaml.safe_load(input_yml) + for fix in fixes['Diagnostics']: + fix_data = all_fixes.setdefault(fix["DiagnosticName"], {}).setdefault( + fix["FilePath"], {}).setdefault( + fix["FileOffset"], { + "replacements": fix["Replacements"], "message": fix["Message"], "count": 0, + "source_files": [] + }) + fix_data["count"] += 1 + fix_data["source_files"].append(fixes['MainSourceFile']) + with open(fixes_filename, "w") as files_file: + json.dump(all_fixes, files_file, indent=4, sort_keys=True) + + return failed_files + + +def _report_status(total_jobs: int, clang_tidy_executor_threads: List[threading.Thread]): + start_time = time.time() + running_jobs = 1 + while running_jobs > 0: + time.sleep(5) + pretty_time_duration = str(datetime.timedelta(seconds=time.time() - start_time)) + running_jobs = sum( + [1 for t in clang_tidy_executor_threads if t.is_alive()]) # Count threads running a job + # files_to_tidy contains a None which can be ignored + print( + f"There are {running_jobs} active jobs. The number of jobs queued is {files_to_tidy.qsize()-1}/{total_jobs}. Duration {pretty_time_duration}." + ) + + +def main(): + """Execute Main entry point.""" + + parser = argparse.ArgumentParser(description='Run multithreaded clang-tidy') + + parser.add_argument('-t', "--threads", type=int, default=multiprocessing.cpu_count(), + help="Run with a specific number of threads") + parser.add_argument("-d", "--output-dir", type=str, default="clang_tidy_fixes", + help="Directory to write all clang-tidy output to") + parser.add_argument("-o", "--fixes-file", type=str, default="clang_tidy_fixes.json", + help="Report json file to write combined fixes to") + parser.add_argument("-c", "--compile-commands", type=str, default="compile_commands.json", + help="compile_commands.json file to use to find the files to tidy") + # TODO: Is there someway to get this without hardcoding this much + parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v3") + parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=".clang-tidy") + args = parser.parse_args() + + clang_tidy_binary = f'/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy' + + with open(args.compile_commands) as compile_commands: + compile_commands = json.load(compile_commands) + + with open(args.clang_tidy_cfg) as clang_tidy_cfg: + clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg) + + for file_doc in compile_commands: + # A few special cases of files to ignore + if not "src/mongo" in file_doc["file"]: + continue + # TODO SERVER-49884 Remove this when we no longer check in generated Bison. + if "parser_gen.cpp" in file_doc["file"]: + continue + files_to_tidy.put(Path(file_doc["file"])) + + total_jobs = files_to_tidy.qsize() + files_to_tidy.put(None) + workers = args.threads + + clang_tidy_executor_threads: List[threading.Thread] = [] + for _ in range(workers): + clang_tidy_executor_threads.append( + threading.Thread(target=_clang_tidy_executor, args=(clang_tidy_binary, clang_tidy_cfg, + args.output_dir))) + clang_tidy_executor_threads[-1].start() + + report_status_thread = threading.Thread(target=_report_status, + args=(total_jobs, clang_tidy_executor_threads)) + report_status_thread.start() + + failed_files = _combine_errors(workers, Path(args.output_dir, args.fixes_file)) + + # Join all threads + report_status_thread.join() + for thread in clang_tidy_executor_threads: + thread.join() + + # Zip up all the files for upload + subprocess.run(["tar", "-czvf", args.output_dir + ".tgz", args.output_dir], check=False) + + return failed_files + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/buildscripts/simple_report.py b/buildscripts/simple_report.py index cdfe0cf5427..4364530eed8 100644 --- a/buildscripts/simple_report.py +++ b/buildscripts/simple_report.py @@ -51,7 +51,7 @@ def _clean_log_file(log_file: pathlib.Path, dedup_lines: bool) -> str: return os.linesep.join(lines) -def make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report: +def _make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report: status = "pass" if exit_code == 0 else "fail" return Report({ 'failures': @@ -64,7 +64,7 @@ def make_report(test_name: str, log_file_contents: str, exit_code: int) -> Repor }) -def try_combine_reports(out: Report): +def _try_combine_reports(out: Report): try: with open("report.json") as fh: report = json.load(fh) @@ -80,7 +80,7 @@ def _dedup_lines(lines: List[str]) -> List[str]: return list(set(lines)) -def put_report(out: Report): +def _put_report(out: Report): with open("report.json", "w") as fh: json.dump(out, fh) @@ -93,9 +93,9 @@ def put_report(out: Report): def main(test_name: str, log_file: pathlib.Path, exit_code: int, dedup_lines: bool): """Given a test name, path to log file and exit code, generate/append an Evergreen report.json.""" log_file_contents = _clean_log_file(log_file, dedup_lines) - report = make_report(test_name, log_file_contents, exit_code) - try_combine_reports(report) - put_report(report) + report = _make_report(test_name, log_file_contents, exit_code) + _try_combine_reports(report) + _put_report(report) if __name__ == "__main__": diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 782667be1ad..e54110ea7e7 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -2353,9 +2353,9 @@ buildvariants: - rhel72-zseries-build - name: generate_buildid_to_debug_symbols_mapping -- &enterprise-rhel-80-64-dynamic-clang-tidy - name: enterprise-rhel80-dynamic-clang-tidy-required - display_name: "! Enterprise Clang Tidy" +- &enterprise-rhel-80-64-dynamic-clang-tidy-condensed + name: enterprise-rhel80-dynamic-clang-tidy-condensed + display_name: "Enterprise Clang Tidy With Extra Checks" cron: "0 4 * * *" modules: - enterprise @@ -2371,11 +2371,18 @@ buildvariants: scons_cache_mode: all show_scons_timings: false clang_tidy_toolchain: v3 - clang_tidy_file: .clang-tidy + clang_tidy_file: .clang-tidy-extra-checks + tasks: + - name: clang_tidy_condensed_TG + +- <<: *enterprise-rhel-80-64-dynamic-clang-tidy-condensed + name: enterprise-rhel80-dynamic-clang-tidy-required + display_name: "! Enterprise Clang Tidy" + cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter tasks: - name: clang_tidy_TG -- <<: *enterprise-rhel-80-64-dynamic-clang-tidy +- <<: *enterprise-rhel-80-64-dynamic-clang-tidy-condensed name: enterprise-rhel80-dynamic-clang-tidy-v4 display_name: "Enterprise Clang Tidy v4 Toolchain" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index a9cbd84dfbd..c7b1a63f055 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -2631,6 +2631,26 @@ tasks: ## clang_tidy - run clang_tidy - name: clang_tidy tags: [] + exec_timeout_secs: 3600 # 1 hour timeout for the task overall + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "scons compile" + vars: + targets: compiledb + compiling_for_test: true + - command: subprocess.exec + type: test + timeout_secs: 3600 # 1 hour timeout for no output + params: + binary: bash + args: + - "./src/evergreen/run_clang_tidy.sh" + +## clang_tidy_condensed - run clang_tidy with python making the results clearer +- name: clang_tidy_condensed + tags: [] exec_timeout_secs: 7200 # 1 hour timeout for the task overall depends_on: - name: version_expansions_gen @@ -2646,7 +2666,7 @@ tasks: params: binary: bash args: - - "./src/evergreen/run_clang_tidy.sh" + - "./src/evergreen/run_clang_tidy_condensed.sh" ## compile_unittests ## - &compile_unittests @@ -7968,7 +7988,6 @@ task_groups: tasks: - build_metrics_tasks_gen - - name: clang_tidy_TG setup_group_can_fail_task: true setup_group: @@ -7989,7 +8008,6 @@ task_groups: teardown_group: - func: "f_expansions_write" - func: "attach report" - - func: "upload clang tidy results" - func: "umount shared scons directory" - func: "cleanup environment" setup_task: @@ -8001,6 +8019,39 @@ task_groups: tasks: - clang_tidy +# This is a copy of clang_tidy_TG and will hopefully replace it +- name: clang_tidy_condensed_TG + setup_group_can_fail_task: true + setup_group: + - command: manifest.load + - func: "git get project and add git tag" + - func: "set task expansion macros" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "configure evergreen api credentials" + - func: "get buildnumber" + - func: "f_expansions_write" + - func: "set up credentials" + - func: "set up win mount script" + - func: "generate compile expansions" + teardown_group: + - func: "f_expansions_write" + - func: "attach report" + - func: "upload clang tidy results" + - func: "umount shared scons directory" + - func: "cleanup environment" + setup_task: + - func: "apply compile expansions" + - func: "f_expansions_write" + - func: "set task expansion macros" + - func: "f_expansions_write" + teardown_task: + tasks: + - clang_tidy_condensed + - name: visibility_test_TG setup_group_can_fail_task: true max_hosts: 1 diff --git a/evergreen/run_clang_tidy.sh b/evergreen/run_clang_tidy.sh index 2c983470f48..b1de18e730f 100755 --- a/evergreen/run_clang_tidy.sh +++ b/evergreen/run_clang_tidy.sh @@ -5,5 +5,10 @@ set -o pipefail set -o verbose cd src +bash buildscripts/clang_tidy.sh ${clang_tidy_toolchain} | tee clang-tidy.log +exit_code=$? + activate_venv -$python buildscripts/clang_tidy.py --clang-tidy-cfg ${clang_tidy_file} --clang-tidy-toolchain ${clang_tidy_toolchain} +$python ./buildscripts/simple_report.py --test-name clang_tidy --log-file clang-tidy.log --exit-code $exit_code --dedup-lines +echo $? +exit $exit_code diff --git a/evergreen/run_clang_tidy_condensed.sh b/evergreen/run_clang_tidy_condensed.sh new file mode 100755 index 00000000000..477a4c66680 --- /dev/null +++ b/evergreen/run_clang_tidy_condensed.sh @@ -0,0 +1,10 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +set -o pipefail +set -o verbose + +cd src +activate_venv +$python buildscripts/clang_tidy_condensed.py --clang-tidy-cfg ${clang_tidy_file} --clang-tidy-toolchain ${clang_tidy_toolchain} +exit $? |