SERVER-67134 Added new evergreen job for new clang tidy checks

5 files changed, 790 insertions, 28 deletions

diff --git a/.clang-tidy-extra-checks b/.clang-tidy-extra-checks
new file mode 100644
index 00000000000..6872ab2bfe7
--- /dev/null
+++ b/.clang-tidy-extra-checks
@@ -0,0 +1,523 @@
+---
+# Generated with /opt/mongodbtoolchain/v3/bin/clang-tidy --dump-config and /opt/mongodbtoolchain/v3/bin/clang-tidy --list-checks --checks=*
+# We want to be explicit about what checks we enable (rather than bugprone-*) to forwards compatibility with clang-tidy.
+# Forwards compatibility is useful to upgrade clang tidy AND for code hinting tools that might use a newer version fo clang tidy
+
+# Here is an explanation for why some of the checks are disabled:
+#
+#  -bugprone-throw-keyword-missing: based on having exception in the name
+#  -bugprone-virtual-near-miss: based on naming this can be avoid by using the virtual, override, final paradigm
+#  -bugprone-easily-swappable-parameters: too many false positives
+#  -/+cert*: see details in https://releases.llvm.org/7.0.0/tools/clang/tools/extra/docs/clang-tidy/checks/list.html
+#      +cert-err34-c: warn use of atoi vs strtol (the latter has error reporting)
+#      +cert-err52-cpp: do not use longjmp or setjmp
+#      +cert-err60-cpp: exception objects must be no-throw copy ctor
+#      +cert-flp30-c: do not use floating point values in for loops
+#  -cppcoreguidelines-pro-type-vararg: warns about c style va_args, these are use sparingly and mostly for printfs
+#  -cppcoreguidelines-pro-type-reinterpret-cast: warns about reinterpret_cast, used frequently in our codebase
+#  -cppcoreguidelines-pro-type-const-cast: warns of using const_cast, this is a well known anti-pattern so almost all cases of this will just be ignored 
+#  -cppcoreguidelines-pro-bounds-array-to-pointer-decay: warns when decaying and array to pointer. This can be fixed by adding an explicit cast.
+#  -cppcoreguidelines-owning-memory (requires gsl::owner<int*>): superseded by using unique_ptr
+#  -cppcoreguidelines-narrowing-conversions (int i = 0; i += 0.1): Too many false positives - TODO enabled
+#  -cppcoreguidelines-pro-bounds-constant-array-index: requires gsl
+#  -cppcoreguidelines-pro-bounds-pointer-arithmetic: too many false positives (will flag char* data = ...; data[i] = 4;)
+#  -cppcoreguidelines-pro-type-union-access: We already have a lot of union code and this check will flag every instance of usiong a union
+#  -cppcoreguidelines-*: Most checks here are style problems rather than real bugs
+#  -hicpp-no-assembler: we use inline assembly and if we don't we should not warn about using it
+#  -hicpp-special-member-functions: alias for cppcoreguidelines-special-member-functions
+#  -google-runtime-references: disallows non-cost references as function args
+#  -google-readability-todo: todos should follow a specific format
+#  -readability-identifier-naming: like function size this is super opinionated. I want to avoid this for now and revist when we can write out own rules.
+#  -readability-function-size: I am not sure how we would want to use this. I also cannot be the only one deciding on what makes a function too complex.
+#  -readability-named-parameter: Not helpful
+#  -modernize-return-braced-init-list: I think this hurts readability
+#  -modernize-redundant-void-arg: I think this hurts readability
+#
+
+Checks: >
+  clang-diagnostic-*,
+  -clang-analyzer-*,
+  -abseil-string-find-startswith
+  -android-cloexec-accept,
+  -android-cloexec-accept4,
+  -android-cloexec-creat,
+  -android-cloexec-dup,
+  -android-cloexec-epoll-create,
+  -android-cloexec-epoll-create1,
+  -android-cloexec-fopen,
+  -android-cloexec-inotify-init,
+  -android-cloexec-inotify-init1,
+  -android-cloexec-memfd-create,
+  -android-cloexec-open,
+  -android-cloexec-socket,
+  boost-use-to-string,
+  bugprone-argument-comment,
+  bugprone-assert-side-effect,
+  bugprone-bool-pointer-implicit-conversion,
+  bugprone-copy-constructor-init,
+  bugprone-dangling-handle,
+  bugprone-exception-escape,
+  bugprone-fold-init-type,
+  bugprone-forward-declaration-namespace,
+  bugprone-forwarding-reference-overload,
+  bugprone-inaccurate-erase,
+  bugprone-incorrect-roundings,
+  bugprone-integer-division,
+  bugprone-lambda-function-name,
+  bugprone-macro-parentheses,
+  bugprone-macro-repeated-side-effects,
+  bugprone-misplaced-operator-in-strlen-in-alloc,
+  bugprone-misplaced-widening-cast,
+  bugprone-move-forwarding-reference,
+  bugprone-multiple-statement-macro,
+  bugprone-narrowing-conversions,
+  bugprone-parent-virtual-call,
+  bugprone-sizeof-container,
+  bugprone-sizeof-expression,
+  bugprone-string-constructor,
+  bugprone-string-integer-assignment,
+  bugprone-string-literal-with-embedded-nul,
+  bugprone-suspicious-enum-usage,
+  bugprone-suspicious-memset-usage,
+  bugprone-suspicious-missing-comma,
+  bugprone-suspicious-semicolon,
+  bugprone-suspicious-string-compare,
+  bugprone-swapped-arguments,
+  bugprone-terminating-continue,
+  -bugprone-throw-keyword-missing,
+  bugprone-undefined-memory-manipulation,
+  bugprone-undelegated-constructor,
+  bugprone-unused-raii,
+  bugprone-unused-return-value,
+  bugprone-use-after-move,
+  -bugprone-virtual-near-miss,
+  -cert-dcl03-c,
+  -cert-dcl21-cpp,
+  -cert-dcl50-cpp,
+  -cert-dcl54-cpp,
+  -cert-dcl58-cpp,
+  -cert-dcl59-cpp,
+  -cert-env33-c,
+  -cert-err09-cpp,
+  cert-err34-c,
+  cert-err52-cpp,
+  -cert-err58-cpp,
+  cert-err60-cpp,
+  -cert-err61-cpp,
+  -cert-fio38-c,
+  cert-flp30-c,
+  -cert-msc30-c,
+  -cert-msc32-c,
+  -cert-msc50-cpp,
+  -cert-msc51-cpp,
+  -cert-oop11-cpp,
+  cppcoreguidelines-avoid-goto,
+  cppcoreguidelines-c-copy-assignment-signature,
+  cppcoreguidelines-interfaces-global-init,
+  cppcoreguidelines-narrowing-conversions,
+  -cppcoreguidelines-no-malloc,
+  -cppcoreguidelines-owning-memory,
+  -cppcoreguidelines-pro-bounds-array-to-pointer-decay,
+  -cppcoreguidelines-pro-bounds-constant-array-index,
+  -cppcoreguidelines-pro-bounds-pointer-arithmetic,
+  -cppcoreguidelines-pro-type-const-cast,
+  -cppcoreguidelines-pro-type-cstyle-cast,
+  -cppcoreguidelines-pro-type-member-init,
+  -cppcoreguidelines-pro-type-reinterpret-cast,
+  -cppcoreguidelines-pro-type-static-cast-downcast,
+  -cppcoreguidelines-pro-type-union-access,
+  -cppcoreguidelines-pro-type-vararg,
+  -cppcoreguidelines-slicing,
+  -cppcoreguidelines-special-member-functions,
+  -fuchsia-default-arguments,
+  -fuchsia-header-anon-namespaces,
+  -fuchsia-multiple-inheritance,
+  -fuchsia-overloaded-operator,
+  -fuchsia-restrict-system-includes,
+  -fuchsia-statically-constructed-objects,
+  -fuchsia-trailing-return,
+  -fuchsia-virtual-inheritance,
+  google-build-explicit-make-pair,
+  google-build-namespaces,
+  -google-build-using-namespace,
+  -google-default-arguments,
+  -google-explicit-constructor,
+  google-global-names-in-headers,
+  google-objc-avoid-throwing-exception,
+  google-objc-global-variable-declaration,
+  -google-readability-braces-around-statements,
+  -google-readability-casting,
+  google-readability-function-size,
+  google-readability-namespace-comments,
+  -google-readability-todo,
+  -google-runtime-int,
+  google-runtime-operator,
+  -google-runtime-references,
+  hicpp-avoid-goto,
+  -hicpp-braces-around-statements,
+  -hicpp-deprecated-headers,
+  -hicpp-exception-baseclass,
+  -hicpp-explicit-conversions,
+  hicpp-function-size,
+  hicpp-invalid-access-moved,
+  -hicpp-member-init,
+  hicpp-move-const-arg,
+  -hicpp-multiway-paths-covered,
+  hicpp-named-parameter,
+  -hicpp-new-delete-operators,
+  -hicpp-no-array-decay,
+  -hicpp-no-assembler,
+  -hicpp-no-malloc,
+  -hicpp-noexcept-move,
+  -hicpp-signed-bitwise,
+  -hicpp-special-member-functions,
+  hicpp-static-assert,
+  hicpp-undelegated-constructor,
+  -hicpp-use-auto,
+  -hicpp-use-emplace,
+  hicpp-use-equals-default,
+  hicpp-use-equals-delete,
+  hicpp-use-noexcept,
+  -hicpp-use-nullptr,
+  -hicpp-use-override,
+  hicpp-vararg,
+  -llvm-header-guard,
+  -llvm-include-order,
+  -llvm-namespace-comment,
+  -llvm-twine-local,
+  misc-definitions-in-headers,
+  misc-misplaced-const,
+  -misc-new-delete-overloads,
+  misc-non-copyable-objects,
+  misc-redundant-expression,
+  misc-static-assert,
+  misc-throw-by-value-catch-by-reference,
+  misc-unconventional-assign-operator,
+  misc-uniqueptr-reset-release,
+  misc-unused-alias-decls,
+  -misc-unused-parameters,
+  -misc-unused-using-decls,
+  modernize-avoid-bind,
+  -modernize-deprecated-headers,
+  -modernize-loop-convert,
+  modernize-make-shared,
+  -modernize-make-unique,
+  -modernize-pass-by-value,
+  -modernize-raw-string-literal,
+  -modernize-redundant-void-arg,
+  modernize-replace-auto-ptr,
+  modernize-replace-random-shuffle,
+  -modernize-return-braced-init-list,
+  modernize-shrink-to-fit,
+  modernize-unary-static-assert,
+  -modernize-use-auto,
+  modernize-use-bool-literals,
+  -modernize-use-default-member-init,
+  modernize-use-emplace,
+  -modernize-use-equals-default,
+  -modernize-use-equals-delete,
+  modernize-use-noexcept,
+  -modernize-use-nullptr,
+  -modernize-use-override,
+  -modernize-use-transparent-functors,
+  modernize-use-uncaught-exceptions,
+  -modernize-use-using,
+  -mpi-buffer-deref,
+  -mpi-type-mismatch,
+  -objc-avoid-nserror-init,
+  -objc-avoid-spinlock,
+  -objc-forbidden-subclassing,
+  -objc-property-declaration,
+  performance-faster-string-find,
+  performance-for-range-copy,
+  performance-implicit-conversion-in-loop,
+  performance-inefficient-algorithm,
+  performance-inefficient-string-concatenation,
+  performance-inefficient-vector-operation,
+  -performance-move-const-arg,
+  performance-move-constructor-init,
+  -performance-noexcept-move-constructor,
+  performance-type-promotion-in-math-fn,
+  -performance-unnecessary-copy-initialization,
+  performance-unnecessary-value-param,
+  -portability-simd-intrinsics,
+  readability-avoid-const-params-in-decls,
+  -readability-braces-around-statements,
+  -readability-container-size-empty,
+  readability-delete-null-pointer,
+  readability-deleted-default,
+  -readability-else-after-return,
+  -readability-function-size,
+  -readability-identifier-naming,
+  -readability-implicit-bool-conversion,
+  readability-inconsistent-declaration-parameter-name,
+  -readability-misleading-indentation,
+  readability-misplaced-array-index,
+  -readability-named-parameter,
+  readability-non-const-parameter,
+  readability-redundant-control-flow,
+  -readability-redundant-declaration,
+  readability-redundant-function-ptr-dereference,
+  -readability-redundant-member-init,
+  -readability-redundant-smartptr-get,
+  -readability-redundant-string-cstr,
+  -readability-redundant-string-init,
+  -readability-simplify-boolean-expr,
+  readability-simplify-subscript-expr,
+  readability-static-accessed-through-instance,
+  -readability-static-definition-in-anonymous-namespace,
+  readability-string-compare,
+  readability-uniqueptr-delete-release,
+  -zircon-temporary-objects
+WarningsAsErrors: '*'
+HeaderFilterRegex: '(mongo/.*|build/.*)'
+AnalyzeTemporaryDtors: false
+FormatStyle:     none
+User:            ubuntu
+CheckOptions:    
+  - key:             bugprone-argument-comment.StrictMode
+    value:           '0'
+  - key:             bugprone-assert-side-effect.AssertMacros
+    value:           assert
+  - key:             bugprone-assert-side-effect.CheckFunctionCalls
+    value:           '0'
+  - key:             bugprone-dangling-handle.HandleClasses
+    value:           'std::basic_string_view;std::experimental::basic_string_view'
+  - key:             bugprone-exception-escape.FunctionsThatShouldNotThrow
+    value:           ''
+  - key:             bugprone-exception-escape.IgnoredExceptions
+    value:           ''
+  - key:             bugprone-misplaced-widening-cast.CheckImplicitCasts
+    value:           '0'
+  - key:             bugprone-sizeof-expression.WarnOnSizeOfCompareToConstant
+    value:           '1'
+  - key:             bugprone-sizeof-expression.WarnOnSizeOfConstant
+    value:           '1'
+  - key:             bugprone-sizeof-expression.WarnOnSizeOfIntegerExpression
+    value:           '0'
+  - key:             bugprone-sizeof-expression.WarnOnSizeOfThis
+    value:           '1'
+  - key:             bugprone-string-constructor.LargeLengthThreshold
+    value:           '8388608'
+  - key:             bugprone-string-constructor.WarnOnLargeLength
+    value:           '1'
+  - key:             bugprone-suspicious-enum-usage.StrictMode
+    value:           '0'
+  - key:             bugprone-suspicious-missing-comma.MaxConcatenatedTokens
+    value:           '5'
+  - key:             bugprone-suspicious-missing-comma.RatioThreshold
+    value:           '0.200000'
+  - key:             bugprone-suspicious-missing-comma.SizeThreshold
+    value:           '5'
+  - key:             bugprone-suspicious-string-compare.StringCompareLikeFunctions
+    value:           ''
+  - key:             bugprone-suspicious-string-compare.WarnOnImplicitComparison
+    value:           '1'
+  - key:             bugprone-suspicious-string-compare.WarnOnLogicalNotComparison
+    value:           '0'
+  - key:             bugprone-unused-return-value.CheckedFunctions
+    value:           '::std::async;::std::launder;::std::remove;::std::remove_if;::std::unique;::std::unique_ptr::release;::std::basic_string::empty;::std::vector::empty'
+  - key:             cppcoreguidelines-no-malloc.Allocations
+    value:           '::malloc;::calloc'
+  - key:             cppcoreguidelines-no-malloc.Deallocations
+    value:           '::free'
+  - key:             cppcoreguidelines-no-malloc.Reallocations
+    value:           '::realloc'
+  - key:             cppcoreguidelines-pro-bounds-constant-array-index.GslHeader
+    value:           ''
+  - key:             cppcoreguidelines-pro-bounds-constant-array-index.IncludeStyle
+    value:           '0'
+  - key:             cppcoreguidelines-pro-type-member-init.IgnoreArrays
+    value:           '0'
+  - key:             cppcoreguidelines-special-member-functions.AllowMissingMoveFunctions
+    value:           '0'
+  - key:             cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
+    value:           '1'
+  - key:             google-build-namespaces.HeaderFileExtensions
+    value:           ',h,hh,hpp,hxx'
+  - key:             google-global-names-in-headers.HeaderFileExtensions
+    value:           ',h,hh,hpp,hxx'
+  - key:             google-readability-braces-around-statements.ShortStatementLines
+    value:           '1'
+  - key:             google-readability-function-size.BranchThreshold
+    value:           '4294967295'
+  - key:             google-readability-function-size.LineThreshold
+    value:           '4294967295'
+  - key:             google-readability-function-size.NestingThreshold
+    value:           '4294967295'
+  - key:             google-readability-function-size.ParameterThreshold
+    value:           '4294967295'
+  - key:             google-readability-function-size.StatementThreshold
+    value:           '800'
+  - key:             google-readability-function-size.VariableThreshold
+    value:           '4294967295'
+  - key:             google-readability-namespace-comments.ShortNamespaceLines
+    value:           '10'
+  - key:             google-readability-namespace-comments.SpacesBeforeComments
+    value:           '2'
+  - key:             google-runtime-int.SignedTypePrefix
+    value:           int
+  - key:             google-runtime-int.TypeSuffix
+    value:           ''
+  - key:             google-runtime-int.UnsignedTypePrefix
+    value:           uint
+  - key:             hicpp-braces-around-statements.ShortStatementLines
+    value:           '0'
+  - key:             hicpp-function-size.BranchThreshold
+    value:           '4294967295'
+  - key:             hicpp-function-size.LineThreshold
+    value:           '4294967295'
+  - key:             hicpp-function-size.NestingThreshold
+    value:           '4294967295'
+  - key:             hicpp-function-size.ParameterThreshold
+    value:           '4294967295'
+  - key:             hicpp-function-size.StatementThreshold
+    value:           '800'
+  - key:             hicpp-function-size.VariableThreshold
+    value:           '4294967295'
+  - key:             hicpp-member-init.IgnoreArrays
+    value:           '0'
+  - key:             hicpp-move-const-arg.CheckTriviallyCopyableMove
+    value:           '1'
+  - key:             hicpp-multiway-paths-covered.WarnOnMissingElse
+    value:           '0'
+  - key:             hicpp-named-parameter.IgnoreFailedSplit
+    value:           '0'
+  - key:             hicpp-no-malloc.Allocations
+    value:           '::malloc;::calloc'
+  - key:             hicpp-no-malloc.Deallocations
+    value:           '::free'
+  - key:             hicpp-no-malloc.Reallocations
+    value:           '::realloc'
+  - key:             hicpp-special-member-functions.AllowMissingMoveFunctions
+    value:           '0'
+  - key:             hicpp-special-member-functions.AllowSoleDefaultDtor
+    value:           '0'
+  - key:             hicpp-use-auto.MinTypeNameLength
+    value:           '5'
+  - key:             hicpp-use-auto.RemoveStars
+    value:           '0'
+  - key:             hicpp-use-emplace.ContainersWithPushBack
+    value:           '::std::vector;::std::list;::std::deque'
+  - key:             hicpp-use-emplace.SmartPointers
+    value:           '::std::shared_ptr;::std::unique_ptr;::std::auto_ptr;::std::weak_ptr'
+  - key:             hicpp-use-emplace.TupleMakeFunctions
+    value:           '::std::make_pair;::std::make_tuple'
+  - key:             hicpp-use-emplace.TupleTypes
+    value:           '::std::pair;::std::tuple'
+  - key:             hicpp-use-equals-default.IgnoreMacros
+    value:           '1'
+  - key:             hicpp-use-noexcept.ReplacementString
+    value:           ''
+  - key:             hicpp-use-noexcept.UseNoexceptFalse
+    value:           '1'
+  - key:             hicpp-use-nullptr.NullMacros
+    value:           ''
+  - key:             llvm-namespace-comment.ShortNamespaceLines
+    value:           '1'
+  - key:             llvm-namespace-comment.SpacesBeforeComments
+    value:           '1'
+  - key:             misc-definitions-in-headers.HeaderFileExtensions
+    value:           ',h,hh,hpp,hxx'
+  - key:             misc-definitions-in-headers.UseHeaderFileExtension
+    value:           '1'
+  - key:             misc-throw-by-value-catch-by-reference.CheckThrowTemporaries
+    value:           '1'
+  - key:             misc-unused-parameters.StrictMode
+    value:           '0'
+  - key:             modernize-loop-convert.MaxCopySize
+    value:           '16'
+  - key:             modernize-loop-convert.MinConfidence
+    value:           reasonable
+  - key:             modernize-loop-convert.NamingStyle
+    value:           CamelCase
+  - key:             modernize-make-shared.IgnoreMacros
+    value:           '1'
+  - key:             modernize-make-shared.IncludeStyle
+    value:           '0'
+  - key:             modernize-make-shared.MakeSmartPtrFunction
+    value:           'std::make_shared'
+  - key:             modernize-make-shared.MakeSmartPtrFunctionHeader
+    value:           memory
+  - key:             modernize-make-unique.IgnoreMacros
+    value:           '1'
+  - key:             modernize-make-unique.IncludeStyle
+    value:           '0'
+  - key:             modernize-make-unique.MakeSmartPtrFunction
+    value:           'std::make_unique'
+  - key:             modernize-make-unique.MakeSmartPtrFunctionHeader
+    value:           memory
+  - key:             modernize-pass-by-value.IncludeStyle
+    value:           llvm
+  - key:             modernize-pass-by-value.ValuesOnly
+    value:           '0'
+  - key:             modernize-raw-string-literal.ReplaceShorterLiterals
+    value:           '0'
+  - key:             modernize-replace-auto-ptr.IncludeStyle
+    value:           llvm
+  - key:             modernize-replace-random-shuffle.IncludeStyle
+    value:           llvm
+  - key:             modernize-use-auto.MinTypeNameLength
+    value:           '5'
+  - key:             modernize-use-auto.RemoveStars
+    value:           '0'
+  - key:             modernize-use-default-member-init.IgnoreMacros
+    value:           '1'
+  - key:             modernize-use-default-member-init.UseAssignment
+    value:           '0'
+  - key:             modernize-use-emplace.ContainersWithPushBack
+    value:           '::std::vector;::std::list;::std::deque'
+  - key:             modernize-use-emplace.SmartPointers
+    value:           '::std::shared_ptr;::std::unique_ptr;::std::auto_ptr;::std::weak_ptr'
+  - key:             modernize-use-emplace.TupleMakeFunctions
+    value:           '::std::make_pair;::std::make_tuple'
+  - key:             modernize-use-emplace.TupleTypes
+    value:           '::std::pair;::std::tuple'
+  - key:             modernize-use-equals-default.IgnoreMacros
+    value:           '1'
+  - key:             modernize-use-noexcept.ReplacementString
+    value:           ''
+  - key:             modernize-use-noexcept.UseNoexceptFalse
+    value:           '1'
+  - key:             modernize-use-nullptr.NullMacros
+    value:           'NULL'
+  - key:             modernize-use-transparent-functors.SafeMode
+    value:           '0'
+  - key:             modernize-use-using.IgnoreMacros
+    value:           '1'
+  - key:             performance-faster-string-find.StringLikeClasses
+    value:           'std::basic_string'
+  - key:             performance-for-range-copy.WarnOnAllAutoCopies
+    value:           '0'
+  - key:             performance-inefficient-string-concatenation.StrictMode
+    value:           '0'
+  - key:             performance-inefficient-vector-operation.VectorLikeClasses
+    value:           '::std::vector'
+  - key:             performance-move-const-arg.CheckTriviallyCopyableMove
+    value:           '1'
+  - key:             performance-move-constructor-init.IncludeStyle
+    value:           llvm
+  - key:             performance-type-promotion-in-math-fn.IncludeStyle
+    value:           llvm
+  - key:             performance-unnecessary-value-param.IncludeStyle
+    value:           llvm
+  - key:             readability-braces-around-statements.ShortStatementLines
+    value:           '0'
+  - key:             readability-implicit-bool-conversion.AllowIntegerConditions
+    value:           '0'
+  - key:             readability-implicit-bool-conversion.AllowPointerConditions
+    value:           '0'
+  - key:             readability-inconsistent-declaration-parameter-name.IgnoreMacros
+    value:           '1'
+  - key:             readability-inconsistent-declaration-parameter-name.Strict
+    value:           '0'
+  - key:             readability-simplify-boolean-expr.ChainedConditionalAssignment
+    value:           '0'
+  - key:             readability-simplify-boolean-expr.ChainedConditionalReturn
+    value:           '0'
+  - key:             readability-simplify-subscript-expr.Types
+    value:           '::std::basic_string;::std::basic_string_view;::std::vector;::std::array'
+  - key:             readability-static-accessed-through-instance.NameSpecifierNestingThreshold
+    value:           '3'
+...
+
diff --git a/buildscripts/clang_tidy_condensed.py b/buildscripts/clang_tidy_condensed.py
new file mode 100755
index 00000000000..995ae5f3292
--- /dev/null
+++ b/buildscripts/clang_tidy_condensed.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Runs clang-tidy in parallel and combines the the results for easier viewing."""
+
+import argparse
+import datetime
+import json
+import os
+import subprocess
+import sys
+import threading
+import queue
+import time
+from typing import Any, Dict, List, Optional
+import multiprocessing
+from pathlib import Path
+
+import yaml
+
+files_to_tidy = queue.SimpleQueue()
+files_to_parse = queue.SimpleQueue()
+
+
+def _clang_tidy_executor(clang_tidy_binary: str, clang_tidy_cfg: Dict[str, Any], output_dir: str):
+    while True:
+        clang_tidy_filename: Optional[Path] = files_to_tidy.get()
+        if clang_tidy_filename is None:
+            files_to_parse.put(None)
+            files_to_tidy.put(None)
+            break
+
+        print(f"Running clang-tidy on {clang_tidy_filename}")
+        clang_tidy_parent_dir = output_dir / clang_tidy_filename.parent
+        os.makedirs(clang_tidy_parent_dir, exist_ok=True)
+
+        output_filename_base = clang_tidy_parent_dir / clang_tidy_filename.name
+        output_filename_fixes = output_filename_base.with_suffix(".yml")
+        clang_tidy_command = [
+            clang_tidy_binary, clang_tidy_filename, f"-export-fixes={output_filename_fixes}",
+            f"-config={json.dumps(clang_tidy_cfg)}"
+        ]
+        proc = subprocess.run(clang_tidy_command, capture_output=True, check=False)
+        if proc.returncode != 0:
+            output_filename_out = output_filename_base.with_suffix(".fail")
+            files_to_parse.put(output_filename_fixes)
+            print(
+                f"Running clang-tidy on {clang_tidy_filename} had errors see {output_filename_out}")
+        else:
+            output_filename_out = output_filename_base.with_suffix(".pass")
+            print(f"Running clang-tidy on {clang_tidy_filename} had no errors")
+
+        with open(output_filename_out, 'wb') as output:
+            output.write(proc.stderr)
+            output.write(proc.stdout)
+
+
+def _combine_errors(clang_tidy_executors: int, fixes_filename: str) -> int:
+    failed_files = 0
+    all_fixes = {}
+    while clang_tidy_executors > 0:
+        item = files_to_parse.get()
+
+        # Once all running threads say they are done we want to exit
+        if item is None:
+            clang_tidy_executors -= 1
+            continue
+
+        failed_files += 1
+
+        # Read the yaml fixes for the file to combine them with the other suggested fixes
+        with open(item) as input_yml:
+            fixes = yaml.safe_load(input_yml)
+        for fix in fixes['Diagnostics']:
+            fix_data = all_fixes.setdefault(fix["DiagnosticName"], {}).setdefault(
+                fix["FilePath"], {}).setdefault(
+                    fix["FileOffset"], {
+                        "replacements": fix["Replacements"], "message": fix["Message"], "count": 0,
+                        "source_files": []
+                    })
+            fix_data["count"] += 1
+            fix_data["source_files"].append(fixes['MainSourceFile'])
+    with open(fixes_filename, "w") as files_file:
+        json.dump(all_fixes, files_file, indent=4, sort_keys=True)
+
+    return failed_files
+
+
+def _report_status(total_jobs: int, clang_tidy_executor_threads: List[threading.Thread]):
+    start_time = time.time()
+    running_jobs = 1
+    while running_jobs > 0:
+        time.sleep(5)
+        pretty_time_duration = str(datetime.timedelta(seconds=time.time() - start_time))
+        running_jobs = sum(
+            [1 for t in clang_tidy_executor_threads if t.is_alive()])  # Count threads running a job
+        # files_to_tidy contains a None which can be ignored
+        print(
+            f"There are {running_jobs} active jobs. The number of jobs queued is {files_to_tidy.qsize()-1}/{total_jobs}. Duration {pretty_time_duration}."
+        )
+
+
+def main():
+    """Execute Main entry point."""
+
+    parser = argparse.ArgumentParser(description='Run multithreaded clang-tidy')
+
+    parser.add_argument('-t', "--threads", type=int, default=multiprocessing.cpu_count(),
+                        help="Run with a specific number of threads")
+    parser.add_argument("-d", "--output-dir", type=str, default="clang_tidy_fixes",
+                        help="Directory to write all clang-tidy output to")
+    parser.add_argument("-o", "--fixes-file", type=str, default="clang_tidy_fixes.json",
+                        help="Report json file to write combined fixes to")
+    parser.add_argument("-c", "--compile-commands", type=str, default="compile_commands.json",
+                        help="compile_commands.json file to use to find the files to tidy")
+    # TODO: Is there someway to get this without hardcoding this much
+    parser.add_argument("-y", "--clang-tidy-toolchain", type=str, default="v3")
+    parser.add_argument("-f", "--clang-tidy-cfg", type=str, default=".clang-tidy")
+    args = parser.parse_args()
+
+    clang_tidy_binary = f'/opt/mongodbtoolchain/{args.clang_tidy_toolchain}/bin/clang-tidy'
+
+    with open(args.compile_commands) as compile_commands:
+        compile_commands = json.load(compile_commands)
+
+    with open(args.clang_tidy_cfg) as clang_tidy_cfg:
+        clang_tidy_cfg = yaml.safe_load(clang_tidy_cfg)
+
+    for file_doc in compile_commands:
+        # A few special cases of files to ignore
+        if not "src/mongo" in file_doc["file"]:
+            continue
+        # TODO SERVER-49884 Remove this when we no longer check in generated Bison.
+        if "parser_gen.cpp" in file_doc["file"]:
+            continue
+        files_to_tidy.put(Path(file_doc["file"]))
+
+    total_jobs = files_to_tidy.qsize()
+    files_to_tidy.put(None)
+    workers = args.threads
+
+    clang_tidy_executor_threads: List[threading.Thread] = []
+    for _ in range(workers):
+        clang_tidy_executor_threads.append(
+            threading.Thread(target=_clang_tidy_executor, args=(clang_tidy_binary, clang_tidy_cfg,
+                                                                args.output_dir)))
+        clang_tidy_executor_threads[-1].start()
+
+    report_status_thread = threading.Thread(target=_report_status,
+                                            args=(total_jobs, clang_tidy_executor_threads))
+    report_status_thread.start()
+
+    failed_files = _combine_errors(workers, Path(args.output_dir, args.fixes_file))
+
+    # Join all threads
+    report_status_thread.join()
+    for thread in clang_tidy_executor_threads:
+        thread.join()
+
+    # Zip up all the files for upload
+    subprocess.run(["tar", "-czvf", args.output_dir + ".tgz", args.output_dir], check=False)
+
+    return failed_files
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 3e551e0b611..e8922fc16b3 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -2081,48 +2081,37 @@ buildvariants:
     - rhel72-zseries-build
   - name: generate_buildid_to_debug_symbols_mapping
 
-
-- name: enterprise-rhel80-dynamic-clang-tidy-required
-  display_name: "! Enterprise Clang Tidy"
-  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
+- &enterprise-rhel-80-64-dynamic-clang-tidy-condensed
+  name: enterprise-rhel80-dynamic-clang-tidy-condensed
+  display_name: "Enterprise Clang Tidy With Extra Checks"
+  cron: "0 4 * * *"
   modules:
   - enterprise
   run_on:
-  - rhel80-xlarge
+  - rhel80-xxlarge
   expansions:
-    additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
     lang_environment: LANG=C
-    compile_flags: --link-model=dynamic -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars
-    # Unlike static builds, dynamic builds have no need to
-    # constrain the number of link jobs. Unfortunately, --jlink=1
-    # means one link job, not 100%. So this is a bit gross but set
-    # it to .99.
-    num_scons_link_jobs_available: 0.99
+    compile_flags: --link-model=dynamic -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_${clang_tidy_toolchain}_clang.vars
     scons_cache_scope: shared
     scons_cache_mode: all
     show_scons_timings: false
     clang_tidy_toolchain: v3
+    clang_tidy_file: .clang-tidy-extra-checks
+  tasks:
+  - name: clang_tidy_condensed_TG
+
+- <<: *enterprise-rhel-80-64-dynamic-clang-tidy-condensed
+  name: enterprise-rhel80-dynamic-clang-tidy-required
+  display_name: "! Enterprise Clang Tidy"
+  cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
   tasks:
   - name: clang_tidy_TG
 
-- name: enterprise-rhel80-dynamic-clang-tidy-v4
+- <<: *enterprise-rhel-80-64-dynamic-clang-tidy-condensed
+  name: enterprise-rhel80-dynamic-clang-tidy-v4
   display_name: "Enterprise Clang Tidy v4 Toolchain"
   cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter
-  modules:
-  - enterprise
-  run_on:
-  - rhel80-xlarge
   expansions:
-    lang_environment: LANG=C
-    compile_flags: --link-model=dynamic -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_clang.vars
-    # Unlike static builds, dynamic builds have no need to
-    # constrain the number of link jobs. Unfortunately, --jlink=1
-    # means one link job, not 100%. So this is a bit gross but set
-    # it to .99.
-    num_scons_link_jobs_available: 0.99
-    scons_cache_scope: shared
-    scons_cache_mode: all
-    show_scons_timings: false
     clang_tidy_toolchain: v4
   tasks:
   - name: clang_tidy_TG
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml
index c025d5696f2..8a41588b3c2 100644
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@@ -791,6 +791,28 @@ functions:
       content_type: text/plain
       display_name: Pip Requirements
 
+  "upload clang tidy results": &upload_clang_tidy_results
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/clang_tidy_fixes.tgz
+        remote_file: ${project}/${build_variant}/${revision}/${task_id}/clang_tidy_fixes.tgz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/gzip
+        display_name: clang_tidy_fixes.tgz
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: src/clang_tidy_fixes/clang_tidy_fixes.json
+        remote_file: ${project}/${build_variant}/${revision}/${task_id}/clang_tidy_fixes.json
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/json
+        display_name: clang_tidy_fixes.json
+
   "send benchmark results":
   - command: json.send
     params:
@@ -2440,7 +2462,7 @@ tasks:
   commands:
   - func: "scons compile"
     vars:
-      targets: generated-sources compiledb
+      targets: compiledb
       compiling_for_test: true
   - command: subprocess.exec
     type: test
@@ -2450,6 +2472,26 @@ tasks:
       args:
       - "./src/evergreen/run_clang_tidy.sh"
 
+## clang_tidy_condensed - run clang_tidy with python making the results clearer
+- name: clang_tidy_condensed
+  tags: []
+  exec_timeout_secs: 7200 # 1 hour timeout for the task overall
+  depends_on:
+  - name: version_expansions_gen
+    variant: generate-tasks-for-version
+  commands:
+  - func: "scons compile"
+    vars:
+      targets: compiledb
+      compiling_for_test: true
+  - command: subprocess.exec
+    type: test
+    timeout_secs: 7000 # 1 hour timeout for no output
+    params:
+      binary: bash
+      args:
+      - "./src/evergreen/run_clang_tidy_condensed.sh"
+
 ## compile_unittests ##
 - &compile_unittests
   name: compile_unittests
@@ -7614,6 +7656,39 @@ task_groups:
   tasks:
   - clang_tidy
 
+# This is a copy of clang_tidy_TG and will hopefully replace it
+- name: clang_tidy_condensed_TG
+  setup_group_can_fail_task: true
+  setup_group:
+  - command: manifest.load
+  - func: "git get project and add git tag"
+  - func: "set task expansion macros"
+  - func: "f_expansions_write"
+  - func: "kill processes"
+  - func: "cleanup environment"
+  - func: "set up venv"
+  - func: "upload pip requirements"
+  - func: "configure evergreen api credentials"
+  - func: "get buildnumber"
+  - func: "f_expansions_write"
+  - func: "set up credentials"
+  - func: "set up win mount script"
+  - func: "generate compile expansions"
+  teardown_group:
+  - func: "f_expansions_write"
+  - func: "attach report"
+  - func: "upload clang tidy results"
+  - func: "umount shared scons directory"
+  - func: "cleanup environment"
+  setup_task:
+  - func: "apply compile expansions"
+  - func: "f_expansions_write"
+  - func: "set task expansion macros"
+  - func: "f_expansions_write"
+  teardown_task:
+  tasks:
+  - clang_tidy_condensed
+
 - name: visibility_test_TG
   setup_group_can_fail_task: true
   max_hosts: 1
diff --git a/evergreen/run_clang_tidy_condensed.sh b/evergreen/run_clang_tidy_condensed.sh
new file mode 100755
index 00000000000..477a4c66680
--- /dev/null
+++ b/evergreen/run_clang_tidy_condensed.sh
@@ -0,0 +1,10 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+set -o pipefail
+set -o verbose
+
+cd src
+activate_venv
+$python buildscripts/clang_tidy_condensed.py --clang-tidy-cfg ${clang_tidy_file} --clang-tidy-toolchain ${clang_tidy_toolchain}
+exit $?