SERVER-60832 Readd and fix gcov code coverage variant

6 files changed, 127 insertions, 12 deletions

diff --git a/SConstruct b/SConstruct
index 446b3dea8e0..150ae43b4b5 100644
--- a/SConstruct
+++ b/SConstruct
@@ -2515,10 +2515,14 @@ if env.TargetOSIs('posix'):
     if has_option( "gcov" ):
         if not (env.TargetOSIs('linux') and env.ToolchainIs('gcc')):
             # TODO: This should become supported under: https://jira.mongodb.org/browse/SERVER-49877
-            env.FatalError("Coverage option 'gcov' is currently only supported on linux with gcc. See SERVER-49877.")
+            # TODO SERVER-63055: clang is not supported due to consistent
+            # failures in ValidateCollections when using gcov.
+            env.FatalError("Coverage option 'gcov' is currently only supported on linux with gcc. See SERVER-49877 and SERVER-63055.")
 
-        env.Append( CCFLAGS=["-fprofile-arcs", "-ftest-coverage", "-fprofile-update=single"] )
-        env.Append( LINKFLAGS=["-fprofile-arcs", "-ftest-coverage", "-fprofile-update=single"] )
+        env.AppendUnique(
+            CCFLAGS=['--coverage'],
+            LINKFLAGS=['--coverage'],
+        )
 
     if optBuild and not optBuildForSize:
         env.Append( CCFLAGS=["-O3" if "O3" in selected_experimental_optimizations else "-O2"] )
@@ -2936,6 +2940,9 @@ def doConfigure(myenv):
             if not AddToLINKFLAGSIfSupported(myenv, f'-fuse-ld={linker_ld}'):
                 myenv.FatalError(f"Linker {linker_ld} could not be configured.")
 
+        if has_option('gcov') and AddToCCFLAGSIfSupported(myenv, '-fprofile-update=single'):
+            myenv.AppendUnique(LINKFLAGS=['-fprofile-update=single'])
+
     detectCompiler = Configure(myenv, help=False, custom_tests = {
         'CheckForCXXLink': CheckForCXXLink,
     })
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index d57f4e131ea..81d3978f5f7 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -8265,6 +8265,106 @@ buildvariants:
     <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions
     compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20
 
+- name: enterprise-rhel-80-64-bit-coverage
+  display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage (v3 gcc)"
+  modules:
+    - enterprise
+  run_on:
+    - rhel80-medium
+  cron: "0 0 * * 0" # Every week starting 00:00 on Sunday
+  stepback: false
+  expansions:
+    additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
+    test_flags: --excludeWithAnyTags=resource_intensive --excludeWithAnyTags=incompatible_with_gcov
+    compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    large_distro_name: rhel80-medium
+    multiversion_platform: rhel80
+    multiversion_edition: enterprise
+    resmoke_jobs_factor: 0.5  # Avoid starting too many mongod's
+    # The gcov instrumentation saves the path the .gcno files were created in as the default path
+    # for the .gcda files. In Evergreen the path will start with /data/mci/[Hashed ID]/src/... where
+    # the hashed ID is unique per task run. GCOV_PREFIX_STRIP is the number of directory levels to
+    # strip from the top of the default path before appending to the GCOV_PREFIX (if any).
+    gcov_environment: GCOV_PREFIX=$(pwd) GCOV_PREFIX_STRIP=4
+    # Mixing --cache and --gcov doesn't work correctly yet. See SERVER-11084
+    exec_timeout_secs: 16200 # 4.5 hour timeout
+    timeout_secs: 9000 # 2.5 hour idle timeout
+    use_scons_cache: false
+  tasks: &enterprise-rhel-80-64-bit-coverage-tasks
+    - name: compile_test_and_package_serial_TG
+      distros:
+      - rhel80-large
+    - name: build_variant_gen
+#    - name: .aggfuzzer # Disabled for FCV 5.3
+    - name: .aggregation !.unwind
+    - name: audit
+    - name: .auth
+    - name: causally_consistent_jscore_txns_passthrough
+    - name: .change_streams
+    - name: .misc_js # serial_run needs extra memory
+      distros:
+      - rhel80-large
+    - name: .concurrency !.ubsan !.no_txns !.stepdowns !.kill_terminate !.disabled_on_code_coverage
+    - name: disk_wiredtiger
+    - name: .encrypt
+    - name: initial_sync_fuzzer_gen
+    - name: .integration !.audit
+    - name: .jscore .common
+    - name: jsCore_txns_large_txns_format
+    - name: jsCore_minimum_batch_size
+    - name: libunwind_tests
+    - name: .logical_session_cache .one_sec
+    - name: .multi_shard .common
+    - name: multiversion_gen
+    - name: .multiversion_fuzzer
+    - name: .multiversion_passthrough
+    - name: .query_fuzzer
+    - name: .random_multiversion_ds
+    - name: .read_write_concern
+    - name: .replica_sets !.ignore_non_generated_replica_sets_jscore_passthrough
+    - name: replica_sets_jscore_passthrough_gen
+    - name: .read_only
+    - name: .rollbackfuzzer
+    - name: retryable_writes_jscore_passthrough_gen
+    - name: sasl
+    - name: search
+    - name: search_auth
+    - name: search_ssl
+    - name: secondary_reads_passthrough_gen
+    - name: session_jscore_passthrough
+    - name: .sharding .jscore !.wo_snapshot
+    - name: .sharding .common
+    - name: snmp
+    - name: update_fuzzer_gen
+
+- name: enterprise-rhel-80-64-bit-coverage-v4-gcc
+  display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage (v4 gcc)"
+  modules:
+    - enterprise
+  run_on:
+    - rhel80-medium
+  cron: "0 0 * * 0" # Every week starting 00:00 on Sunday
+  stepback: false
+  expansions:
+    additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
+    test_flags: --excludeWithAnyTags=resource_intensive --excludeWithAnyTags=incompatible_with_gcov
+    compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_gcc.vars
+    large_distro_name: rhel80-medium
+    multiversion_platform: rhel80
+    multiversion_edition: enterprise
+    resmoke_jobs_factor: 0.5  # Avoid starting too many mongod's
+    # The gcov instrumentation saves the path the .gcno files were created in as the default path
+    # for the .gcda files. In Evergreen the path will start with /data/mci/[Hashed ID]/src/... where
+    # the hashed ID is unique per task run. GCOV_PREFIX_STRIP is the number of directory levels to
+    # strip from the top of the default path before appending to the GCOV_PREFIX (if any).
+    gcov_environment: GCOV_PREFIX=$(pwd) GCOV_PREFIX_STRIP=4
+    # Mixing --cache and --gcov doesn't work correctly yet. See SERVER-11084
+    exec_timeout_secs: 32400 # 9 hour timeout
+    timeout_secs: 18000 # 5 hour idle timeout
+    use_scons_cache: false
+  tasks:
+    *enterprise-rhel-80-64-bit-coverage-tasks
+
 - name: enterprise-rhel-80-64-bit-bson-column
   display_name: "~ Shared Library Enterprise RHEL 8.0 (BSONColumn validate)"
   modules:
diff --git a/evergreen/functions/code_coverage_data_process.sh b/evergreen/functions/code_coverage_data_process.sh
index c23114884ff..a3af4a9f825 100755
--- a/evergreen/functions/code_coverage_data_process.sh
+++ b/evergreen/functions/code_coverage_data_process.sh
@@ -6,25 +6,25 @@ if [ -d "./build" ]; then
   if [ -n "$file_list" ]; then
     for gcda_file in $file_list; do
       echo "Processing file $gcda_file"
-      /opt/mongodbtoolchain/v3/bin/gcov -i $gcda_file
-      base_name=$(echo $gcda_file | rev | cut -f1 -d '/' | cut -f2 -d '.' | rev)
+      /opt/mongodbtoolchain/v3/bin/gcov -i "$gcda_file"
+      base_name=$(echo "$gcda_file" | rev | cut -f1 -d '/' | cut -f2 -d '.' | rev)
       gcov_file=$base_name.gcda.gcov
       if [ -f "$gcov_file" ]; then
         # Add a prefix to the intermediate file, since it does not have a unique name.
         # Convert the '/' to '#' in the file path.
-        file_prefix=$(echo $gcda_file | sed -e 's,^\./,,' | rev | cut -f2- -d '/' | rev | tr -s '/' '#')
-        new_gcov_file=$file_prefix #$base_name.gcda.gcov
-        if [ ! -f $new_gcov_file ]; then
+        file_prefix=$(echo "$gcda_file" | sed -e 's,^\./,,' | rev | cut -f2- -d '/' | rev | tr -s '/' '#')
+        new_gcov_file="$file_prefix #$base_name.gcda.gcov"
+        if [ ! -f "$new_gcov_file" ]; then
           echo "Renaming gcov intermediate file $gcov_file to $new_gcov_file"
-          mv $gcov_file $new_gcov_file
+          mv "$gcov_file" "$new_gcov_file"
         else
           # We treat this as a fatal condition and remove all of the coverage files.
           echo "Not renaming $gcov_file as $new_gcov_file already exists!"
-          rm -f *.gcda.gcov
+          rm -f ./*.gcda.gcov
           exit 1
         fi
       fi
-      rm $gcda_file
+      rm "$gcda_file"
     done
   fi
 fi
diff --git a/jstests/concurrency/fsm_workloads/agg_merge_when_not_matched_insert.js b/jstests/concurrency/fsm_workloads/agg_merge_when_not_matched_insert.js
index 627f60df143..477b88a7575 100644
--- a/jstests/concurrency/fsm_workloads/agg_merge_when_not_matched_insert.js
+++ b/jstests/concurrency/fsm_workloads/agg_merge_when_not_matched_insert.js
@@ -10,6 +10,7 @@
  *  requires_sharding,
  *  assumes_balancer_off,
  *  requires_non_retryable_writes,
+ *  incompatible_with_gcov,
  *]
  */
 load('jstests/concurrency/fsm_libs/extend_workload.js');                 // for extendWorkload
diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 4e4ec6aad5f..82e6e06b67c 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -106,7 +106,7 @@ if has_option('gcov'):
             'MONGO_GCOV',
         ],
     )
-quick_exit_obj = baseEnv.LibraryObject(
+quick_exit_obj = quick_exit_env.LibraryObject(
     target='quick_exit',
     source=[
         'util/quick_exit.cpp',
diff --git a/src/mongo/util/quick_exit.cpp b/src/mongo/util/quick_exit.cpp
index caf113da50a..b934067fd8c 100644
--- a/src/mongo/util/quick_exit.cpp
+++ b/src/mongo/util/quick_exit.cpp
@@ -65,6 +65,8 @@
 
 #ifdef MONGO_GCOV
 extern "C" void __gcov_flush();
+extern "C" void __gcov_dump();
+extern "C" void __gcov_reset();
 #endif
 
 namespace mongo {
@@ -80,8 +82,13 @@ void quickExitWithoutLogging(int code) {
         quickExitMutex->lock();
 
 #ifdef MONGO_GCOV
+#if (defined(__clang__) && __clang_major__ >= 12) || __GNUC__ >= 11
+    __gcov_dump();
+    __gcov_reset();
+#else
     __gcov_flush();
 #endif
+#endif
 
 #if __has_feature(address_sanitizer)
     // Always dump coverage data first because older versions of sanitizers may not write coverage