SERVER-65684 Fix stack usage with ASAN and gcov

7 files changed, 96 insertions, 50 deletions

diff --git a/SConstruct b/SConstruct
index 93d2e562c7d..7637ab4cabb 100755
--- a/SConstruct
+++ b/SConstruct
@@ -338,8 +338,9 @@ add_option(
 
 add_option(
     'opt',
-    choices=['on', 'size', 'off'],
+    choices=['on', 'debug', 'size', 'off', 'auto'],
     const='on',
+    default='auto',
     help='Enable compile-time optimization',
     nargs='?',
     type='choice',
@@ -1470,26 +1471,6 @@ boostLibs = ["filesystem", "program_options", "system", "iostreams", "thread", "
 onlyServer = len(COMMAND_LINE_TARGETS) == 0 or (len(COMMAND_LINE_TARGETS) == 1 and str(
     COMMAND_LINE_TARGETS[0]) in ["mongod", "mongos", "test"])
 
-releaseBuild = has_option("release")
-
-dbg_opt_mapping = {
-    # --dbg, --opt   :   dbg    opt
-    ("on", None): (True, False),  # special case interaction
-    ("on", "on"): (True, True),
-    ("on", "off"): (True, False),
-    ("off", None): (False, True),
-    ("off", "on"): (False, True),
-    ("off", "off"): (False, False),
-    ("on", "size"): (True, True),
-    ("off", "size"): (False, True),
-}
-debugBuild, optBuild = dbg_opt_mapping[(get_option('dbg'), get_option('opt'))]
-optBuildForSize = True if optBuild and get_option('opt') == "size" else False
-
-if releaseBuild and (debugBuild or not optBuild):
-    print("Error: A --release build may not have debugging, and must have optimization")
-    Exit(1)
-
 noshell = has_option("noshell")
 
 jsEngine = get_option("js-engine")
@@ -1839,6 +1820,31 @@ def is_toolchain(self, *args):
 env.AddMethod(get_toolchain_name, 'ToolchainName')
 env.AddMethod(is_toolchain, 'ToolchainIs')
 
+releaseBuild = has_option("release")
+optBuild = get_option('opt')
+debugBuild = get_option('dbg') == "on"
+
+if env.ToolchainIs('clang'):
+    # LLVM utilizes the stack extensively without optimization enabled, which
+    # causes the built product to easily blow through our 1M stack size whenever
+    # either gcov or sanitizers are enabled. Ref: SERVER-65684
+    if has_option('gcov') and optBuild not in ("on", "debug"):
+        env.FatalError(("Error: A clang --gcov build must have either debug or full "),
+                       ("optimization to prevent crashes due to excessive stack usage"))
+
+    if has_option('sanitize') and optBuild not in ("on", "debug"):
+        env.FatalError(("Error: A clang --sanitize build must have either debug or "),
+                       ("full optimization to prevent crashes due to excessive stack usage"))
+
+# Special cases - if debug is not enabled and optimization is not specified,
+# default to full optimizationm otherwise turn it off.
+if optBuild == "auto":
+    optBuild = "on" if not debugBuild else "off"
+
+if releaseBuild and (debugBuild or optBuild != "on"):
+    env.FatalError(
+        "Error: A --release build may not have debugging, and must have full optimization")
+
 if env['TARGET_ARCH']:
     if not detectSystem.CheckForProcessor(env['TARGET_ARCH']):
         env.ConfError("Could not detect processor specified in TARGET_ARCH variable")
@@ -2154,7 +2160,7 @@ if link_model.startswith("dynamic"):
 
                 env['LIBDEPS_TAG_EXPANSIONS'].append(libdeps_tags_expand_incomplete)
 
-if optBuild:
+if optBuild != "off":
     env.SetConfigHeaderDefine("MONGO_CONFIG_OPTIMIZED_BUILD")
 
 # Enable the fast decider if explicitly requested or if in 'auto' mode
@@ -2667,23 +2673,30 @@ elif env.TargetOSIs('windows'):
     # /MDd: Defines _DEBUG, _MT, _DLL, and uses MSVCRTD.lib/MSVCRD###.DLL
     env.Append(CCFLAGS=["/MDd" if debugBuild else "/MD"])
 
-    if optBuild:
+    if optBuild == "off":
+        env.Append(CCFLAGS=["/Od"])
+        if debugBuild:
+            # /RTC1: - Enable Stack Frame Run-Time Error Checking; Reports when a variable is used
+            # without having been initialized (implies /Od: no optimizations)
+            env.Append(CCFLAGS=["/RTC1"])
+    else:
         # /O1:  optimize for size
         # /O2:  optimize for speed (as opposed to size)
         # /Oy-: disable frame pointer optimization (overrides /O2, only affects 32-bit)
         # /INCREMENTAL: NO - disable incremental link - avoid the level of indirection for function
         # calls
 
-        optStr = "/O2" if not optBuildForSize else "/O1"
-        env.Append(CCFLAGS=[optStr, "/Oy-"])
-        env.Append(LINKFLAGS=["/INCREMENTAL:NO"])
-    else:
-        env.Append(CCFLAGS=["/Od"])
+        optFlags = []
+        if optBuild == "size":
+            optFlags += ["/Os"]
+        elif optBuild == "debug":
+            optFlags += ["/Ox", "/Zo"]
+        else:
+            optFlags += ["/O2"]
+        optFlags += ["/Oy-"]
 
-    if debugBuild and not optBuild:
-        # /RTC1: - Enable Stack Frame Run-Time Error Checking; Reports when a variable is used
-        # without having been initialized (implies /Od: no optimizations)
-        env.Append(CCFLAGS=["/RTC1"])
+        env.Append(CCFLAGS=optFlags)
+        env.Append(LINKFLAGS=["/INCREMENTAL:NO"])
 
     # Support large object files since some unit-test sources contain a lot of code
     env.Append(CCFLAGS=["/bigobj"])
@@ -2919,15 +2932,21 @@ if env.TargetOSIs('posix'):
             LINKFLAGS=['--coverage'],
         )
 
-    if optBuild and not optBuildForSize:
-        env.Append(CCFLAGS=["-O3" if "O3" in selected_experimental_optimizations else "-O2"])
-    elif optBuild and optBuildForSize:
-        env.Append(CCFLAGS=["-Os"])
-    else:
+    if optBuild == "off":
         env.Append(CCFLAGS=["-O0"])
+    else:
+        if optBuild == "size":
+            env.Append(CCFLAGS=["-Os"])
+        elif optBuild == "debug":
+            env.Append(CCFLAGS=["-Og"])
+        else:
+            if "O3" in selected_experimental_optimizations:
+                env.Append(CCFLAGS=["-O3"])
+            else:
+                env.Append(CCFLAGS=["-O2"])
 
-    if optBuild and "treevec" in selected_experimental_optimizations:
-        env.Append(CCFLAGS=["-ftree-vectorize"])
+        if "treevec" in selected_experimental_optimizations:
+            env.Append(CCFLAGS=["-ftree-vectorize"])
 
 wiredtiger = False
 if get_option('wiredtiger') == 'on':
@@ -4224,7 +4243,7 @@ def doConfigure(myenv):
                 if AddToCCFLAGSIfSupported(myenv, "-fno-sanitize=vptr"):
                     myenv.AppendUnique(LINKFLAGS=["-fno-sanitize=vptr"])
 
-    if myenv.ToolchainIs('msvc') and optBuild:
+    if myenv.ToolchainIs('msvc') and optBuild != "off":
         # http://blogs.msdn.com/b/vcblog/archive/2013/09/11/introducing-gw-compiler-switch.aspx
         #
         myenv.Append(CCFLAGS=["/Gw", "/Gy"])
@@ -4263,7 +4282,7 @@ def doConfigure(myenv):
             if myenv.ToolchainIs('clang') and usingLibStdCxx:
                 env.FatalError(
                     'The --detect-odr-violations flag does not work with clang and libstdc++')
-            if optBuild:
+            if optBuild != "off":
                 env.FatalError(
                     'The --detect-odr-violations flag is expected to only be reliable with --opt=off'
                 )
@@ -4364,7 +4383,7 @@ def doConfigure(myenv):
             AddToCCFLAGSIfSupported(myenv, "-fno-semantic-interposition")
 
     # Avoid deduping symbols on OS X debug builds, as it takes a long time.
-    if not optBuild and myenv.ToolchainIs('clang') and env.TargetOSIs('darwin'):
+    if optBuild == "off" and myenv.ToolchainIs('clang') and env.TargetOSIs('darwin'):
         AddToLINKFLAGSIfSupported(myenv, "-Wl,-no_deduplicate")
 
     # Apply any link time optimization settings as selected by the 'lto' option.
@@ -4393,7 +4412,7 @@ def doConfigure(myenv):
         else:
             myenv.ConfError("Don't know how to enable --lto on current toolchain")
 
-    if get_option('runtime-hardening') == "on" and optBuild:
+    if get_option('runtime-hardening') == "on" and optBuild != "off":
         # Older glibc doesn't work well with _FORTIFY_SOURCE=2. Selecting 2.11 as the minimum was an
         # emperical decision, as that is the oldest non-broken glibc we seem to require. It is possible
         # that older glibc's work, but we aren't trying.
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index da7bcc99826..1dc73ad0e80 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -416,7 +416,7 @@ buildvariants:
   expansions:
     additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
     test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov
-    compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic
+    compile_flags: --dbg=on --gcov --opt=off --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic
     large_distro_name: rhel80-medium
     multiversion_platform: rhel80
     multiversion_edition: enterprise
@@ -497,7 +497,7 @@ buildvariants:
   expansions:
     additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
     test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov
-    compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --link-model=dynamic
+    compile_flags: --dbg=on --gcov --opt=debug --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --link-model=dynamic
     large_distro_name: rhel80-medium
     multiversion_platform: rhel80
     multiversion_edition: enterprise
@@ -530,7 +530,7 @@ buildvariants:
   expansions:
     additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
     test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov
-    compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_gcc.vars --link-model=dynamic
+    compile_flags: --dbg=on --gcov --opt=off --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_gcc.vars --link-model=dynamic
     large_distro_name: rhel80-medium
     multiversion_platform: rhel80
     multiversion_edition: enterprise
@@ -560,7 +560,7 @@ buildvariants:
   expansions:
     additional_package_targets: archive-mongocryptd archive-mongocryptd-debug
     test_flags: --excludeWithAnyTags=resource_intensive,incompatible_with_gcov
-    compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_clang.vars --link-model=dynamic
+    compile_flags: --dbg=on --gcov --opt=debug --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_clang.vars --link-model=dynamic
     large_distro_name: rhel80-medium
     multiversion_platform: rhel80
     multiversion_edition: enterprise
diff --git a/src/mongo/db/exec/sbe/stages/column_scan.cpp b/src/mongo/db/exec/sbe/stages/column_scan.cpp
index a003515427f..91a6f02b43b 100644
--- a/src/mongo/db/exec/sbe/stages/column_scan.cpp
+++ b/src/mongo/db/exec/sbe/stages/column_scan.cpp
@@ -40,7 +40,7 @@ namespace mongo {
 namespace sbe {
 namespace {
 TranslatedCell translateCell(PathView path, const SplitCellView& splitCellView) {
-    value::ColumnStoreEncoder encoder;
+    value::ColumnStoreEncoder encoder{};
     SplitCellView::Cursor<value::ColumnStoreEncoder> cellCursor =
         splitCellView.subcellValuesGenerator<value::ColumnStoreEncoder>(std::move(encoder));
     return TranslatedCell{splitCellView.arrInfo, path, std::move(cellCursor)};
diff --git a/src/mongo/transport/service_executor_utils.cpp b/src/mongo/transport/service_executor_utils.cpp
index 42ec7fcbf5f..6b37efcd383 100644
--- a/src/mongo/transport/service_executor_utils.cpp
+++ b/src/mongo/transport/service_executor_utils.cpp
@@ -46,6 +46,7 @@
 
 #if !defined(_WIN32)
 #include <sys/resource.h>
+#include <unistd.h>
 #endif
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kDefault
@@ -86,14 +87,31 @@ Status launchServiceWorkerThread(unique_function<void()> task) noexcept {
         struct rlimit limits;
         invariant(getrlimit(RLIMIT_STACK, &limits) == 0);
         if (limits.rlim_cur >= kStackSize) {
-            int failed = pthread_attr_setstacksize(&attrs, kStackSize);
+
+            size_t stackSizeToSet = kStackSize;
+
+#if !defined(_WIN32) && (__SANITIZE_ADDRESS__ || __has_feature(address_sanitizer))
+            // If we are using address sanitizer, we set the stack at
+            // ~75% (rounded up to a multiple of the page size) of our
+            // usual desired. Since ASAN is known to use stack more
+            // aggressively and should positively detect stack overflow,
+            // this gives us increased confidence during testing that we
+            // aren't flirting with our real 1MB limit for any tested
+            // workloads. Note: This calculation only works on POSIX
+            // platforms. If we ever decide to use the MSVC
+            // implementation of ASAN, we will need to revisit it.
+            long page_size = sysconf(_SC_PAGE_SIZE);
+            stackSizeToSet =
+                ((((stackSizeToSet * 3) >> 2) + page_size - 1) / page_size) * page_size;
+#endif
+            int failed = pthread_attr_setstacksize(&attrs, stackSizeToSet);
             if (failed) {
                 LOGV2_WARNING(22949,
                               "pthread_attr_setstacksize failed: {error}",
                               "pthread_attr_setstacksize failed",
                               "error"_attr = errorMessage(posixError(failed)));
             }
-        } else if (limits.rlim_cur < kStackSize) {
+        } else {
             LOGV2_WARNING(22950,
                           "Stack size set to {stackSizeKiB}KiB. We suggest 1024KiB",
                           "Stack size not set to suggested 1024KiB",
diff --git a/src/third_party/unwind/SConscript b/src/third_party/unwind/SConscript
index 01b5c41d2a1..0d74cfb8a78 100644
--- a/src/third_party/unwind/SConscript
+++ b/src/third_party/unwind/SConscript
@@ -129,6 +129,11 @@ env.Append(CCFLAGS=[
     '-Wno-unused-variable',
 ])
 
+if env.ToolchainIs('gcc'):
+    env.Append(CCFLAGS=[
+        '-Wno-uninitialized',
+    ])
+
 if env['TARGET_ARCH'] == 'ppc64le':
     env.Append(CCFLAGS=[
         '-Wno-unused-value',
diff --git a/src/third_party/wiredtiger/SConscript b/src/third_party/wiredtiger/SConscript
index 8505aa6b3a3..2ecdcc40d20 100644
--- a/src/third_party/wiredtiger/SConscript
+++ b/src/third_party/wiredtiger/SConscript
@@ -82,6 +82,7 @@ elif env.TargetOSIs('linux'):
     else:
         env.Append(CPPPATH=["build_linux"])
         env.Append(CPPDEFINES=["_GNU_SOURCE"])
+        env.Append(CCFLAGS=["-Wno-uninitialized"])
 else:
     print("Wiredtiger is not supported on this platform. " +
           "Please generate an approriate wiredtiger_config.h")
diff --git a/src/third_party/zstandard/SConscript b/src/third_party/zstandard/SConscript
index 541026b9cc9..b1ab276ab82 100644
--- a/src/third_party/zstandard/SConscript
+++ b/src/third_party/zstandard/SConscript
@@ -49,3 +49,6 @@ env.Library(
         'init-no-global-side-effects',
     ],
 )
+
+if env.ToolchainIs('gcc'):
+    env.Append(CCFLAGS=["-Wno-stringop-overflow"])