SERVER-48291 Ensure runtime is dynamically linked in dynamic builds

Prior to this point, a dynamic build might have resulted in some runtime
libraries being statically linked into shared objects and executables in
cases where "shared" runtime libraries were actually linker scripts that
linked static versions. This was the case with the MongoDB toolchain and
some distro toolchains, including those installed as updated compiler
versions in RHEL.

The effect of having runtime libraries statically linked was that
symbols from those libraries would end up scattered over the compiled
objects, increasing object sizes and slowing down server startup. Now,
whenever a dynamic build is selected, the user can choose whether to
create "shim" runtime libraries that wrap the static ones.

The default behavior on Linux is that dynamic builds will detect whether
runtime libraries are linker scripts and create shim libraries if any are
found. On Windows, the default is to always use a dynamic runtime library
with dynamic builds. For other platforms, the prior behavior remains
unchanged.

8 files changed, 320 insertions, 78 deletions

diff --git a/SConstruct b/SConstruct
index 2f96c965b83..31ec7c11b74 100644
--- a/SConstruct
+++ b/SConstruct
@@ -429,6 +429,15 @@ add_option("cxx-std",
     help="Select the C++ langauge standard to build with",
 )
 
+add_option("dynamic-runtime",
+    choices=["force", "off", "auto"],
+    const="on",
+    default="auto",
+    help="Force the static compiler and C++ runtimes to be linked dynamically",
+    nargs="?",
+    type="choice",
+)
+
 def find_mongo_custom_variables():
     files = []
     paths = [path for path in sys.path if 'site_scons' in path]
@@ -1146,6 +1155,25 @@ for var in ['CC', 'CXX']:
 env.AddMethod(mongo_platform.env_os_is_wrapper, 'TargetOSIs')
 env.AddMethod(mongo_platform.env_get_os_name_wrapper, 'GetTargetOSName')
 
+
+def shim_library(env, name, needs_link=False, *args, **kwargs):
+    nodes = env.Library(
+        target=f"shim_{name}" if name else name,
+        source=[
+            f"shim_{name}.cpp" if name else name,
+        ],
+        *args,
+        **kwargs
+    )
+
+    for n in nodes:
+        setattr(n.attributes, "needs_link", needs_link)
+
+    return nodes
+
+env.AddMethod(shim_library, 'ShimLibrary')
+
+
 def conf_error(env, msg, *args):
     print(msg.format(*args))
     print("See {0} for details".format(env.File('$CONFIGURELOG').abspath))
@@ -1634,6 +1662,106 @@ if link_model.startswith("dynamic"):
                     return []
                 env['LIBDEPS_TAG_EXPANSIONS'].append(libdeps_tags_expand_incomplete)
 
+
+# If requested, wrap the static runtime libraries in shims and use those to link
+# them dynamically. This allows us to "convert" runtimes in toolchains that have
+# linker scripts in place of shared libraries which actually link the static
+# library instead. The benefit of making this conversion is that shared
+# libraries produced by these toolchains are smaller because we don't end up
+# spreading runtime symbols all over the place, and in turn they should also
+# get loaded by the dynamic linker more quickly as well.
+dynamicRT = get_option("dynamic-runtime")
+
+if get_option("link-model") != "dynamic" and dynamicRT == "auto":
+    dynamicRT = "off"
+
+if dynamicRT == "force" and not (env.TargetOSIs('linux') or env.TargetOSIs('windows')):
+    env.FatalError("A dynamic runtime can be forced only on Windows and Linux at this time.")
+
+if env.ToolchainIs('msvc'):
+    # /MD:  use the multithreaded, DLL version of the run-time library
+    # /MT:  use the multithreaded, static version of the run-time library
+    # /MDd: As /MD but also defines _DEBUG
+    # /MTd: As /MT but also defines _DEBUG
+
+    winRuntimeLibMap = {
+          #dyn   #dbg
+        ( False, False ) : "/MT",
+        ( False, True  ) : "/MTd",
+        ( True,  False ) : "/MD",
+        ( True,  True  ) : "/MDd",
+    }
+
+    # On Windows, the dynamic CRT is the default.
+    env.Append(CCFLAGS=[winRuntimeLibMap[(dynamicRT != "off", debugBuild)]])
+
+if dynamicRT == "auto":
+    if env.TargetOSIs('linux') and env.ToolchainIs('gcc', 'clang'):
+        def CheckRuntimeLibraries(context):
+            context.Message("Checking whether any runtime libraries are linker scripts... ")
+
+            result = {}
+            libs = [ 'libgcc', 'libgcc_s', 'libgcc_eh' ]
+
+            if get_option('libc++'):
+                libs.append('libc++')
+            else:
+                libs.append('libstdc++')
+
+            compiler = subprocess.Popen(
+                [context.env['CXX'], "-print-search-dirs"],
+                stdout=subprocess.PIPE
+            )
+
+            # This just pulls out the library paths and *only* the library
+            # paths, deleting all other lines. It also removes the leading
+            # "libraries" tag from the line so only the paths are left in
+            # the output.
+            sed = subprocess.Popen(
+                [
+                    "sed",
+                    "/^lib/b 1;d;:1;s,.*:[^=]*=,,",
+                ],
+                stdin=compiler.stdout,
+                stdout=subprocess.PIPE
+            )
+            compiler.stdout.close()
+
+            search_paths = sed.communicate()[0].decode('utf-8').split(':')
+
+            for lib in libs:
+                for search_path in search_paths:
+                    lib_file = os.path.join(search_path, lib + ".so")
+                    if os.path.exists(lib_file):
+                        file_type = subprocess.check_output(["file", lib_file]).decode('utf-8')
+                        match = re.search('ASCII text', file_type)
+                        result[lib] = bool(match)
+                        break
+            if any(result.values()):
+                ret = "yes"
+            else:
+                ret = "no"
+            context.Result(ret)
+            return ret
+
+        detectStaticRuntime = Configure(detectEnv, help=False, custom_tests = {
+            'CheckRuntimeLibraries' : CheckRuntimeLibraries,
+        })
+        if detectStaticRuntime.CheckRuntimeLibraries() == "yes":
+            dynamicRT = "force"
+        else:
+            dynamicRT = "off"
+
+        detectStaticRuntime.Finish()
+
+elif dynamicRT == "force":
+    if get_option("link-model") != "dynamic":
+        env.FatalError("A dynamic runtime can only be forced with dynamic linking.")
+
+    if not env.ToolchainIs('gcc', 'clang'):
+        env.FatalError("Don't know how to bundle a dynamic runtime on this toolchain.")
+
+
 if optBuild:
     env.SetConfigHeaderDefine("MONGO_CONFIG_OPTIMIZED_BUILD")
 
@@ -2090,11 +2218,6 @@ elif env.TargetOSIs('windows'):
     if not any(flag.startswith('/DEBUG') for flag in env['LINKFLAGS']):
         env.Append(LINKFLAGS=["/DEBUG"])
 
-    # /MD:  use the multithreaded, DLL version of the run-time library (MSVCRT.lib/MSVCR###.DLL)
-    # /MDd: Defines _DEBUG, _MT, _DLL, and uses MSVCRTD.lib/MSVCRD###.DLL
-
-    env.Append(CCFLAGS=["/MDd" if debugBuild else "/MD"])
-
     if optBuild:
         # /O1:  optimize for size
         # /O2:  optimize for speed (as opposed to size)
@@ -2243,8 +2366,14 @@ if env.TargetOSIs('posix'):
     # On OS X, clang doesn't want the pthread flag at link time, or it
     # issues warnings which make it impossible for us to declare link
     # warnings as errors. See http://stackoverflow.com/a/19382663.
-    if not (env.TargetOSIs('darwin') and env.ToolchainIs('clang')):
-        env.Append( LINKFLAGS=["-pthread"] )
+    #
+    # We don't need it anyway since we explicitly link to -lpthread,
+    # so all we need beyond that is the preprocessor variable.
+    if not env.ToolchainIs('clang'):
+        env.Append(
+            CPPDEFINES=[("_REENTRANT", "1")],
+            LINKFLAGS=["-pthread"]
+        )
 
     # SERVER-9761: Ensure early detection of missing symbols in dependent libraries at program
     # startup.
@@ -3810,8 +3939,10 @@ def doConfigure(myenv):
                     language='C++')
     if posix_system:
         conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_HEADER_UNISTD_H")
+        conf.CheckLib('c')
         conf.CheckLib('rt')
         conf.CheckLib('dl')
+        conf.CheckLib('pthread')
 
     if posix_monotonic_clock:
         conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_POSIX_MONOTONIC_CLOCK")
@@ -4777,6 +4908,7 @@ module_sconscripts = moduleconfig.get_module_sconscripts(mongo_modules)
 # and they are exported here, as well.
 Export([
     'debugBuild',
+    'dynamicRT',
     'endian',
     'free_monitoring',
     'get_option',
diff --git a/src/SConscript b/src/SConscript
index fbf8e1fcecb..3c37fe2cd1c 100644
--- a/src/SConscript
+++ b/src/SConscript
@@ -3,21 +3,168 @@
 # This is the principle SConscript file, invoked by the SConstruct.  Its job is
 # to delegate to any and all per-module SConscript files.
 
+from functools import partial
+
+from site_scons.mongo import insort_wrapper
+
+import SCons
+
+Import('dynamicRT')
 Import('env')
 Import('get_option')
+Import('has_option')
 Import('module_sconscripts')
 
 
-env = env.Clone()
+def shim_hack(target, source, env, inject_target=None, disable_var=None):
+    # If we allowed conftests to become dependent, any TryLink
+    # that happened after we made the below modifications would
+    # cause the configure steps to try to compile tcmalloc and any
+    # of its dependencies. Oops!
+    if any('conftest' in str(t) for t in target):
+        return target, source
+
+    # It is possible that 'env' isn't a unique
+    # OverrideEnvironment, since if you didn't pass any kw args
+    # into your builder call, you just reuse the env you were
+    # called with. That could mean that we see the same
+    # environment here multiple times. But that is really OK,
+    # since the operation we are performing would be performed on
+    # all of them anyway. The flag serves as a way to disable the
+    # auto-injection for the handful of libraries where we must do
+    # so to avoid forming a cycle.
+    if not env.get(disable_var, False):
+        lds = env.get('LIBDEPS', [])
+        shim_target = f"$BUILD_DIR/{inject_target}"
+        if shim_target not in lds:
+            insort_wrapper(lds, shim_target)
+            env['LIBDEPS'] = lds
+
+    return target, source
+
+
+def add_shim_hack(env, hack_method):
+    for builder_name in ('Program', 'SharedLibrary', 'LoadableModule', 'StaticLibrary'):
+        builder = env['BUILDERS'][builder_name]
+        base_emitter = builder.emitter
+        builder.emitter = SCons.Builder.ListEmitter([hack_method, base_emitter])
+
+
+# Here, we "hoist" libgcc*.a symbols out of the toolchain and stuff them into
+# our own library so they don't get added piecemeal to every shared object and
+# executable in a build directly out of the toolchain.
+libcrtEnv = env.Clone(LIBS=[])
+libcxxEnv = env.Clone(LIBS=[])
+if dynamicRT == "force":
+
+    if env.ToolchainIs('gcc', 'clang'):
+
+        env.AppendUnique(
+            LINKFLAGS=[
+                '-nodefaultlibs',
+                '-l:libgcc_s.so'
+            ],
+        )
+
+        libcrtEnv.AppendUnique(
+            LINKFLAGS=[
+                "-Wl,-z,muldefs",
+                "-static-libgcc",
+                "-Wl,--push-state",
+                "-Wl,-Bstatic",
+                "-Wl,--no-warn-execstack",
+                "-Wl,--whole-archive",
+                "-lgcc",
+                "-lgcc_eh",
+                "-Wl,--no-whole-archive",
+                "-Wl,--pop-state",
+            ]
+        )
+
+        if has_option("libc++"):
+            cxx_lib = "c++"
+        else:
+            cxx_lib = "stdc++"
+
+        libcxxEnv.AppendUnique(
+            LINKFLAGS=[
+                "-Wl,-z,muldefs",
+                f"-static-lib{cxx_lib}",
+                "-Wl,--push-state",
+                "-Wl,-Bstatic",
+                "-Wl,--no-warn-execstack",
+                "-Wl,--whole-archive",
+                f"-l{cxx_lib}",
+                "-Wl,--no-whole-archive",
+                "-Wl,--pop-state",
+            ]
+        )
+
+libcrtEnv.ShimLibrary(
+    name="crt",
+    needs_link=(dynamicRT == "force" and env.ToolchainIs('gcc', 'clang')),
+    LIBDEPS_TAGS=[
+        # TODO: Remove all of these when SERVER-48291 is merged into stable build tools
+        # The shim allocator must be linked to every node, including what would
+        # be considered a leaf node to ensure the system allocator
+        # is not linked in before tcmalloc. This tag allows nodes tagged as
+        # leaf nodes to still get the correct allocator.
+        'lint-leaf-node-allowed-dep',
+        # This tag allows the allocator to be linked to nodes marked as not
+        # allowed to have public dependencies.
+        'lint-public-dep-allowed'
+    ],
+)
+
+libcxxEnv.ShimLibrary(
+    name="cxx",
+    needs_link=(dynamicRT == "force" and env.ToolchainIs('gcc', 'clang')),
+    LIBDEPS_TAGS=[
+        # TODO: Remove all of these when SERVER-48291 is merged into stable build tools
+        # The shim allocator must be linked to every node, including what would
+        # be considered a leaf node to ensure the system allocator
+        # is not linked in before tcmalloc. This tag allows nodes tagged as
+        # leaf nodes to still get the correct allocator.
+        'lint-leaf-node-allowed-dep',
+        # This tag allows the allocator to be linked to nodes marked as not
+        # allowed to have public dependencies.
+        'lint-public-dep-allowed'
+    ],
+)
+
+
 if get_option("build-tools") == "next":
     # Add any "global" dependencies here. This is where we make every build node
     # depend on a list of other build nodes, such as an allocator or libunwind
     # or libstdx or similar.
     env.AppendUnique(
         LIBDEPS_GLOBAL=[
+            '$BUILD_DIR/shim_crt' if dynamicRT == "force" else [],
+            '$BUILD_DIR/shim_cxx' if dynamicRT == "force" else [],
             '$BUILD_DIR/third_party/shim_allocator',
         ],
     )
+else:
+    add_shim_hack(
+        env,
+        partial(
+            shim_hack,
+            inject_target='third_party/shim_allocator',
+            disable_var='DISABLE_ALLOCATOR_SHIM_INJECTION'))
+    if dynamicRT == "force":
+        add_shim_hack(
+            env,
+            partial(
+                shim_hack,
+                inject_target='shim_cxx',
+                disable_var='DISABLE_CXX_SHIM_INJECTION'))
+        add_shim_hack(
+            env,
+            partial(
+                shim_hack,
+                inject_target='shim_crt',
+                disable_var='DISABLE_CRT_SHIM_INJECTION'))
+
 
 # NOTE: We must do third_party first as it adds methods to the environment
 # that we need in the mongo sconscript
diff --git a/src/mongo/installer/msi/ca/SConscript b/src/mongo/installer/msi/ca/SConscript
index 01068b4a3ce..0693c6223e0 100644
--- a/src/mongo/installer/msi/ca/SConscript
+++ b/src/mongo/installer/msi/ca/SConscript
@@ -34,7 +34,7 @@ ca = env.SharedLibrary(
     LIBDEPS_GLOBAL=[
         dep
         for dep in env.get('LIBDEPS_GLOBAL', [])
-        if not dep.endswith('shim_allocator')
+        if dep and not dep.endswith('shim_allocator')
     ],
     # TODO: Remove when SERVER-48291 is merged into stable build tools
     DISABLE_ALLOCATOR_SHIM_INJECTION=True,
diff --git a/src/shim_crt.cpp b/src/shim_crt.cpp
new file mode 100644
index 00000000000..4c49e6668c1
--- /dev/null
+++ b/src/shim_crt.cpp
@@ -0,0 +1,3 @@
+// This file intentionally blank. shim_crt.cpp rolls together the compiler
+// runtime libraries to provide a single source of those symbols to other
+// compiled objects in a dynamically linked build.
diff --git a/src/shim_cxx.cpp b/src/shim_cxx.cpp
new file mode 100644
index 00000000000..d18e22432d7
--- /dev/null
+++ b/src/shim_cxx.cpp
@@ -0,0 +1,3 @@
+// This file intentionally blank. shim_cxx.cpp rolls together the C++
+// runtime libraries to provide a single source of those symbols to other
+// compiled objects in a dynamically linked build.
diff --git a/src/third_party/SConscript b/src/third_party/SConscript
index 0c2674ff356..b24ebb1f505 100644
--- a/src/third_party/SConscript
+++ b/src/third_party/SConscript
@@ -1,12 +1,8 @@
 # -*- mode: python -*-
 
-import SCons
-
-from site_scons.mongo import insort_wrapper
 import json
 
 Import("env use_system_version_of_library usemozjs get_option")
-Import("get_option")
 Import("use_libunwind")
 Import("use_system_libunwind")
 Import("use_vendored_libunwind")
@@ -85,59 +81,6 @@ def injectMozJS(thisEnv):
 env.AddMethod(injectMozJS, 'InjectMozJS');
 
 
-def add_shim_allocator_hack(target, source, env):
-
-    # If we allowed conftests to become dependent, any TryLink
-    # that happened after we made the below modifications would
-    # cause the configure steps to try to compile tcmalloc and any
-    # of its dependencies. Oops!
-    if any('conftest' in str(t) for t in target):
-        return target, source
-
-    # It is possible that 'env' isn't a unique
-    # OverrideEnvironment, since if you didn't pass any kw args
-    # into your builder call, you just reuse the env you were
-    # called with. That could mean that we see the same
-    # environment here multiple times. But that is really OK,
-    # since the operation we are performing would be performed on
-    # all of them anyway. The flag serves as a way to disable the
-    # auto-injection for the handful of libraries where we must do
-    # so to avoid forming a cycle.
-    if not env.get('DISABLE_ALLOCATOR_SHIM_INJECTION', False):
-        lds = env.get('LIBDEPS', [])
-        shim_allocator = '$BUILD_DIR/third_party/shim_allocator'
-        if shim_allocator not in lds:
-            insort_wrapper(lds, shim_allocator)
-            env['LIBDEPS'] = lds
-
-    return target, source
-
-
-if get_option("build-tools") == "stable":
-    for builder_name in ('Program', 'SharedLibrary', 'LoadableModule', 'StaticLibrary'):
-        builder = env['BUILDERS'][builder_name]
-        base_emitter = builder.emitter
-        builder.emitter = SCons.Builder.ListEmitter([add_shim_allocator_hack, base_emitter])
-
-
-def shim_library(env, name, needs_link=False, *args, **kwargs):
-    nodes = env.Library(
-        target=f"shim_{name}" if name else name,
-        source=[
-            f"shim_{name}.cpp" if name else name,
-        ],
-        *args,
-        **kwargs
-    )
-
-    for n in nodes:
-        setattr(n.attributes, "needs_link", needs_link)
-
-    return nodes
-
-env.AddMethod(shim_library, 'ShimLibrary')
-
-
 if not use_system_version_of_library('tcmalloc'):
     # GPerftools does this slightly differently than the others.
     thirdPartyEnvironmentModifications['gperftools'] = {}
@@ -290,8 +233,8 @@ if use_libunwind:
         LIBDEPS_GLOBAL=[
             dep
             for dep in env.get('LIBDEPS_GLOBAL', [])
-            if not dep.endswith('shim_allocator')
-        ]
+            if dep and not dep.endswith('shim_allocator')
+        ],
     )
     if use_system_libunwind:
         unwindEnv = unwindEnv.Clone(
@@ -447,7 +390,9 @@ else:
             'zlib' + zlibSuffix + '/zlib',
         ])
 
-zlibEnv.ShimLibrary(name="zlib")
+zlibEnv.ShimLibrary(
+    name="zlib",
+)
 
 zstdEnv = env.Clone()
 if use_system_version_of_library("zstd"):
@@ -464,7 +409,9 @@ else:
             'zstandard' + zstdSuffix + '/zstd',
         ])
 
-zstdEnv.ShimLibrary(name="zstd")
+zstdEnv.ShimLibrary(
+    name="zstd",
+)
 
 benchmarkEnv = env.Clone()
 if use_system_version_of_library("google-benchmark"):
@@ -504,14 +451,16 @@ if "tom" in env["MONGO_CRYPTO"]:
             'tomcrypt' + tomcryptSuffix + '/tomcrypt',
         ])
 
-    tomcryptEnv.ShimLibrary(name="tomcrypt")
+    tomcryptEnv.ShimLibrary(
+        name="tomcrypt",
+    )
 
 
 gperftoolsEnv = env.Clone(
     LIBDEPS_GLOBAL=[
         dep
         for dep in env.get('LIBDEPS_GLOBAL', [])
-        if not dep.endswith('shim_allocator')
+        if dep and not dep.endswith('shim_allocator')
     ],
 )
 if gperftoolsEnv['MONGO_ALLOCATOR'] in ["tcmalloc", "tcmalloc-experimental"]:
@@ -598,7 +547,9 @@ timelibEnv = timelibEnv.Clone(
         'timelib' + timelibSuffix + '/timelib',
     ])
 
-timelibEnv.ShimLibrary(name='timelib')
+timelibEnv.ShimLibrary(
+    name='timelib',
+)
 
 wiredtigerEnv = env.Clone()
 if wiredtiger:
@@ -650,7 +601,9 @@ else:
 	    'IntelRDFPMathLib20U1/intel_decimal128',
 	])
 
-intelDecimal128Env.ShimLibrary(name="intel_decimal128")
+intelDecimal128Env.ShimLibrary(
+    name="intel_decimal128",
+)
 
 icuEnv = env.Clone()
 if use_system_version_of_library("icu"):
@@ -669,7 +622,9 @@ else:
             'icu4c' + icuSuffix + '/source/icu_i18n',
         ])
 
-icuEnv.ShimLibrary(name="icu")
+icuEnv.ShimLibrary(
+    name="icu",
+)
 
 kmsEnv = env.Clone()
 if get_option('ssl') == 'on':
@@ -687,5 +642,7 @@ if get_option('ssl') == 'on':
                 'kms-message/kms-message',
             ])
 
-    kmsEnv.ShimLibrary(name="kms_message")
+    kmsEnv.ShimLibrary(
+        name="kms_message",
+    )
 
diff --git a/src/third_party/gperftools/SConscript b/src/third_party/gperftools/SConscript
index 2025d7a9ea6..8f4f456816a 100644
--- a/src/third_party/gperftools/SConscript
+++ b/src/third_party/gperftools/SConscript
@@ -128,7 +128,7 @@ env.Library(
     LIBDEPS_GLOBAL=[
         dep
         for dep in env.get('LIBDEPS_GLOBAL', [])
-        if not dep.endswith('shim_allocator')
+        if dep and not dep.endswith('shim_allocator')
     ],
     # TODO: Remove when SERVER-48291 is merged into stable build tools
     DISABLE_ALLOCATOR_SHIM_INJECTION=True,
diff --git a/src/third_party/unwind/SConscript b/src/third_party/unwind/SConscript
index 983478dd22a..5ea497b0c53 100644
--- a/src/third_party/unwind/SConscript
+++ b/src/third_party/unwind/SConscript
@@ -131,7 +131,7 @@ env.Library(
     LIBDEPS_GLOBAL=[
         dep
         for dep in env.get('LIBDEPS_GLOBAL', [])
-        if not dep.endswith('shim_allocator')
+        if dep and not (dep.endswith('shim_allocator') or dep.endswith('shim_cxx'))
     ],
     # TODO: Remove when SERVER-48291 is merged into stable build tools
     DISABLE_ALLOCATOR_SHIM_INJECTION=True,