diff options
| -rw-r--r-- | SConstruct | 146 | ||||
| -rw-r--r-- | src/SConscript | 149 | ||||
| -rw-r--r-- | src/mongo/installer/msi/ca/SConscript | 2 | ||||
| -rw-r--r-- | src/shim_crt.cpp | 3 | ||||
| -rw-r--r-- | src/shim_cxx.cpp | 3 | ||||
| -rw-r--r-- | src/third_party/SConscript | 91 | ||||
| -rw-r--r-- | src/third_party/gperftools/SConscript | 2 | ||||
| -rw-r--r-- | src/third_party/unwind/SConscript | 2 |
8 files changed, 320 insertions, 78 deletions
diff --git a/SConstruct b/SConstruct index 2f96c965b83..31ec7c11b74 100644 --- a/SConstruct +++ b/SConstruct @@ -429,6 +429,15 @@ add_option("cxx-std", help="Select the C++ langauge standard to build with", ) +add_option("dynamic-runtime", + choices=["force", "off", "auto"], + const="on", + default="auto", + help="Force the static compiler and C++ runtimes to be linked dynamically", + nargs="?", + type="choice", +) + def find_mongo_custom_variables(): files = [] paths = [path for path in sys.path if 'site_scons' in path] @@ -1146,6 +1155,25 @@ for var in ['CC', 'CXX']: env.AddMethod(mongo_platform.env_os_is_wrapper, 'TargetOSIs') env.AddMethod(mongo_platform.env_get_os_name_wrapper, 'GetTargetOSName') + +def shim_library(env, name, needs_link=False, *args, **kwargs): + nodes = env.Library( + target=f"shim_{name}" if name else name, + source=[ + f"shim_{name}.cpp" if name else name, + ], + *args, + **kwargs + ) + + for n in nodes: + setattr(n.attributes, "needs_link", needs_link) + + return nodes + +env.AddMethod(shim_library, 'ShimLibrary') + + def conf_error(env, msg, *args): print(msg.format(*args)) print("See {0} for details".format(env.File('$CONFIGURELOG').abspath)) @@ -1634,6 +1662,106 @@ if link_model.startswith("dynamic"): return [] env['LIBDEPS_TAG_EXPANSIONS'].append(libdeps_tags_expand_incomplete) + +# If requested, wrap the static runtime libraries in shims and use those to link +# them dynamically. This allows us to "convert" runtimes in toolchains that have +# linker scripts in place of shared libraries which actually link the static +# library instead. The benefit of making this conversion is that shared +# libraries produced by these toolchains are smaller because we don't end up +# spreading runtime symbols all over the place, and in turn they should also +# get loaded by the dynamic linker more quickly as well. +dynamicRT = get_option("dynamic-runtime") + +if get_option("link-model") != "dynamic" and dynamicRT == "auto": + dynamicRT = "off" + +if dynamicRT == "force" and not (env.TargetOSIs('linux') or env.TargetOSIs('windows')): + env.FatalError("A dynamic runtime can be forced only on Windows and Linux at this time.") + +if env.ToolchainIs('msvc'): + # /MD: use the multithreaded, DLL version of the run-time library + # /MT: use the multithreaded, static version of the run-time library + # /MDd: As /MD but also defines _DEBUG + # /MTd: As /MT but also defines _DEBUG + + winRuntimeLibMap = { + #dyn #dbg + ( False, False ) : "/MT", + ( False, True ) : "/MTd", + ( True, False ) : "/MD", + ( True, True ) : "/MDd", + } + + # On Windows, the dynamic CRT is the default. + env.Append(CCFLAGS=[winRuntimeLibMap[(dynamicRT != "off", debugBuild)]]) + +if dynamicRT == "auto": + if env.TargetOSIs('linux') and env.ToolchainIs('gcc', 'clang'): + def CheckRuntimeLibraries(context): + context.Message("Checking whether any runtime libraries are linker scripts... ") + + result = {} + libs = [ 'libgcc', 'libgcc_s', 'libgcc_eh' ] + + if get_option('libc++'): + libs.append('libc++') + else: + libs.append('libstdc++') + + compiler = subprocess.Popen( + [context.env['CXX'], "-print-search-dirs"], + stdout=subprocess.PIPE + ) + + # This just pulls out the library paths and *only* the library + # paths, deleting all other lines. It also removes the leading + # "libraries" tag from the line so only the paths are left in + # the output. + sed = subprocess.Popen( + [ + "sed", + "/^lib/b 1;d;:1;s,.*:[^=]*=,,", + ], + stdin=compiler.stdout, + stdout=subprocess.PIPE + ) + compiler.stdout.close() + + search_paths = sed.communicate()[0].decode('utf-8').split(':') + + for lib in libs: + for search_path in search_paths: + lib_file = os.path.join(search_path, lib + ".so") + if os.path.exists(lib_file): + file_type = subprocess.check_output(["file", lib_file]).decode('utf-8') + match = re.search('ASCII text', file_type) + result[lib] = bool(match) + break + if any(result.values()): + ret = "yes" + else: + ret = "no" + context.Result(ret) + return ret + + detectStaticRuntime = Configure(detectEnv, help=False, custom_tests = { + 'CheckRuntimeLibraries' : CheckRuntimeLibraries, + }) + if detectStaticRuntime.CheckRuntimeLibraries() == "yes": + dynamicRT = "force" + else: + dynamicRT = "off" + + detectStaticRuntime.Finish() + +elif dynamicRT == "force": + if get_option("link-model") != "dynamic": + env.FatalError("A dynamic runtime can only be forced with dynamic linking.") + + if not env.ToolchainIs('gcc', 'clang'): + env.FatalError("Don't know how to bundle a dynamic runtime on this toolchain.") + + if optBuild: env.SetConfigHeaderDefine("MONGO_CONFIG_OPTIMIZED_BUILD") @@ -2090,11 +2218,6 @@ elif env.TargetOSIs('windows'): if not any(flag.startswith('/DEBUG') for flag in env['LINKFLAGS']): env.Append(LINKFLAGS=["/DEBUG"]) - # /MD: use the multithreaded, DLL version of the run-time library (MSVCRT.lib/MSVCR###.DLL) - # /MDd: Defines _DEBUG, _MT, _DLL, and uses MSVCRTD.lib/MSVCRD###.DLL - - env.Append(CCFLAGS=["/MDd" if debugBuild else "/MD"]) - if optBuild: # /O1: optimize for size # /O2: optimize for speed (as opposed to size) @@ -2243,8 +2366,14 @@ if env.TargetOSIs('posix'): # On OS X, clang doesn't want the pthread flag at link time, or it # issues warnings which make it impossible for us to declare link # warnings as errors. See http://stackoverflow.com/a/19382663. - if not (env.TargetOSIs('darwin') and env.ToolchainIs('clang')): - env.Append( LINKFLAGS=["-pthread"] ) + # + # We don't need it anyway since we explicitly link to -lpthread, + # so all we need beyond that is the preprocessor variable. + if not env.ToolchainIs('clang'): + env.Append( + CPPDEFINES=[("_REENTRANT", "1")], + LINKFLAGS=["-pthread"] + ) # SERVER-9761: Ensure early detection of missing symbols in dependent libraries at program # startup. @@ -3810,8 +3939,10 @@ def doConfigure(myenv): language='C++') if posix_system: conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_HEADER_UNISTD_H") + conf.CheckLib('c') conf.CheckLib('rt') conf.CheckLib('dl') + conf.CheckLib('pthread') if posix_monotonic_clock: conf.env.SetConfigHeaderDefine("MONGO_CONFIG_HAVE_POSIX_MONOTONIC_CLOCK") @@ -4777,6 +4908,7 @@ module_sconscripts = moduleconfig.get_module_sconscripts(mongo_modules) # and they are exported here, as well. Export([ 'debugBuild', + 'dynamicRT', 'endian', 'free_monitoring', 'get_option', diff --git a/src/SConscript b/src/SConscript index fbf8e1fcecb..3c37fe2cd1c 100644 --- a/src/SConscript +++ b/src/SConscript @@ -3,21 +3,168 @@ # This is the principle SConscript file, invoked by the SConstruct. Its job is # to delegate to any and all per-module SConscript files. +from functools import partial + +from site_scons.mongo import insort_wrapper + +import SCons + +Import('dynamicRT') Import('env') Import('get_option') +Import('has_option') Import('module_sconscripts') -env = env.Clone() +def shim_hack(target, source, env, inject_target=None, disable_var=None): + # If we allowed conftests to become dependent, any TryLink + # that happened after we made the below modifications would + # cause the configure steps to try to compile tcmalloc and any + # of its dependencies. Oops! + if any('conftest' in str(t) for t in target): + return target, source + + # It is possible that 'env' isn't a unique + # OverrideEnvironment, since if you didn't pass any kw args + # into your builder call, you just reuse the env you were + # called with. That could mean that we see the same + # environment here multiple times. But that is really OK, + # since the operation we are performing would be performed on + # all of them anyway. The flag serves as a way to disable the + # auto-injection for the handful of libraries where we must do + # so to avoid forming a cycle. + if not env.get(disable_var, False): + lds = env.get('LIBDEPS', []) + shim_target = f"$BUILD_DIR/{inject_target}" + if shim_target not in lds: + insort_wrapper(lds, shim_target) + env['LIBDEPS'] = lds + + return target, source + + +def add_shim_hack(env, hack_method): + for builder_name in ('Program', 'SharedLibrary', 'LoadableModule', 'StaticLibrary'): + builder = env['BUILDERS'][builder_name] + base_emitter = builder.emitter + builder.emitter = SCons.Builder.ListEmitter([hack_method, base_emitter]) + + +# Here, we "hoist" libgcc*.a symbols out of the toolchain and stuff them into +# our own library so they don't get added piecemeal to every shared object and +# executable in a build directly out of the toolchain. +libcrtEnv = env.Clone(LIBS=[]) +libcxxEnv = env.Clone(LIBS=[]) +if dynamicRT == "force": + + if env.ToolchainIs('gcc', 'clang'): + + env.AppendUnique( + LINKFLAGS=[ + '-nodefaultlibs', + '-l:libgcc_s.so' + ], + ) + + libcrtEnv.AppendUnique( + LINKFLAGS=[ + "-Wl,-z,muldefs", + "-static-libgcc", + "-Wl,--push-state", + "-Wl,-Bstatic", + "-Wl,--no-warn-execstack", + "-Wl,--whole-archive", + "-lgcc", + "-lgcc_eh", + "-Wl,--no-whole-archive", + "-Wl,--pop-state", + ] + ) + + if has_option("libc++"): + cxx_lib = "c++" + else: + cxx_lib = "stdc++" + + libcxxEnv.AppendUnique( + LINKFLAGS=[ + "-Wl,-z,muldefs", + f"-static-lib{cxx_lib}", + "-Wl,--push-state", + "-Wl,-Bstatic", + "-Wl,--no-warn-execstack", + "-Wl,--whole-archive", + f"-l{cxx_lib}", + "-Wl,--no-whole-archive", + "-Wl,--pop-state", + ] + ) + +libcrtEnv.ShimLibrary( + name="crt", + needs_link=(dynamicRT == "force" and env.ToolchainIs('gcc', 'clang')), + LIBDEPS_TAGS=[ + # TODO: Remove all of these when SERVER-48291 is merged into stable build tools + # The shim allocator must be linked to every node, including what would + # be considered a leaf node to ensure the system allocator + # is not linked in before tcmalloc. This tag allows nodes tagged as + # leaf nodes to still get the correct allocator. + 'lint-leaf-node-allowed-dep', + # This tag allows the allocator to be linked to nodes marked as not + # allowed to have public dependencies. + 'lint-public-dep-allowed' + ], +) + +libcxxEnv.ShimLibrary( + name="cxx", + needs_link=(dynamicRT == "force" and env.ToolchainIs('gcc', 'clang')), + LIBDEPS_TAGS=[ + # TODO: Remove all of these when SERVER-48291 is merged into stable build tools + # The shim allocator must be linked to every node, including what would + # be considered a leaf node to ensure the system allocator + # is not linked in before tcmalloc. This tag allows nodes tagged as + # leaf nodes to still get the correct allocator. + 'lint-leaf-node-allowed-dep', + # This tag allows the allocator to be linked to nodes marked as not + # allowed to have public dependencies. + 'lint-public-dep-allowed' + ], +) + + if get_option("build-tools") == "next": # Add any "global" dependencies here. This is where we make every build node # depend on a list of other build nodes, such as an allocator or libunwind # or libstdx or similar. env.AppendUnique( LIBDEPS_GLOBAL=[ + '$BUILD_DIR/shim_crt' if dynamicRT == "force" else [], + '$BUILD_DIR/shim_cxx' if dynamicRT == "force" else [], '$BUILD_DIR/third_party/shim_allocator', ], ) +else: + add_shim_hack( + env, + partial( + shim_hack, + inject_target='third_party/shim_allocator', + disable_var='DISABLE_ALLOCATOR_SHIM_INJECTION')) + if dynamicRT == "force": + add_shim_hack( + env, + partial( + shim_hack, + inject_target='shim_cxx', + disable_var='DISABLE_CXX_SHIM_INJECTION')) + add_shim_hack( + env, + partial( + shim_hack, + inject_target='shim_crt', + disable_var='DISABLE_CRT_SHIM_INJECTION')) + # NOTE: We must do third_party first as it adds methods to the environment # that we need in the mongo sconscript diff --git a/src/mongo/installer/msi/ca/SConscript b/src/mongo/installer/msi/ca/SConscript index 01068b4a3ce..0693c6223e0 100644 --- a/src/mongo/installer/msi/ca/SConscript +++ b/src/mongo/installer/msi/ca/SConscript @@ -34,7 +34,7 @@ ca = env.SharedLibrary( LIBDEPS_GLOBAL=[ dep for dep in env.get('LIBDEPS_GLOBAL', []) - if not dep.endswith('shim_allocator') + if dep and not dep.endswith('shim_allocator') ], # TODO: Remove when SERVER-48291 is merged into stable build tools DISABLE_ALLOCATOR_SHIM_INJECTION=True, diff --git a/src/shim_crt.cpp b/src/shim_crt.cpp new file mode 100644 index 00000000000..4c49e6668c1 --- /dev/null +++ b/src/shim_crt.cpp @@ -0,0 +1,3 @@ +// This file intentionally blank. shim_crt.cpp rolls together the compiler +// runtime libraries to provide a single source of those symbols to other +// compiled objects in a dynamically linked build. diff --git a/src/shim_cxx.cpp b/src/shim_cxx.cpp new file mode 100644 index 00000000000..d18e22432d7 --- /dev/null +++ b/src/shim_cxx.cpp @@ -0,0 +1,3 @@ +// This file intentionally blank. shim_cxx.cpp rolls together the C++ +// runtime libraries to provide a single source of those symbols to other +// compiled objects in a dynamically linked build. diff --git a/src/third_party/SConscript b/src/third_party/SConscript index 0c2674ff356..b24ebb1f505 100644 --- a/src/third_party/SConscript +++ b/src/third_party/SConscript @@ -1,12 +1,8 @@ # -*- mode: python -*- -import SCons - -from site_scons.mongo import insort_wrapper import json Import("env use_system_version_of_library usemozjs get_option") -Import("get_option") Import("use_libunwind") Import("use_system_libunwind") Import("use_vendored_libunwind") @@ -85,59 +81,6 @@ def injectMozJS(thisEnv): env.AddMethod(injectMozJS, 'InjectMozJS'); -def add_shim_allocator_hack(target, source, env): - - # If we allowed conftests to become dependent, any TryLink - # that happened after we made the below modifications would - # cause the configure steps to try to compile tcmalloc and any - # of its dependencies. Oops! - if any('conftest' in str(t) for t in target): - return target, source - - # It is possible that 'env' isn't a unique - # OverrideEnvironment, since if you didn't pass any kw args - # into your builder call, you just reuse the env you were - # called with. That could mean that we see the same - # environment here multiple times. But that is really OK, - # since the operation we are performing would be performed on - # all of them anyway. The flag serves as a way to disable the - # auto-injection for the handful of libraries where we must do - # so to avoid forming a cycle. - if not env.get('DISABLE_ALLOCATOR_SHIM_INJECTION', False): - lds = env.get('LIBDEPS', []) - shim_allocator = '$BUILD_DIR/third_party/shim_allocator' - if shim_allocator not in lds: - insort_wrapper(lds, shim_allocator) - env['LIBDEPS'] = lds - - return target, source - - -if get_option("build-tools") == "stable": - for builder_name in ('Program', 'SharedLibrary', 'LoadableModule', 'StaticLibrary'): - builder = env['BUILDERS'][builder_name] - base_emitter = builder.emitter - builder.emitter = SCons.Builder.ListEmitter([add_shim_allocator_hack, base_emitter]) - - -def shim_library(env, name, needs_link=False, *args, **kwargs): - nodes = env.Library( - target=f"shim_{name}" if name else name, - source=[ - f"shim_{name}.cpp" if name else name, - ], - *args, - **kwargs - ) - - for n in nodes: - setattr(n.attributes, "needs_link", needs_link) - - return nodes - -env.AddMethod(shim_library, 'ShimLibrary') - - if not use_system_version_of_library('tcmalloc'): # GPerftools does this slightly differently than the others. thirdPartyEnvironmentModifications['gperftools'] = {} @@ -290,8 +233,8 @@ if use_libunwind: LIBDEPS_GLOBAL=[ dep for dep in env.get('LIBDEPS_GLOBAL', []) - if not dep.endswith('shim_allocator') - ] + if dep and not dep.endswith('shim_allocator') + ], ) if use_system_libunwind: unwindEnv = unwindEnv.Clone( @@ -447,7 +390,9 @@ else: 'zlib' + zlibSuffix + '/zlib', ]) -zlibEnv.ShimLibrary(name="zlib") +zlibEnv.ShimLibrary( + name="zlib", +) zstdEnv = env.Clone() if use_system_version_of_library("zstd"): @@ -464,7 +409,9 @@ else: 'zstandard' + zstdSuffix + '/zstd', ]) -zstdEnv.ShimLibrary(name="zstd") +zstdEnv.ShimLibrary( + name="zstd", +) benchmarkEnv = env.Clone() if use_system_version_of_library("google-benchmark"): @@ -504,14 +451,16 @@ if "tom" in env["MONGO_CRYPTO"]: 'tomcrypt' + tomcryptSuffix + '/tomcrypt', ]) - tomcryptEnv.ShimLibrary(name="tomcrypt") + tomcryptEnv.ShimLibrary( + name="tomcrypt", + ) gperftoolsEnv = env.Clone( LIBDEPS_GLOBAL=[ dep for dep in env.get('LIBDEPS_GLOBAL', []) - if not dep.endswith('shim_allocator') + if dep and not dep.endswith('shim_allocator') ], ) if gperftoolsEnv['MONGO_ALLOCATOR'] in ["tcmalloc", "tcmalloc-experimental"]: @@ -598,7 +547,9 @@ timelibEnv = timelibEnv.Clone( 'timelib' + timelibSuffix + '/timelib', ]) -timelibEnv.ShimLibrary(name='timelib') +timelibEnv.ShimLibrary( + name='timelib', +) wiredtigerEnv = env.Clone() if wiredtiger: @@ -650,7 +601,9 @@ else: 'IntelRDFPMathLib20U1/intel_decimal128', ]) -intelDecimal128Env.ShimLibrary(name="intel_decimal128") +intelDecimal128Env.ShimLibrary( + name="intel_decimal128", +) icuEnv = env.Clone() if use_system_version_of_library("icu"): @@ -669,7 +622,9 @@ else: 'icu4c' + icuSuffix + '/source/icu_i18n', ]) -icuEnv.ShimLibrary(name="icu") +icuEnv.ShimLibrary( + name="icu", +) kmsEnv = env.Clone() if get_option('ssl') == 'on': @@ -687,5 +642,7 @@ if get_option('ssl') == 'on': 'kms-message/kms-message', ]) - kmsEnv.ShimLibrary(name="kms_message") + kmsEnv.ShimLibrary( + name="kms_message", + ) diff --git a/src/third_party/gperftools/SConscript b/src/third_party/gperftools/SConscript index 2025d7a9ea6..8f4f456816a 100644 --- a/src/third_party/gperftools/SConscript +++ b/src/third_party/gperftools/SConscript @@ -128,7 +128,7 @@ env.Library( LIBDEPS_GLOBAL=[ dep for dep in env.get('LIBDEPS_GLOBAL', []) - if not dep.endswith('shim_allocator') + if dep and not dep.endswith('shim_allocator') ], # TODO: Remove when SERVER-48291 is merged into stable build tools DISABLE_ALLOCATOR_SHIM_INJECTION=True, diff --git a/src/third_party/unwind/SConscript b/src/third_party/unwind/SConscript index 983478dd22a..5ea497b0c53 100644 --- a/src/third_party/unwind/SConscript +++ b/src/third_party/unwind/SConscript @@ -131,7 +131,7 @@ env.Library( LIBDEPS_GLOBAL=[ dep for dep in env.get('LIBDEPS_GLOBAL', []) - if not dep.endswith('shim_allocator') + if dep and not (dep.endswith('shim_allocator') or dep.endswith('shim_cxx')) ], # TODO: Remove when SERVER-48291 is merged into stable build tools DISABLE_ALLOCATOR_SHIM_INJECTION=True, |