diff options
| -rw-r--r-- | SConstruct | 42 | ||||
| -rwxr-xr-x | buildscripts/setup_spawnhost_coredump | 83 | ||||
| -rw-r--r-- | etc/scons/mongodbtoolchain_v4_clang.vars | 1 | ||||
| -rw-r--r-- | etc/scons/mongodbtoolchain_v4_gcc.vars | 1 | ||||
| -rw-r--r-- | site_scons/site_tools/gdb_index.py | 134 | ||||
| -rw-r--r-- | site_scons/site_tools/ninja.py | 91 | ||||
| -rw-r--r-- | src/mongo/tools/mongo_tidy_checks/SConscript | 1 |
7 files changed, 313 insertions, 40 deletions
diff --git a/SConstruct b/SConstruct index 7b63eeb60d8..944a7c0feaa 100644 --- a/SConstruct +++ b/SConstruct @@ -856,13 +856,13 @@ def variable_arch_converter(val): return val -def split_dwarf_converter(val): +def bool_var_converter(val, var): try: return to_boolean(val) except ValueError as exc: if val.lower() != "auto": raise ValueError( - f'Invalid SPLIT_DWARF value {s}, must be a boolean-like string or "auto"') from exc + f'Invalid {var} value {s}, must be a boolean-like string or "auto"') from exc return "auto" @@ -1390,7 +1390,20 @@ env_vars.Add( 'SPLIT_DWARF', help= 'Set the boolean (auto, on/off true/false 1/0) to enable gsplit-dwarf (non-Windows). Incompatible with DWARF_VERSION=5', - converter=split_dwarf_converter, + converter=functools.partial(bool_var_converter, var='SPLIT_DWARF'), + default="auto", +) + +env_vars.Add( + 'GDB', + help="Configures the path to the 'gdb' debugger binary.", +) + +env_vars.Add( + 'GDB_INDEX', + help= + 'Set the boolean (auto, on/off true/false 1/0) to enable creation of a gdb_index in binaries.', + converter=functools.partial(bool_var_converter, var='GDB_INDEX'), default="auto", ) @@ -4343,10 +4356,6 @@ def doConfigure(myenv): myenv.AddToCCFLAGSIfSupported('-fno-limit-debug-info') if myenv.ToolchainIs('gcc', 'clang'): - # Usually, --gdb-index is too expensive in big static binaries, but for dynamic - # builds it works well. - if link_model.startswith("dynamic"): - myenv.AddToLINKFLAGSIfSupported('-Wl,--gdb-index') # Pass -gdwarf{32,64} if an explicit value was selected # or defaulted. Fail the build if we can't honor the @@ -5717,13 +5726,18 @@ if get_option('ninja') != 'disabled': env['NINJA_GENERATED_SOURCE_ALIAS_NAME'] = 'generated-sources' -if get_option('separate-debug') == "on" or env.TargetOSIs("windows"): +gdb_index = env.get('GDB_INDEX') +if gdb_index == 'auto' and link_model == 'dynamic': + gdb_index = True - # The current ninja builder can't handle --separate-debug on non-Windows platforms - # like linux or macOS, because they depend on adding extra actions to the link step, - # which cannot be translated into the ninja bulider. - if not env.TargetOSIs("windows") and get_option('ninja') != 'disabled': - env.FatalError("Cannot use --separate-debug with Ninja on non-Windows platforms.") +if gdb_index == True: + gdb_index = Tool('gdb_index') + if gdb_index.exists(env): + gdb_index.generate(env) + elif env.get('GDB_INDEX') != 'auto': + env.FatalError('Could not enable explicit request for gdb index generation.') + +if get_option('separate-debug') == "on" or env.TargetOSIs("windows"): separate_debug = Tool('separate_debug') if not separate_debug.exists(env): @@ -5750,8 +5764,6 @@ if env['SPLIT_DWARF']: env.FatalError( 'Running split dwarf outside of DWARF4 has shown compilation issues when using DWARF5 and gdb index. Disabling this functionality for now. Use SPLIT_DWARF=0 to disable building with split dwarf or use DWARF_VERSION=4 to pin to DWARF version 4.' ) - if env.ToolchainIs('gcc', 'clang'): - env.AddToLINKFLAGSIfSupported('-Wl,--gdb-index') env.Tool('split_dwarf') env["AUTO_ARCHIVE_TARBALL_SUFFIX"] = "tgz" diff --git a/buildscripts/setup_spawnhost_coredump b/buildscripts/setup_spawnhost_coredump index f20fda7c481..0af8b52b3a6 100755 --- a/buildscripts/setup_spawnhost_coredump +++ b/buildscripts/setup_spawnhost_coredump @@ -177,6 +177,89 @@ fi' >> .bash_profile # harmless. gdb expects the .debug files to live adjacent to the physical binary. find bin -type f -perm -o=x -exec ln -s {} . \; + # This script checks a bin for the dwarf version and then generates an index if the bin does not already have an index. eu-readelf is used in place of readelf as it is much faster. + cat > add_index.sh <<EOF +#!/bin/bash + +set -o pipefail + +target_dir="\$(dirname \${1})" + +dwarf_version="\$($TOOLCHAIN_ROOT/bin/eu-readelf --debug-dump=info \$1 | grep --line-buffered -E '^\s+Version:' | head -1 | awk -F, '{print(\$1)}' | awk '{print(\$2)}')" + +if [[ \$dwarf_version == 5 ]] +then + $TOOLCHAIN_ROOT/bin/gdb --batch-silent --quiet --nx --eval-command "save gdb-index -dwarf-5 \$target_dir" \$1 + if [ -f \${1}.debug_names ] + then + $TOOLCHAIN_ROOT/bin/objcopy --dump-section .debug_str=\${1}.debug_str.new \$1 + cat \${1}.debug_str >>\${1}.debug_str.new + $TOOLCHAIN_ROOT/bin/objcopy --add-section .debug_names=\${1}.debug_names --set-section-flags .debug_names=readonly --update-section .debug_str=\${1}.debug_str.new \${1} \${1} + rm -f \${1}.debug_names \${1}.debug_str.new \${1}.debug_str + fi + +elif [[ \$dwarf_version == 4 ]] +then + $TOOLCHAIN_ROOT/bin/gdb --batch-silent --quiet --nx --eval-command "save gdb-index \$target_dir" \$1 + if [ -f \${1}.gdb-index ] + then + $TOOLCHAIN_ROOT/bin/objcopy --add-section .gdb_index=\${1}.gdb-index --set-section-flags .gdb_index=readonly \${1} \${1} + rm -f \${1}.gdb-index + fi + +else + echo "Can't determine debug info for \$1" +fi +EOF + + # After creating the index file in a separate debug file, the debuglink CRC + # is no longer value, this will simply recreate the debuglink and therefore + # update the CRC to match. + cat > recalc_debuglink.sh <<EOF +#!/bin/bash + +set -o pipefail + +debuglink="\$($TOOLCHAIN_ROOT/bin/eu-readelf -S \$1 | grep '.gnu_debuglink')" + +if [ ! -z "\$debuglink" ] +then + $TOOLCHAIN_ROOT/bin/objcopy --remove-section ".gnu_debuglink" \$1 + $TOOLCHAIN_ROOT/bin/objcopy --add-gnu-debuglink "$(basename \$1).debug" \$1 +fi +EOF + +# this script creates a symlink in the toolchain lib/debug directory which is in the +# build-id format. This allows gdb to load the separate debug file and skip CRC +# checking. +cat > create_build_id_links.sh <<EOF +#!/bin/bash + +set -o pipefail + +build_id="\$($TOOLCHAIN_ROOT/bin/eu-readelf -n \$1 | grep 'Build ID:' | awk -F: '{print \$2}' | sed 's/ *//')" +gdb_debug_dir="\$(readlink $TOOLCHAIN_ROOT/bin/gdb)" +gdb_debug_dir="\$(dirname \$gdb_debug_dir)" +gdb_debug_dir="\$(dirname \$gdb_debug_dir)/lib/debug/.build-id/\${build_id:0:2}" +gdb_debug_file="\${build_id:2}.debug" +mkdir -p \$gdb_debug_dir +ln -s \$PWD/\$1 \$gdb_debug_dir/\$gdb_debug_file +EOF + + chmod +x ./add_index.sh + chmod +x ./recalc_debuglink.sh + chmod +x ./create_build_id_links.sh + cpus=$(getconf _NPROCESSORS_ONLN) + + # notice we don't search lib directory as we assume dynamic builds build the index during + # the build. + find bin -type f -perm -o=x | xargs --max-args=1 --max-procs=$cpus ./add_index.sh + find bin -type f -perm -o=x | xargs --max-args=1 --max-procs=$cpus ./recalc_debuglink.sh + + # This script constructs symblinks based off the build-id so GDB can skip the crc check + # normally performed during .gnu_debuglink loading. + find bin lib -name "*.debug" -type f -perm -o=x | xargs --max-args=1 --max-procs=$cpus ./create_build_id_links.sh + # Boost-Pretty-Printer supports auto-detection for the boost version but relies on the system # installed version of boost. To avoid this behavior we explicitly specify the boost_version. # Moreover, the most recent version of boost that Boost-Pretty-Printer verifies it supports is diff --git a/etc/scons/mongodbtoolchain_v4_clang.vars b/etc/scons/mongodbtoolchain_v4_clang.vars index 957effdc764..a134f88d5ed 100644 --- a/etc/scons/mongodbtoolchain_v4_clang.vars +++ b/etc/scons/mongodbtoolchain_v4_clang.vars @@ -35,6 +35,7 @@ CC = os.path.join(toolchain_bindir, 'clang') CXX = os.path.join(toolchain_bindir, 'clang++') DWP = os.path.join(toolchain_bindir, 'dwp') READELF = os.path.join(toolchain_bindir, 'readelf') +GDB = os.path.join(toolchain_bindir, 'gdb') try: AR = subprocess.check_output([CXX, '-print-prog-name=ar']).decode('utf-8').strip() diff --git a/etc/scons/mongodbtoolchain_v4_gcc.vars b/etc/scons/mongodbtoolchain_v4_gcc.vars index 154eb91713c..6f70522187a 100644 --- a/etc/scons/mongodbtoolchain_v4_gcc.vars +++ b/etc/scons/mongodbtoolchain_v4_gcc.vars @@ -34,6 +34,7 @@ CC = os.path.join(toolchain_bindir, 'gcc') CXX = os.path.join(toolchain_bindir, 'g++') DWP = os.path.join(toolchain_bindir, 'dwp') READELF = os.path.join(toolchain_bindir, 'readelf') +GDB = os.path.join(toolchain_bindir, 'gdb') try: AR = subprocess.check_output([CXX, '-print-prog-name=ar']).decode('utf-8').strip() diff --git a/site_scons/site_tools/gdb_index.py b/site_scons/site_tools/gdb_index.py new file mode 100644 index 00000000000..5850b8560d3 --- /dev/null +++ b/site_scons/site_tools/gdb_index.py @@ -0,0 +1,134 @@ +# Copyright 2020 MongoDB Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import SCons + + +def _update_builder(env, builder): + + verbose = '' if env.Verbose() else None + + base_action = builder.action + if not isinstance(base_action, SCons.Action.ListAction): + base_action = SCons.Action.ListAction([base_action]) + + # There are cases were a gdb-index file is NOT generated from gdb 'save gdb-index' command, + # mostly shim libraries where there is no code in the library, and the following Actions would + # then fail. The files are created make sure there is always a file to operate on, if its an + # empty file then the following actions are basically NOOP, and its cleaner then writing + # conditions into each action. + + # Because this is all taking under one task, the list action will always run all actions if + # the target is out of date. So the gdb-index files would always be regenerated, and there is + # no value in keeping them around, it will just waste disk space. Therefore they should be + # removed as if they never existed from the task. The build system doesn't need to know about + # them. + if env.get('DWARF_VERSION') <= 4: + base_action.list.extend([ + SCons.Action.Action( + 'touch ${TARGET}.gdb-index', + verbose, + ), + SCons.Action.Action( + '$GDB --batch-silent --quiet --nx --eval-command "save gdb-index ${TARGET.dir}" $TARGET', + "$GDB_INDEX_GEN_INDEX_STR", + ), + SCons.Action.Action( + '$OBJCOPY --add-section .gdb_index=${TARGET}.gdb-index --set-section-flags .gdb_index=readonly ${TARGET} ${TARGET}', + "$GDB_INDEX_ADD_SECTION_STR", + ), + SCons.Action.Action( + 'rm -f ${TARGET}.gdb-index', + verbose, + ), + ]) + else: + base_action.list.extend([ + SCons.Action.Action( + 'touch ${TARGET}.debug_names ${TARGET}.debug_str', + verbose, + ), + SCons.Action.Action( + '$GDB --batch-silent --quiet --nx --eval-command "save gdb-index -dwarf-5 ${TARGET.dir}" $TARGET', + "$GDB_INDEX_GEN_INDEX_STR", + ), + SCons.Action.Action( + '$OBJCOPY --dump-section .debug_str=${TARGET}.debug_str.new $TARGET', + verbose, + ), + SCons.Action.Action( + 'cat ${TARGET}.debug_str >>${TARGET}.debug_str.new', + verbose, + ), + SCons.Action.Action( + '$OBJCOPY --add-section .debug_names=${TARGET}.debug_names --set-section-flags .debug_names=readonly --update-section .debug_str=${TARGET}.debug_str.new ${TARGET} ${TARGET}', + "$GDB_INDEX_ADD_SECTION_STR", + ), + SCons.Action.Action( + 'rm -f ${TARGET}.debug_names ${TARGET}.debug_str.new ${TARGET}.debug_str', + verbose, + ), + ]) + + builder.action = base_action + + +def generate(env): + if env.get("OBJCOPY", None) is None: + env["OBJCOPY"] = env.WhereIs("objcopy") + if env.get("GDB", None) is None: + env["GDB"] = env.WhereIs("gdb") + + if not env.Verbose(): + env.Append( + GDB_INDEX_GEN_INDEX_STR="Using $GDB to generate index for $TARGET", + GDB_INDEX_ADD_SECTION_STR="Adding index sections into $TARGET", + ) + + for builder in ["Program", "SharedLibrary", "LoadableModule"]: + _update_builder(env, env["BUILDERS"][builder]) + + +def exists(env): + result = False + if env.TargetOSIs("posix"): + objcopy = env.get("OBJCOPY", None) or env.WhereIs("objcopy") + gdb = env.get("GDB", None) or env.WhereIs("gdb") + try: + dwarf_version = int(env.get('DWARF_VERSION')) + except ValueError: + dwarf_version = None + + unset_vars = [] + if not objcopy: + unset_vars += ['OBJCOPY'] + if not gdb: + unset_vars += ['GDB'] + if not dwarf_version: + unset_vars += ['DWARF_VERSION'] + + if not unset_vars: + print("Enabled generation of gdb index into binaries.") + result = True + else: + print(f"Disabled generation gdb index because {', '.join(unset_vars)} were not set.") + return result diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py index a55205fc612..78c9e87e66a 100644 --- a/site_scons/site_tools/ninja.py +++ b/site_scons/site_tools/ninja.py @@ -267,6 +267,35 @@ def generate_depfile(env, node, dependencies): f.write(depfile_contents) +def _extract_cmdstr_for_list_action(ninja_build_list): + cmdline = "" + for cmd in ninja_build_list: + + # Occasionally a command line will expand to a + # whitespace only string (i.e. ' '). Which is not a + # valid command but does not trigger the empty command + # condition if not cmdstr. So here we trim the whitespace + # to make strings like the above become empty strings and + # so they will be skipped. + cmdstr = cmd["variables"]["cmd"].strip() + if not cmdstr: + continue + + # Skip duplicate commands + if cmdstr in cmdline: + continue + + if cmdline: + cmdline += " && " + + cmdline += cmdstr + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() + + return cmdline + + class SConsToNinjaTranslator: """Translates SCons Actions into Ninja build objects.""" @@ -384,31 +413,8 @@ class SConsToNinjaTranslator: all_outputs = list({output for build in results for output in build["outputs"]}) dependencies = list({dep for build in results for dep in build["implicit"]}) - if results[0]["rule"] == "CMD": - cmdline = "" - for cmd in results: - - # Occasionally a command line will expand to a - # whitespace only string (i.e. ' '). Which is not a - # valid command but does not trigger the empty command - # condition if not cmdstr. So here we strip preceding - # and proceeding whitespace to make strings like the - # above become empty strings and so will be skipped. - cmdstr = cmd["variables"]["cmd"].strip() - if not cmdstr: - continue - - # Skip duplicate commands - if cmdstr in cmdline: - continue - - if cmdline: - cmdline += " && " - - cmdline += cmdstr - - # Remove all preceding and proceeding whitespace - cmdline = cmdline.strip() + if all([result['rule'] == 'CMD' for result in results]): + cmdline = _extract_cmdstr_for_list_action(results) # Make sure we didn't generate an empty cmdline if cmdline: @@ -431,6 +437,34 @@ class SConsToNinjaTranslator: return ninja_build + elif results[0]["rule"] == "LINK" and all( + [result['rule'] == 'CMD' for result in results[1:]]): + cmdline = _extract_cmdstr_for_list_action(results[1:]) + + # Make sure we didn't generate an empty cmdline + if cmdline: + + env = node.env if node.env else self.env + sources = [get_path(src_file(s)) for s in node.sources] + + ninja_build = results[0] + + ninja_build.update({ + "outputs": all_outputs, + "rule": "LINK_CHAINED_CMD", + "implicit": dependencies, + }) + + ninja_build['variables'].update({ + "cmd": cmdline, + "env": get_command_env(env, all_outputs, sources), + }) + + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["pool"] + + return ninja_build + elif results[0]["rule"] == "phony": return { "outputs": all_outputs, @@ -535,6 +569,13 @@ class NinjaState: "rspfile_content": "$rspc", "pool": "local_pool", }, + "LINK_CHAINED_CMD": { + "command": "$env$LINK @$out.rsp && $cmd", + "description": "Linked $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, "AR": { "command": "$env$AR @$out.rsp", "description": "Archived $out", diff --git a/src/mongo/tools/mongo_tidy_checks/SConscript b/src/mongo/tools/mongo_tidy_checks/SConscript index fdface30c39..d89da3e5926 100644 --- a/src/mongo/tools/mongo_tidy_checks/SConscript +++ b/src/mongo/tools/mongo_tidy_checks/SConscript @@ -71,6 +71,7 @@ env['CCFLAGS'] = [ '-Woverloaded-virtual', '-fno-strict-aliasing', '-O3', + '-g2', '-DNDEBUG', '-fPIC', '-fno-exceptions', |