From 3e1461b80c42deda1f6b9478e2e574c6d88052f2 Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Sat, 20 May 2017 09:02:32 -0400 Subject: SERVER-27380 Re-enable the thin archive tool This refactors the thin_archive tool to use emitters and scanners to note that when linking to a thin archive, you must also depend on the children of that thin archive. Failing to do so is an error, because a changed .o does not necessarily lead to a different .a, which would subvert the SCons dependency mechanism. This also includes a refactoring of the ABILINK tool to use a similar mechanism, to achieve the opposite effect. For ABILINK, we want to depend not on the actual .so, but on the hash of its abidw result. We use emitters, actions, and scanners to produce an associated .abidw file for each .so we build, and then update the scanner to depend on the .abidw of our libraries, not the library itself. This allows us to elide needless relinks. --- SConstruct | 21 ++++++--- site_scons/site_tools/abilink.py | 84 ++++++++++++++++++----------------- site_scons/site_tools/thin_archive.py | 50 ++++++++++++--------- 3 files changed, 88 insertions(+), 67 deletions(-) diff --git a/SConstruct b/SConstruct index 6222566afe7..e4813b3de65 100644 --- a/SConstruct +++ b/SConstruct @@ -1189,13 +1189,6 @@ env['BUILDERS']['LibraryObject'] = env['BUILDERS']['StaticObject'] if link_model.startswith("dynamic"): - # Add in the abi linking tool if the user requested and it is - # supported on this platform. - if env.get('ABIDW'): - abilink = Tool('abilink') - if abilink.exists(env): - abilink(env) - # Redirect the 'Library' target, which we always use instead of 'StaticLibrary' for things # that can be built in either mode, to point to SharedLibrary. env['BUILDERS']['Library'] = env['BUILDERS']['SharedLibrary'] @@ -1359,6 +1352,20 @@ if env['_LIBDEPS'] == '$_LIBDEPS_OBJS': libdeps.setup_environment(env, emitting_shared=(link_model.startswith("dynamic"))) +# Both the abidw tool and the thin archive tool must be loaded after +# libdeps, so that the scanners they inject can see the library +# dependencies added by libdeps. +if link_model.startswith("dynamic"): + # Add in the abi linking tool if the user requested and it is + # supported on this platform. + if env.get('ABIDW'): + abilink = Tool('abilink') + if abilink.exists(env): + abilink(env) + +if env['_LIBDEPS'] == '$_LIBDEPS_LIBS': + env.Tool('thin_archive') + if env.TargetOSIs('linux', 'freebsd', 'openbsd'): env['LINK_LIBGROUP_START'] = '-Wl,--start-group' env['LINK_LIBGROUP_END'] = '-Wl,--end-group' diff --git a/site_scons/site_tools/abilink.py b/site_scons/site_tools/abilink.py index 6ca2c7dc6b8..4e4fdcd6310 100644 --- a/site_scons/site_tools/abilink.py +++ b/site_scons/site_tools/abilink.py @@ -15,6 +15,11 @@ import SCons import subprocess +# TODO: Make a SUFF variable for the suffix to write to +# TODO: Prevent using abilink when -gsplit-dwarf is in play, since it doesn't work +# TODO: Make a variable for the md5sum utility (allow any hasher) +# TODO: Add an ABILINKCOM variable to the Action, so it can be silenced. + def _detect(env): try: abidw = env['ABIDW'] @@ -26,53 +31,52 @@ def _detect(env): return env.WhereIs('abidw') -def generate(env): - - class AbilinkNode(SCons.Node.FS.File): - def __init__(self, name, directory, fs): - SCons.Node.FS.File.__init__(self, name, directory, fs) - - def get_contents(self): - if not self.rexists(): - return '' - - fname = self.rfile().abspath - contents = None - - try: - # TODO: If there were python bindings for libabigail, we - # could avoid the shell out (and probably be faster, as we - # could get exactly the information we want). - contents = subprocess.check_output([env.subst('$ABIDW'), fname]) - except subprocess.CalledProcessError, e: - # ABIDW sometimes fails. In that case, log an error - # and fall back to the normal contents - print "WARNING: ABIDW failed for target %s, please file a bug report" % fname - try: - contents = open(fname, "rb").read() - except EnvironmentError, e: - if not e.filename: - e.filename = fname - raise +def _add_emitter(builder): + base_emitter = builder.emitter - return contents + def new_emitter(target, source, env): + new_targets = [] + for t in target: + abidw = str(t) + ".abidw" + abidw = (t.builder.target_factory or env.File)(abidw) + new_targets.append(abidw) + setattr(t.attributes, "abidw", abidw) + targets = target + new_targets + return (targets, source) - def get_content_hash(self): - return SCons.Util.MD5signature(self.get_contents()) + new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter]) + builder.emitter = new_emitter - env['ABIDW'] = _detect(env) +def _add_scanner(builder): + old_scanner = builder.target_scanner + path_function = old_scanner.path_function - def ShlibNode(env, name, directory = None, create = 1): - return env.fs._lookup(env.subst(name), directory, AbilinkNode, create) + def new_scanner(node, env, path): + old_results = old_scanner(node, env, path) + new_results = [] + for base in old_results: + abidw = getattr(env.Entry(base).attributes, "abidw", None) + new_results.append(abidw if abidw else base) + return new_results - env.AddMethod(ShlibNode, 'ShlibNode') + builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function) - def shlib_target_factory(arg): - return env.ShlibNode(arg) - - env['BUILDERS']['SharedLibrary'].target_factory = shlib_target_factory - env['BUILDERS']['LoadableModule'].target_factory = shlib_target_factory +def _add_action(builder): + actions = builder.action + builder.action = actions + SCons.Action.Action("$ABIDW $TARGET | md5sum > ${TARGET}.abidw") def exists(env): result = _detect(env) != None return result + +def generate(env): + + if not exists(env): + return + + builder = env['BUILDERS']['SharedLibrary'] + _add_emitter(builder) + _add_action(builder) + _add_scanner(builder) + _add_scanner(env['BUILDERS']['Program']) + _add_scanner(env['BUILDERS']['LoadableModule']) diff --git a/site_scons/site_tools/thin_archive.py b/site_scons/site_tools/thin_archive.py index b4bc5f97d6f..511c0ef6e54 100644 --- a/site_scons/site_tools/thin_archive.py +++ b/site_scons/site_tools/thin_archive.py @@ -45,32 +45,35 @@ def exists(env): return bool(isgnu) +def _add_emitter(builder): + base_emitter = builder.emitter -def generate(env): - if not exists(env): - return - - class ThinArchiveNode(SCons.Node.FS.File): - def __init__(self, name, directory, fs): - SCons.Node.FS.File.__init__(self, name, directory, fs) - - def get_contents(self): - child_sigs = sorted([child.get_csig() for child in self.children()]) - return ''.join(child_sigs) - - def get_content_hash(self): - return SCons.Util.MD5signature(self.get_contents()) + def new_emitter(target, source, env): + for t in target: + setattr(t.attributes, "thin_archive", True) + return (target, source) + new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter]) + builder.emitter = new_emitter - def _ThinArchiveNode(env, name, directory = None, create = 1): - return env.fs._lookup(env.subst(name), directory, ThinArchiveNode, create) +def _add_scanner(builder): + old_scanner = builder.target_scanner + path_function = old_scanner.path_function - env.AddMethod(_ThinArchiveNode, 'ThinArchiveNode') + def new_scanner(node, env, path): + old_results = old_scanner(node, env, path) + new_results = [] + for base in old_results: + new_results.append(base) + if getattr(env.Entry(base).attributes, "thin_archive", None): + new_results.extend(base.children()) + return new_results - def archive_target_factory(arg): - return env.ThinArchiveNode(arg) + builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function) - env['BUILDERS']['StaticLibrary'].target_factory = archive_target_factory +def generate(env): + if not exists(env): + return env['ARFLAGS'] = SCons.Util.CLVar([arflag if arflag != "rc" else "rcsTD" for arflag in env['ARFLAGS']]) @@ -80,3 +83,10 @@ def generate(env): # Disable running ranlib, since we added 's' above env['RANLIBCOM'] = noop_action env['RANLIBCOMSTR'] = 'Skipping ranlib for thin archive $TARGET' + + builder = env['BUILDERS']['StaticLibrary'] + _add_emitter(builder) + + _add_scanner(env['BUILDERS']['SharedLibrary']) + _add_scanner(env['BUILDERS']['LoadableModule']) + _add_scanner(env['BUILDERS']['Program']) -- cgit v1.2.1