SERVER-27380 Re-enable the thin archive tool

This refactors the thin_archive tool to use emitters and scanners
to note that when linking to a thin archive, you must also depend on
the children of that thin archive. Failing to do so is an error,
because a changed .o does not necessarily lead to a different .a,
which would subvert the SCons dependency mechanism.

This also includes a refactoring of the ABILINK tool to use a similar
mechanism, to achieve the opposite effect. For ABILINK, we want to
depend not on the actual .so, but on the hash of its abidw result. We
use emitters, actions, and scanners to produce an associated .abidw
file for each .so we build, and then update the scanner to depend on
the .abidw of our libraries, not the library itself. This allows us to
elide needless relinks.

2 files changed, 74 insertions, 60 deletions

diff --git a/site_scons/site_tools/abilink.py b/site_scons/site_tools/abilink.py
index 6ca2c7dc6b8..4e4fdcd6310 100644
--- a/site_scons/site_tools/abilink.py
+++ b/site_scons/site_tools/abilink.py
@@ -15,6 +15,11 @@
 import SCons
 import subprocess
 
+# TODO: Make a SUFF variable for the suffix to write to
+# TODO: Prevent using abilink when -gsplit-dwarf is in play, since it doesn't work
+# TODO: Make a variable for the md5sum utility (allow any hasher)
+# TODO: Add an ABILINKCOM variable to the Action, so it can be silenced.
+
 def _detect(env):
     try:
         abidw = env['ABIDW']
@@ -26,53 +31,52 @@ def _detect(env):
 
     return env.WhereIs('abidw')
 
-def generate(env):
-
-    class AbilinkNode(SCons.Node.FS.File):
-        def __init__(self, name, directory, fs):
-            SCons.Node.FS.File.__init__(self, name, directory, fs)
-
-        def get_contents(self):
-            if not self.rexists():
-                return ''
-
-            fname = self.rfile().abspath
-            contents = None
-
-            try:
-                # TODO: If there were python bindings for libabigail, we
-                # could avoid the shell out (and probably be faster, as we
-                # could get exactly the information we want).
-                contents = subprocess.check_output([env.subst('$ABIDW'), fname])
-            except subprocess.CalledProcessError, e:
-                # ABIDW sometimes fails. In that case, log an error
-                # and fall back to the normal contents
-                print "WARNING: ABIDW failed for target %s, please file a bug report" % fname
-                try:
-                    contents = open(fname, "rb").read()
-                except EnvironmentError, e:
-                    if not e.filename:
-                        e.filename = fname
-                    raise
+def _add_emitter(builder):
+    base_emitter = builder.emitter
 
-            return contents
+    def new_emitter(target, source, env):
+        new_targets = []
+        for t in target:
+            abidw = str(t) + ".abidw"
+            abidw = (t.builder.target_factory or env.File)(abidw)
+            new_targets.append(abidw)
+            setattr(t.attributes, "abidw", abidw)
+        targets = target + new_targets
+        return (targets, source)
 
-        def get_content_hash(self):
-            return SCons.Util.MD5signature(self.get_contents())
+    new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter])
+    builder.emitter = new_emitter
 
-    env['ABIDW'] = _detect(env)
+def _add_scanner(builder):
+    old_scanner = builder.target_scanner
+    path_function = old_scanner.path_function
 
-    def ShlibNode(env, name, directory = None, create = 1):
-        return env.fs._lookup(env.subst(name), directory, AbilinkNode, create)
+    def new_scanner(node, env, path):
+        old_results = old_scanner(node, env, path)
+        new_results = []
+        for base in old_results:
+            abidw = getattr(env.Entry(base).attributes, "abidw", None)
+            new_results.append(abidw if abidw else base)
+        return new_results
 
-    env.AddMethod(ShlibNode, 'ShlibNode')
+    builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function)
 
-    def shlib_target_factory(arg):
-        return env.ShlibNode(arg)
-
-    env['BUILDERS']['SharedLibrary'].target_factory = shlib_target_factory
-    env['BUILDERS']['LoadableModule'].target_factory = shlib_target_factory
+def _add_action(builder):
+    actions = builder.action
+    builder.action = actions + SCons.Action.Action("$ABIDW $TARGET | md5sum > ${TARGET}.abidw")
 
 def exists(env):
     result = _detect(env) != None
     return result
+
+def generate(env):
+
+    if not exists(env):
+        return
+
+    builder = env['BUILDERS']['SharedLibrary']
+    _add_emitter(builder)
+    _add_action(builder)
+    _add_scanner(builder)
+    _add_scanner(env['BUILDERS']['Program'])
+    _add_scanner(env['BUILDERS']['LoadableModule'])
diff --git a/site_scons/site_tools/thin_archive.py b/site_scons/site_tools/thin_archive.py
index b4bc5f97d6f..511c0ef6e54 100644
--- a/site_scons/site_tools/thin_archive.py
+++ b/site_scons/site_tools/thin_archive.py
@@ -45,32 +45,35 @@ def exists(env):
 
     return bool(isgnu)
 
+def _add_emitter(builder):
+    base_emitter = builder.emitter
 
-def generate(env):
-    if not exists(env):
-        return
-
-    class ThinArchiveNode(SCons.Node.FS.File):
-        def __init__(self, name, directory, fs):
-            SCons.Node.FS.File.__init__(self, name, directory, fs)
-
-        def get_contents(self):
-            child_sigs = sorted([child.get_csig() for child in self.children()])
-            return ''.join(child_sigs)
-
-        def get_content_hash(self):
-            return SCons.Util.MD5signature(self.get_contents())
+    def new_emitter(target, source, env):
+        for t in target:
+            setattr(t.attributes, "thin_archive", True)
+        return (target, source)
 
+    new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter])
+    builder.emitter = new_emitter
 
-    def _ThinArchiveNode(env, name, directory = None, create = 1):
-        return env.fs._lookup(env.subst(name), directory, ThinArchiveNode, create)
+def _add_scanner(builder):
+    old_scanner = builder.target_scanner
+    path_function = old_scanner.path_function
 
-    env.AddMethod(_ThinArchiveNode, 'ThinArchiveNode')
+    def new_scanner(node, env, path):
+        old_results = old_scanner(node, env, path)
+        new_results = []
+        for base in old_results:
+            new_results.append(base)
+            if getattr(env.Entry(base).attributes, "thin_archive", None):
+                new_results.extend(base.children())
+        return new_results
 
-    def archive_target_factory(arg):
-        return env.ThinArchiveNode(arg)
+    builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, path_function=path_function)
 
-    env['BUILDERS']['StaticLibrary'].target_factory = archive_target_factory
+def generate(env):
+    if not exists(env):
+        return
 
     env['ARFLAGS'] = SCons.Util.CLVar([arflag if arflag != "rc" else "rcsTD" for arflag in env['ARFLAGS']])
 
@@ -80,3 +83,10 @@ def generate(env):
     # Disable running ranlib, since we added 's' above
     env['RANLIBCOM'] = noop_action
     env['RANLIBCOMSTR'] = 'Skipping ranlib for thin archive $TARGET'
+
+    builder = env['BUILDERS']['StaticLibrary']
+    _add_emitter(builder)
+
+    _add_scanner(env['BUILDERS']['SharedLibrary'])
+    _add_scanner(env['BUILDERS']['LoadableModule'])
+    _add_scanner(env['BUILDERS']['Program'])