SERVER-33911 Implement collapsed library builds under a new link model

4 files changed, 127 insertions, 182 deletions

diff --git a/site_scons/libdeps.py b/site_scons/libdeps.py
index f002c4f067e..5bc530ba1e7 100644
--- a/site_scons/libdeps.py
+++ b/site_scons/libdeps.py
@@ -63,14 +63,25 @@ missing_syslibdep = 'MISSING_LIBDEP_'
 class dependency(object):
     Public, Private, Interface = range(3)
 
-    def __init__(self, value, dynamic, deptype):
+    def __init__(self, value, deptype):
         self.target_node = value
-        # In static mode, all dependencies are public
-        self.dependency_type = deptype if dynamic else dependency.Public
+        self.dependency_type = deptype
 
     def __str__(self):
         return str(self.target_node)
 
+dependency_visibility_ignored = {
+    dependency.Public : dependency.Public,
+    dependency.Private : dependency.Public,
+    dependency.Interface : dependency.Public,
+}
+
+dependency_visibility_honored = {
+    dependency.Public : dependency.Public,
+    dependency.Private : dependency.Private,
+    dependency.Interface : dependency.Interface,
+}
+
 class DependencyCycleError(SCons.Errors.UserError):
     """Exception representing a cycle discovered in library dependencies."""
 
@@ -223,128 +234,78 @@ def __append_direct_libdeps(node, prereq_nodes):
         node.attributes.libdeps_direct = []
     node.attributes.libdeps_direct.extend(prereq_nodes)
 
-def libdeps_emitter(target, source, env):
-    """SCons emitter that takes values from the LIBDEPS environment variable and
-    converts them to File node objects, binding correct path information into
-    those File objects.
+def make_libdeps_emitter(dependency_builder, dependency_map=dependency_visibility_ignored):
 
-    Emitters run on a particular "target" node during the initial execution of
-    the SConscript file, rather than during the later build phase.  When they
-    run, the "env" environment's working directory information is what you
-    expect it to be -- that is, the working directory is considered to be the
-    one that contains the SConscript file.  This allows specification of
-    relative paths to LIBDEPS elements.
+    def libdeps_emitter(target, source, env):
+        """SCons emitter that takes values from the LIBDEPS environment variable and
+        converts them to File node objects, binding correct path information into
+        those File objects.
 
-    This emitter also adds LIBSUFFIX and LIBPREFIX appropriately.
+        Emitters run on a particular "target" node during the initial execution of
+        the SConscript file, rather than during the later build phase.  When they
+        run, the "env" environment's working directory information is what you
+        expect it to be -- that is, the working directory is considered to be the
+        one that contains the SConscript file.  This allows specification of
+        relative paths to LIBDEPS elements.
 
-    NOTE: For purposes of LIBDEPS_DEPENDENTS propagation, only the first member
-    of the "target" list is made a prerequisite of the elements of LIBDEPS_DEPENDENTS.
-    """
+        This emitter also adds LIBSUFFIX and LIBPREFIX appropriately.
 
-    lib_builder = env['BUILDERS']['StaticLibrary']
-    lib_node_factory = lib_builder.target_factory or env.File
-
-    prog_builder = env['BUILDERS']['Program']
-    prog_node_factory = prog_builder.target_factory or env.File
-
-    prereqs = [dependency(l, False, dependency.Public) for l in env.get(libdeps_env_var, []) if l]
-    prereqs.extend(dependency(l, False, dependency.Interface) for l in env.get(libdeps_env_var + '_INTERFACE', []) if l)
-    prereqs.extend(dependency(l, False, dependency.Private) for l in env.get(libdeps_env_var + '_PRIVATE', []) if l)
-
-    for prereq in prereqs:
-        prereqWithIxes = SCons.Util.adjustixes(
-            prereq.target_node, lib_builder.get_prefix(env), lib_builder.get_suffix(env))
-        prereq.target_node = lib_node_factory(prereqWithIxes)
-
-    for t in target:
-        # target[0] must be a Node and not a string, or else libdeps will fail to
-        # work properly.
-        __append_direct_libdeps(t, prereqs)
-
-    for dependent in env.get('LIBDEPS_DEPENDENTS', []):
-        if dependent is None:
-            continue
-
-        # Ignore any tuple'd in visibility override.
-        if isinstance(dependent, tuple):
-            dependent = dependent[0]
-
-        dependentWithIxes = SCons.Util.adjustixes(
-            dependent, lib_builder.get_prefix(env), lib_builder.get_suffix(env))
-        dependentNode = lib_node_factory(dependentWithIxes)
-        __append_direct_libdeps(dependentNode, [dependency(target[0], False, dependency.Public)])
-
-    for dependent in env.get('PROGDEPS_DEPENDENTS', []):
-        if dependent is None:
-            continue
-        dependentWithIxes = SCons.Util.adjustixes(
-            dependent, prog_builder.get_prefix(env), prog_builder.get_suffix(env))
-        dependentNode = prog_node_factory(dependentWithIxes)
-        __append_direct_libdeps(dependentNode, [dependency(target[0], False, dependency.Public)])
-
-    return target, source
-
-def shlibdeps_emitter(target, source, env):
-    """SCons emitter that takes values from the LIBDEPS environment variable and
-    converts them to File node objects, binding correct path information into
-    those File objects.
-
-    Emitters run on a particular "target" node during the initial execution of
-    the SConscript file, rather than during the later build phase.  When they
-    run, the "env" environment's working directory information is what you
-    expect it to be -- that is, the working directory is considered to be the
-    one that contains the SConscript file.  This allows specification of
-    relative paths to LIBDEPS elements.
-
-    This emitter also adds LIBSUFFIX and LIBPREFIX appropriately.
-
-    NOTE: For purposes of LIBDEPS_DEPENDENTS propagation, only the first member
-    of the "target" list is made a prerequisite of the elements of LIBDEPS_DEPENDENTS.
-    """
+        NOTE: For purposes of LIBDEPS_DEPENDENTS propagation, only the first member
+        of the "target" list is made a prerequisite of the elements of LIBDEPS_DEPENDENTS.
+        """
+
+        lib_builder = env['BUILDERS'][dependency_builder]
+        lib_node_factory = lib_builder.target_factory or env.File
+
+        prog_builder = env['BUILDERS']['Program']
+        prog_node_factory = prog_builder.target_factory or env.File
 
-    lib_builder = env['BUILDERS']['SharedLibrary']
-    lib_node_factory = lib_builder.target_factory or env.File
+        prereqs = [dependency(l, dependency_map[dependency.Public]) for l in env.get(libdeps_env_var, []) if l]
+        prereqs.extend(dependency(l, dependency_map[dependency.Interface]) for l in env.get(libdeps_env_var + '_INTERFACE', []) if l)
+        prereqs.extend(dependency(l, dependency_map[dependency.Private]) for l in env.get(libdeps_env_var + '_PRIVATE', []) if l)
 
-    prog_builder = env['BUILDERS']['Program']
-    prog_node_factory = prog_builder.target_factory or env.File
+        for prereq in prereqs:
+            prereqWithIxes = SCons.Util.adjustixes(
+                prereq.target_node, lib_builder.get_prefix(env), lib_builder.get_suffix(env))
+            prereq.target_node = lib_node_factory(prereqWithIxes)
 
-    prereqs = [dependency(l, True, dependency.Public) for l in env.get(libdeps_env_var, []) if l]
-    prereqs.extend(dependency(l, True, dependency.Interface) for l in env.get(libdeps_env_var + '_INTERFACE', []) if l)
-    prereqs.extend(dependency(l, True, dependency.Private) for l in env.get(libdeps_env_var + '_PRIVATE', []) if l)
+        for t in target:
+            # target[0] must be a Node and not a string, or else libdeps will fail to
+            # work properly.
+            __append_direct_libdeps(t, prereqs)
 
-    for prereq in prereqs:
-        prereqWithIxes = SCons.Util.adjustixes(
-            prereq.target_node, lib_builder.get_prefix(env), lib_builder.get_suffix(env))
-        prereq.target_node = lib_node_factory(prereqWithIxes)
+        for dependent in env.get('LIBDEPS_DEPENDENTS', []):
+            if dependent is None:
+                continue
 
-    for t in target:
-        # target[0] must be a Node and not a string, or else libdeps will fail to
-        # work properly.
-        __append_direct_libdeps(t, prereqs)
+            visibility = dependency.Private
+            if isinstance(dependent, tuple):
+                visibility = dependent[1]
+                dependent = dependent[0]
 
-    for dependent in env.get('LIBDEPS_DEPENDENTS', []):
-        if dependent is None:
-            continue
+            dependentWithIxes = SCons.Util.adjustixes(
+                dependent, lib_builder.get_prefix(env), lib_builder.get_suffix(env))
+            dependentNode = lib_node_factory(dependentWithIxes)
+            __append_direct_libdeps(dependentNode, [dependency(target[0], dependency_map[visibility])])
 
-        visibility = dependency.Private
-        if isinstance(dependent, tuple):
-            visibility = dependent[1]
-            dependent = dependent[0]
+        for dependent in env.get('PROGDEPS_DEPENDENTS', []):
+            if dependent is None:
+                continue
 
-        dependentWithIxes = SCons.Util.adjustixes(
-            dependent, lib_builder.get_prefix(env), lib_builder.get_suffix(env))
-        dependentNode = lib_node_factory(dependentWithIxes)
-        __append_direct_libdeps(dependentNode, [dependency(target[0], True, visibility)])
+            visibility = dependency.Public
+            if isinstance(dependent, tuple):
+                # TODO: Error here? Non-public PROGDEPS_DEPENDENTS probably are meaningless
+                visibility = dependent[1]
+                dependent = dependent[0]
 
-    for dependent in env.get('PROGDEPS_DEPENDENTS', []):
-        if dependent is None:
-            continue
-        dependentWithIxes = SCons.Util.adjustixes(
-            dependent, prog_builder.get_prefix(env), prog_builder.get_suffix(env))
-        dependentNode = prog_node_factory(dependentWithIxes)
-        __append_direct_libdeps(dependentNode, [dependency(target[0], True, dependency.Public)])
+            dependentWithIxes = SCons.Util.adjustixes(
+                dependent, prog_builder.get_prefix(env), prog_builder.get_suffix(env))
+            dependentNode = prog_node_factory(dependentWithIxes)
+            __append_direct_libdeps(dependentNode, [dependency(target[0], dependency_map[visibility])])
 
-    return target, source
+        return target, source
+
+    return libdeps_emitter
 
 def expand_libdeps_tags(source, target, env, for_signature):
     results = []
@@ -368,38 +329,53 @@ def setup_environment(env, emitting_shared=False):
     env[libdeps_env_var] = SCons.Util.CLVar()
     env[syslibdeps_env_var] = SCons.Util.CLVar()
 
-    env.Append(LIBEMITTER=libdeps_emitter)
-    if emitting_shared:
-        env['_LIBDEPS_LIBS'] = '$_LIBDEPS_GET_LIBS'
-        env.Append(
-            PROGEMITTER=shlibdeps_emitter,
-            SHLIBEMITTER=shlibdeps_emitter)
-    else:
-
-        def expand_libdeps_with_extraction_flags(source, target, env, for_signature):
-            result = []
-            libs = get_libdeps(source, target, env, for_signature)
-            for lib in libs:
-                if 'init-no-global-side-effects' in env.Entry(lib).get_env().get('LIBDEPS_TAGS', []):
-                    result.append(str(lib))
-                else:
-                    result.extend(env.subst('$LINK_WHOLE_ARCHIVE_LIB_START'
-                                            '$TARGET'
-                                            '$LINK_WHOLE_ARCHIVE_LIB_END', target=lib).split())
-            return result
-
-        env['_LIBDEPS_LIBS_WITH_TAGS'] = expand_libdeps_with_extraction_flags
+    # We need a way for environments to alter just which libdeps
+    # emitter they want, without altering the overall program or
+    # library emitter which may have important effects. The
+    # subsitution rules for emitters are a little strange, so build
+    # ourselves a little trampoline to use below so we don't have to
+    # deal with it.
+    def make_indirect_emitter(variable):
+        def indirect_emitter(target, source, env):
+            return env[variable](target, source, env)
+        return indirect_emitter
+
+    env.Append(
+        LIBDEPS_LIBEMITTER=make_libdeps_emitter('StaticLibrary'),
+        LIBEMITTER=make_indirect_emitter('LIBDEPS_LIBEMITTER'),
+
+        LIBDEPS_SHAREMITTER=make_libdeps_emitter('SharedArchive'),
+        SHAREMITTER=make_indirect_emitter('LIBDEPS_SHAREMITTER'),
+
+        LIBDEPS_SHLIBEMITTER=make_libdeps_emitter('SharedLibrary', dependency_visibility_honored),
+        SHLIBEMITTER=make_indirect_emitter('LIBDEPS_SHLIBEMITTER'),
+
+        LIBDEPS_PROGEMITTER=make_libdeps_emitter('SharedLibrary' if emitting_shared else 'StaticLibrary'),
+        PROGEMITTER=make_indirect_emitter('LIBDEPS_PROGEMITTER'),
+    )
+
+    def expand_libdeps_with_extraction_flags(source, target, env, for_signature):
+        result = []
+        libs = get_libdeps(source, target, env, for_signature)
+        for lib in libs:
+            if 'init-no-global-side-effects' in env.Entry(lib).get_env().get('LIBDEPS_TAGS', []):
+                result.append(str(lib))
+            else:
+                result.extend(env.subst('$LINK_WHOLE_ARCHIVE_LIB_START'
+                                        '$TARGET'
+                                        '$LINK_WHOLE_ARCHIVE_LIB_END', target=lib).split())
+        return result
+
+    env['_LIBDEPS_LIBS_WITH_TAGS'] = expand_libdeps_with_extraction_flags
+
+    env['_LIBDEPS_LIBS'] = ('$LINK_WHOLE_ARCHIVE_START '
+                            '$LINK_LIBGROUP_START '
+                            '$_LIBDEPS_LIBS_WITH_TAGS '
+                            '$LINK_LIBGROUP_END '
+                            '$LINK_WHOLE_ARCHIVE_END')
 
-        env['_LIBDEPS_LIBS'] = ('$LINK_WHOLE_ARCHIVE_START '
-                                '$LINK_LIBGROUP_START '
-                                '$_LIBDEPS_LIBS_WITH_TAGS '
-                                '$LINK_LIBGROUP_END '
-                                '$LINK_WHOLE_ARCHIVE_END')
-        env.Append(
-            PROGEMITTER=libdeps_emitter,
-            SHLIBEMITTER=libdeps_emitter)
     env.Prepend(_LIBFLAGS='$_LIBDEPS_TAGS $_LIBDEPS $_SYSLIBDEPS ')
-    for builder_name in ('Program', 'SharedLibrary', 'LoadableModule'):
+    for builder_name in ('Program', 'SharedLibrary', 'LoadableModule', 'SharedArchive'):
         try:
             update_scanner(env['BUILDERS'][builder_name])
         except KeyError:
diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py
index 87821f0341b..9f3de6166df 100644
--- a/site_scons/site_tools/auto_install_binaries.py
+++ b/site_scons/site_tools/auto_install_binaries.py
@@ -11,6 +11,8 @@ def generate(env):
         env.subst('$PROGSUFFIX') : 'bin',
         '.dylib' : 'lib',
         '.so' : 'lib',
+        '.dll' : 'bin',
+        '.lib' : 'lib',
     }
 
     def auto_install(env, target, source, **kwargs):
diff --git a/site_scons/site_tools/mergelib.py b/site_scons/site_tools/mergelib.py
deleted file mode 100644
index d8fcf3347ba..00000000000
--- a/site_scons/site_tools/mergelib.py
+++ /dev/null
@@ -1,32 +0,0 @@
-"""Builder for static libraries composed of the contents of other static libraries.
-
-The following rule creates a library "mylib" whose contents are the contents of
-"firstlib", "secondlib", and all LIBDEPS dependencies of "firstlib" and
-"secondlib".  This creates self-contained static and shared libraries that can
-be distributed to customers.
-
-MergeLibrary('mylib', ['firstlib', 'secondlib'])
-
-"""
-
-import libdeps
-from SCons.Action import Action
-from SCons.Builder import Builder
-
-def merge_library_method(env, target, source, LIBDEPS=None, **kwargs):
-    return env._MergeLibrary(target, [], LIBDEPS=source, **kwargs)
-
-def exists( env ):
-    return True
-
-def generate( env ):
-    merge_library = Builder(
-        action='$ARCOM $_LIBDEPS_OBJS',
-        src_prefix='$LIBPREFIX',
-        src_suffix='$LIBSUFFIX',
-        prefix='$LIBPREFIX',
-        suffix='$LIBSUFFIX',
-        emitter=libdeps.libdeps_emitter )
-    libdeps.update_scanner( merge_library )
-    env['BUILDERS']['_MergeLibrary'] = merge_library
-    env.AddMethod( merge_library_method, 'MergeLibrary' )
diff --git a/site_scons/site_tools/thin_archive.py b/site_scons/site_tools/thin_archive.py
index 511c0ef6e54..15357874438 100644
--- a/site_scons/site_tools/thin_archive.py
+++ b/site_scons/site_tools/thin_archive.py
@@ -37,13 +37,13 @@ def exists(env):
     if pipe.wait() != 0:
         return False
 
-    isgnu = False
+    found = False
     for line in pipe.stdout:
-        if isgnu:
+        if found:
             continue  # consume all data
-        isgnu = re.search(r'^GNU ar', line)
+        found = re.search(r'^GNU ar|^LLVM', line)
 
-    return bool(isgnu)
+    return bool(found)
 
 def _add_emitter(builder):
     base_emitter = builder.emitter
@@ -84,9 +84,8 @@ def generate(env):
     env['RANLIBCOM'] = noop_action
     env['RANLIBCOMSTR'] = 'Skipping ranlib for thin archive $TARGET'
 
-    builder = env['BUILDERS']['StaticLibrary']
-    _add_emitter(builder)
+    for builder in ['StaticLibrary', 'SharedArchive']:
+        _add_emitter(env['BUILDERS'][builder])
 
-    _add_scanner(env['BUILDERS']['SharedLibrary'])
-    _add_scanner(env['BUILDERS']['LoadableModule'])
-    _add_scanner(env['BUILDERS']['Program'])
+    for builder in ['SharedLibrary', 'LoadableModule', 'Program']:
+        _add_scanner(env['BUILDERS'][builder])