SERVER-37488 SERVER-37596 Manage debug info and symbol maps info for embedded builds

(cherry picked from commit 6d475fdb5a76acab760ce4b6709b60a4c8c9aec6)

10 files changed, 327 insertions, 45 deletions

diff --git a/SConstruct b/SConstruct
index d1b982329c0..d8a555b9db6 100644
--- a/SConstruct
+++ b/SConstruct
@@ -226,6 +226,15 @@ add_option('dbg',
     type='choice',
 )
 
+add_option('separate-debug',
+    choices=['on', 'off'],
+    const='on',
+    default='off',
+    help='Produce separate debug files (only effective in --install-mode=hygienic)',
+    nargs='?',
+    type='choice',
+)
+
 add_option('spider-monkey-dbg',
     choices=['on', 'off'],
     const='on',
@@ -1305,7 +1314,8 @@ if link_model.startswith("dynamic"):
     def library(env, target, source, *args, **kwargs):
         sharedLibrary = env.SharedLibrary(target, source, *args, **kwargs)
         sharedArchive = env.SharedArchive(target, source=sharedLibrary[0].sources, *args, **kwargs)
-        return (sharedLibrary, sharedArchive)
+        sharedLibrary.extend(sharedArchive)
+        return sharedLibrary
 
     env['BUILDERS']['Library'] = library
     env['BUILDERS']['LibraryObject'] = env['BUILDERS']['SharedObject']
@@ -3427,6 +3437,10 @@ env = doConfigure( env )
 
 # TODO: Later, this should live somewhere more graceful.
 if get_option('install-mode') == 'hygienic':
+
+    if get_option('separate-debug') == "on":
+        env.Tool('separate_debug')
+
     env.Tool('auto_install_binaries')
     if env['PLATFORM'] == 'posix':
         env.AppendUnique(
@@ -3456,6 +3470,8 @@ if get_option('install-mode') == 'hygienic':
                 "-Wl,-install_name,@rpath/${TARGET.file}",
             ],
         )
+elif get_option('separate-debug') == "on":
+    env.FatalError('Cannot use --separate-debug without --install-mode=hygienic')
 
 # Now that we are done with configure checks, enable icecream, if available.
 env.Tool('icecream')
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 467e2ab0eeb..5c02e84cc10 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -3551,6 +3551,16 @@ tasks:
 
           fi
 
+          # CMake doesn't seem to do the dSYM for us for the framework
+          if [ -e $WORKDIR/src/build/mongo-embedded-sdk-$VERSION-tmp/Frameworks ]; then
+              pushd $WORKDIR/src/build/mongo-embedded-sdk-$VERSION-tmp/Frameworks
+              xcrun dsymutil -o bson.framework.dSYM bson.framework/bson
+              xcrun strip -Sx bson.framework/bson
+              xcrun dsymutil -o mongoc.framework.dSYM mongoc.framework/mongoc
+              xcrun strip -Sx mongoc.framework/mongoc
+              popd
+          fi
+
           mv $WORKDIR/src/build/mongo-embedded-sdk-$VERSION-tmp $WORKDIR/src/build/mongo-embedded-sdk-$VERSION
 
 - name: embedded_sdk_install_dev
@@ -3568,10 +3578,36 @@ tasks:
           --use-system-mongo-c=on
           --wiredtiger=off
           --allocator=system
+          --separate-debug
           CPPPATH='$BUILD_ROOT/mongo-embedded-sdk-$MONGO_VERSION/include/libbson-1.0 $BUILD_ROOT/mongo-embedded-sdk-$MONGO_VERSION/include/libmongoc-1.0'
         task_compile_flags_extra: >-
           --link-model=dynamic-sdk
 
+    # Unfortunately, it is very hard to get the symbol map filename
+    # correct in SCons. We work around that here by letting SCons give
+    # them a well known name, and then renaming them appropriately
+    # once we have finished the build.
+    - command: shell.exec
+      params:
+        working_dir: "src/build"
+        script: |
+          set -o errexit
+          set -o verbose
+
+          cd mongo-embedded-sdk-${version}
+
+          if [ -e Frameworks/mongo_embedded.framework/BCSymbolMaps ]; then
+              pushd Frameworks/mongo_embedded.framework
+              mv BCSymbolMaps/libmongo_embedded.dylib.bcsymbolmap BCSymbolMaps/$(dwarfdump -u mongo_embedded | awk '{ print $2 }').bcsymbolmap
+              popd
+          fi
+
+          if [ -e Frameworks/mongoc_embedded.framework/BCSymbolMaps ]; then
+              pushd Frameworks/mongoc_embedded.framework
+              mv BCSymbolMaps/libmongoc_embedded.dylib.bcsymbolmap BCSymbolMaps/$(dwarfdump -u mongoc_embedded | awk '{ print $2 }').bcsymbolmap
+              popd
+          fi
+
 - name: embedded_sdk_s3_put
   commands:
     # Not using archive.targz_pack here because I can't get it to work.
@@ -3666,11 +3702,10 @@ tasks:
               ${compile_env|} buildscripts/runiossim.sh ${ios_sim_device} ${ios_sim_runtime} "build/mongo-embedded-sdk-${version}/bin/mongo_embedded_test" --tempPath /data
               ${compile_env|} buildscripts/runiossim.sh ${ios_sim_device} ${ios_sim_runtime} "build/mongo-embedded-sdk-${version}/bin/mongoc_embedded_test" --tempPath /data
           elif [ ${enable_embedded_tests|false} = "android_emulator" ]; then
-              # strip because the binaries with debug symbols are too big for the default storage on the emulator
-              find build/mongo-embedded-sdk-${version}/bin -type f | xargs $(dirname $(pwd))/android_toolchain-${android_toolchain_target_arch}-${android_toolchain_api_version}/bin/*-linux-android*-strip
-              find build/mongo-embedded-sdk-${version}/lib -type f -name "*.so" | xargs $(dirname $(pwd))/android_toolchain-${android_toolchain_target_arch}-${android_toolchain_api_version}/bin/*-linux-android*-strip
-              ${compile_env|} buildscripts/runandroidsim.sh $(dirname $(pwd))/android_sdk ${android_toolchain_system_image_arch} ${android_system_image_version} "build/mongo-embedded-sdk-${version}" "bin/mongo_embedded_test" --tempPath /data
-              ${compile_env|} buildscripts/runandroidsim.sh $(dirname $(pwd))/android_sdk ${android_toolchain_system_image_arch} ${android_system_image_version} "build/mongo-embedded-sdk-${version}" "bin/mongoc_embedded_test" --tempPath /data
+              cp -r build/mongo-embedded-sdk-${version} build/mongo-embedded-sdk-${version}-exec
+              find build/mongo-embedded-sdk-${version}-exec/bin build/mongo-embedded-sdk-${version}-exec/lib -type f -name "*.debug" -delete
+              ${compile_env|} buildscripts/runandroidsim.sh $(dirname $(pwd))/android_sdk ${android_toolchain_system_image_arch} ${android_system_image_version} "build/mongo-embedded-sdk-${version}-exec" "bin/mongo_embedded_test" --tempPath /data
+              ${compile_env|} buildscripts/runandroidsim.sh $(dirname $(pwd))/android_sdk ${android_toolchain_system_image_arch} ${android_system_image_version} "build/mongo-embedded-sdk-${version}-exec" "bin/mongoc_embedded_test" --tempPath /data
           elif [ ${enable_embedded_tests|false} = "native" ]; then
             "build/mongo-embedded-sdk-${version}/bin/mongo_embedded_test"
             "build/mongo-embedded-sdk-${version}/bin/mongoc_embedded_test"
@@ -4034,6 +4069,30 @@ tasks:
               done
           done
 
+          for platform in iPhone AppleTV Watch; do
+              for framework in bson mongoc; do
+                  _OSDSYM=$(printf ./%sOS/Frameworks/%s.framework.dSYM/Contents/Resources/DWARF/%s $platform $framework $framework)
+                  _SimulatorDSYM=$(printf ./%sSimulator/Frameworks/%s.framework.dSYM/Contents/Resources/DWARF/%s $platform $framework $framework)
+                  ${compile_env|} xcrun lipo $_OSDSYM $_SimulatorDSYM -create -output $_OSDSYM
+              done
+
+              for framework in mongo_embedded mongoc_embedded; do
+                  _OSDSYM=$(printf ./%sOS/Frameworks/%s.framework.dSYM/Contents/Resources/DWARF/lib%s.dylib $platform $framework $framework)
+                  _SimulatorDSYM=$(printf ./%sSimulator/Frameworks/%s.framework.dSYM/Contents/Resources/DWARF/lib%s.dylib $platform $framework $framework)
+                  ${compile_env|} xcrun lipo $_OSDSYM $_SimulatorDSYM -create -output $_OSDSYM
+              done
+          done
+
+          for platform in iPhone AppleTV Watch; do
+              for framework in mongo_embedded mongoc_embedded; do
+                  _OSBCSymbolMaps=$(printf ./%sOS/Frameworks/%s.framework/BCSymbolMaps $platform $framework)
+                  _SimulatorBCSymbolMaps=$(printf ./%sSimulator/Frameworks/%s.framework/BCSymbolMaps $platform $framework)
+                  if [ -e $_SimulatorBCSymbolMaps -a -e $_OSBCSymbolMaps ]; then
+                      cp $_SimulatorBCSymbolMaps/* $_OSBCSymbolMaps/
+                  fi
+              done
+          done
+
           popd
 
           cat <<EOF > mongo-embedded-sdk-${version}/README-Licenses.txt
@@ -4043,28 +4102,37 @@ tasks:
             - mongo_embedded: iPhoneOS/Frameworks/mongo_embedded.framework/LICENSE-Embedded.txt
             - mongoc_embedded: iPhoneOS/Frameworks/mongoc_embedded.framework/LICENSE-Embedded.txt
           EOF
-          tar --exclude "mongo-embedded-sdk-${version}/*/*.tgz" -zcvf mongo-embedded-sdk-${version}.tgz ./mongo-embedded-sdk-${version}/*OS ./mongo-embedded-sdk-${version}/README-Licenses.txt
+          tar --exclude "mongo-embedded-sdk-${version}/*/*.tgz" -zcvf mongo-embedded-sdk-${version}.tgz \
+              ./mongo-embedded-sdk-${version}/*OS \
+              ./mongo-embedded-sdk-${version}/README-Licenses.txt
 
           cat <<EOF > mongo-embedded-sdk-${version}/README-Licenses.txt
           The software accompanying this file is Copyright (C) 2018 MongoDB, Inc. and
           is licensed to you on the terms set forth in the following files:
             - mongo-c-driver: iPhoneOS/Frameworks/mongoc.framework/COPYING
           EOF
-          tar --exclude "mongo-embedded-sdk-${version}/*/*.tgz" -zcvf mongo-embedded-sdk-${version}-mongo-c-driver.tgz ./mongo-embedded-sdk-${version}/*OS/Frameworks/bson.framework ./mongo-embedded-sdk-${version}/*OS/Frameworks/mongoc.framework  ./mongo-embedded-sdk-${version}/README-Licenses.txt
+          tar --exclude "mongo-embedded-sdk-${version}/*/*.tgz" -zcvf mongo-embedded-sdk-${version}-mongo-c-driver.tgz \
+              ./mongo-embedded-sdk-${version}/*OS/Frameworks/bson.framework{,.dSYM} \
+              ./mongo-embedded-sdk-${version}/*OS/Frameworks/mongoc.framework{,.dSYM} \
+              ./mongo-embedded-sdk-${version}/README-Licenses.txt
 
           cat <<EOF > mongo-embedded-sdk-${version}/README-Licenses.txt
           The software accompanying this file is Copyright (C) 2018 MongoDB, Inc. and
           is licensed to you on the terms set forth in the following files:
             - mongo_embedded: iPhoneOS/Frameworks/mongo_embedded.framework/LICENSE-Embedded.txt
           EOF
-          tar --exclude "mongo-embedded-sdk-${version}/*/*.tgz" -zcvf mongo-embedded-sdk-${version}-mongo-embedded.tgz ./mongo-embedded-sdk-${version}/*OS/Frameworks/mongo_embedded.framework  ./mongo-embedded-sdk-${version}/README-Licenses.txt
+          tar --exclude "mongo-embedded-sdk-${version}/*/*.tgz" -zcvf mongo-embedded-sdk-${version}-mongo-embedded.tgz \
+              ./mongo-embedded-sdk-${version}/*OS/Frameworks/mongo_embedded.framework{,.dSYM} \
+              ./mongo-embedded-sdk-${version}/README-Licenses.txt
 
           cat <<EOF > mongo-embedded-sdk-${version}/README-Licenses.txt
           The software accompanying this file is Copyright (C) 2018 MongoDB, Inc. and
           is licensed to you on the terms set forth in the following files:
             - mongoc_embedded: iPhoneOS/Frameworks/mongoc_embedded.framework/LICENSE-Embedded.txt
           EOF
-          tar --exclude "mongo-embedded-sdk-${version}/*/*.tgz" -zcvf mongo-embedded-sdk-${version}-mongoc-embedded.tgz ./mongo-embedded-sdk-${version}/*OS/Frameworks/mongoc_embedded.framework  ./mongo-embedded-sdk-${version}/README-Licenses.txt
+          tar --exclude "mongo-embedded-sdk-${version}/*/*.tgz" -zcvf mongo-embedded-sdk-${version}-mongoc-embedded.tgz \
+              ./mongo-embedded-sdk-${version}/*OS/Frameworks/mongoc_embedded.framework{,.dSYM} \
+              ./mongo-embedded-sdk-${version}/README-Licenses.txt
 
     - command: s3.put
       params:
@@ -10511,8 +10579,7 @@ buildvariants:
         -DCMAKE_INSTALL_BINDIR=Frameworks
         -DENABLE_SSL=DARWIN
         -DENABLE_ZLIB=BUNDLED
-        -DCMAKE_C_FLAGS="-flto -Wunguarded-availability"
-        -DCMAKE_SHARED_LINKER_FLAGS="-flto"
+        -DCMAKE_C_FLAGS="-Wunguarded-availability"
     disable_unit_tests: true
     enable_embedded_tests: native
     dump_scons_config_on_failure: true
@@ -10544,8 +10611,8 @@ buildvariants:
         -DENABLE_SASL=OFF
         -DENABLE_ZLIB=OFF
         -DENABLE_SNAPPY=OFF
-        -DCMAKE_C_FLAGS="-fembed-bitcode -flto -Wunguarded-availability"
-        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode -flto"
+        -DCMAKE_C_FLAGS="-fembed-bitcode -Wunguarded-availability"
+        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode"
     disable_unit_tests: true
     dump_scons_config_on_failure: true
   tasks:
@@ -10576,8 +10643,8 @@ buildvariants:
         -DENABLE_SASL=OFF
         -DENABLE_ZLIB=OFF
         -DENABLE_SNAPPY=OFF
-        -DCMAKE_C_FLAGS="-fembed-bitcode -flto -Wunguarded-availability"
-        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode -flto"
+        -DCMAKE_C_FLAGS="-fembed-bitcode -Wunguarded-availability"
+        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode"
     disable_unit_tests: true
     enable_embedded_tests: ios_tvos_simulator
     ios_sim_device: iPhone-7
@@ -10611,8 +10678,8 @@ buildvariants:
         -DENABLE_SASL=OFF
         -DENABLE_ZLIB=OFF
         -DENABLE_SNAPPY=OFF
-        -DCMAKE_C_FLAGS="-fembed-bitcode -flto -Wunguarded-availability"
-        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode -flto"
+        -DCMAKE_C_FLAGS="-fembed-bitcode -Wunguarded-availability"
+        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode"
     disable_unit_tests: true
     dump_scons_config_on_failure: true
   tasks:
@@ -10643,8 +10710,8 @@ buildvariants:
         -DENABLE_SASL=OFF
         -DENABLE_ZLIB=OFF
         -DENABLE_SNAPPY=OFF
-        -DCMAKE_C_FLAGS="-fembed-bitcode -flto -Wunguarded-availability"
-        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode -flto"
+        -DCMAKE_C_FLAGS="-fembed-bitcode -Wunguarded-availability"
+        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode"
     disable_unit_tests: true
     enable_embedded_tests: ios_tvos_simulator
     ios_sim_device: Apple-TV-1080p
@@ -10678,8 +10745,8 @@ buildvariants:
         -DENABLE_SASL=OFF
         -DENABLE_ZLIB=OFF
         -DENABLE_SNAPPY=OFF
-        -DCMAKE_C_FLAGS="-fembed-bitcode -flto -fapplication-extension -Wunguarded-availability"
-        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode -flto -fapplication-extension"
+        -DCMAKE_C_FLAGS="-fembed-bitcode -fapplication-extension -Wunguarded-availability"
+        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode -fapplication-extension"
     disable_unit_tests: true
     dump_scons_config_on_failure: true
   tasks:
@@ -10710,8 +10777,8 @@ buildvariants:
         -DENABLE_SASL=OFF
         -DENABLE_ZLIB=OFF
         -DENABLE_SNAPPY=OFF
-        -DCMAKE_C_FLAGS="-fembed-bitcode -flto -fapplication-extension -Wunguarded-availability"
-        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode -flto -fapplication-extension"
+        -DCMAKE_C_FLAGS="-fembed-bitcode -fapplication-extension -Wunguarded-availability"
+        -DCMAKE_SHARED_LINKER_FLAGS="-fembed-bitcode -fapplication-extension"
     disable_unit_tests: true
     enable_embedded_tests: ios_tvos_simulator
     ios_sim_device: Apple-Watch-Series-3-42mm
diff --git a/etc/scons/android_toolchain.vars b/etc/scons/android_toolchain.vars
index db541face37..d54b004433b 100644
--- a/etc/scons/android_toolchain.vars
+++ b/etc/scons/android_toolchain.vars
@@ -18,6 +18,8 @@ if not toolchain_path:
 CC=os.path.join(toolchain_path, "bin", "clang" + compiler_suffix)
 CXX=os.path.join(toolchain_path, "bin", "clang++" + compiler_suffix)
 AR=subprocess.check_output([CXX, '-print-prog-name=ar']).strip()
+OBJCOPY=subprocess.check_output([CXX, '-print-prog-name=objcopy']).strip()
+
 
 LINKFLAGS='-static-libstdc++ -fuse-ld=gold'
 CPPDEFINES='__ANDROID_API__=21 ASIO_DISABLE_STD_STRING_VIEW'
diff --git a/site_scons/site_tools/auto_install_binaries.py b/site_scons/site_tools/auto_install_binaries.py
index 9f3de6166df..9a71b4919b7 100644
--- a/site_scons/site_tools/auto_install_binaries.py
+++ b/site_scons/site_tools/auto_install_binaries.py
@@ -10,6 +10,12 @@ def generate(env):
     suffix_map = {
         env.subst('$PROGSUFFIX') : 'bin',
         '.dylib' : 'lib',
+        # TODO: These 'lib' answers are incorrect. The location for the debug info
+        # should be the same as the target itself, which might be bin or lib. We need
+        # a solution for that. When that is fixed, add 'Program' back into the list
+        # of separate debug targets in the separate_debug.py tool.
+        '.dSYM' : 'lib',
+        '.debug' : 'lib',
         '.so' : 'lib',
         '.dll' : 'bin',
         '.lib' : 'lib',
diff --git a/site_scons/site_tools/separate_debug.py b/site_scons/site_tools/separate_debug.py
new file mode 100644
index 00000000000..3edd50518e1
--- /dev/null
+++ b/site_scons/site_tools/separate_debug.py
@@ -0,0 +1,143 @@
+# Copyright 2018 MongoDB Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import SCons
+
+def _update_builder(env, builder, bitcode):
+
+    old_scanner = builder.target_scanner
+    old_path_function = old_scanner.path_function
+
+    def new_scanner(node, env, path=()):
+        results = old_scanner.function(node, env, path)
+        origin = getattr(node.attributes, "debug_file_for", None)
+        if origin:
+            origin_results = old_scanner(origin, env, path)
+            for origin_result in origin_results:
+                origin_result_debug_file = getattr(origin_result.attributes, "separate_debug_file", None)
+                if origin_result_debug_file:
+                    results.append(origin_result_debug_file)
+        # TODO: Do we need to do the same sort of drag along for bcsymbolmap files?
+        return results
+
+    builder.target_scanner = SCons.Scanner.Scanner(
+        function=new_scanner,
+        path_function=old_path_function,
+    )
+
+    base_action = builder.action
+    if not isinstance(base_action, SCons.Action.ListAction):
+        base_action = SCons.Action.ListAction([base_action])
+
+    # TODO: Make variables for dsymutil and strip, and for the action
+    # strings. We should really be running these tools as found by
+    # xcrun by default. We should achieve that by upgrading the
+    # site_scons/site_tools/xcode.py tool to search for these for
+    # us. We could then also remove a lot of the compiler and sysroot
+    # setup from the etc/scons/xcode_*.vars files, which would be a
+    # win as well.
+    if env.TargetOSIs('darwin'):
+        if bitcode:
+            base_action.list.append(
+                SCons.Action.Action(
+                    "dsymutil $TARGET --symbol-map=${TARGET}.bcsymbolmap -o ${TARGET}.dSYM",
+                    "Generating debug info for $TARGET into ${TARGET}.dSYM"
+                )
+            )
+
+        else:
+            base_action.list.append(
+                SCons.Action.Action(
+                    "dsymutil $TARGET -o ${TARGET}.dSYM",
+                    "Generating debug info for $TARGET into ${TARGET}.dSYM"
+                )
+            )
+        base_action.list.append(
+            SCons.Action.Action(
+                "strip -Sx ${TARGET}",
+                "Stripping ${TARGET}"
+            )
+        )
+    elif env.TargetOSIs('posix'):
+        base_action.list.extend([
+            SCons.Action.Action(
+                "${OBJCOPY} --only-keep-debug $TARGET ${TARGET}.debug",
+                "Generating debug info for $TARGET into ${TARGET}.debug"
+            ),
+            SCons.Action.Action(
+                "${OBJCOPY} --strip-debug --add-gnu-debuglink ${TARGET}.debug ${TARGET}",
+                "Stripping debug info from ${TARGET} and adding .gnu.debuglink to ${TARGET}.debug"
+            ),
+        ])
+    else:
+        pass
+
+    base_emitter = builder.emitter
+    def new_emitter(target, source, env):
+
+        bitcode_file = None
+        if env.TargetOSIs('darwin'):
+            debug_file = env.Dir(str(target[0]) + ".dSYM")
+            if bitcode:
+                bitcode_file = env.File(str(target[0]) + ".bcsymbolmap")
+        elif env.TargetOSIs('posix'):
+            debug_file = env.File(str(target[0]) + ".debug")
+        else:
+            pass
+
+        setattr(debug_file.attributes, "debug_file_for", target[0])
+        setattr(target[0].attributes, "separate_debug_file", debug_file)
+
+        target.append(debug_file)
+
+        if bitcode_file:
+            setattr(bitcode_file.attributes, "bcsymbolmap_file_for", target[0])
+            setattr(target[0].attributes, "bcsymbolmap_file", bitcode_file)
+            target.append(bitcode_file)
+
+        return (target, source)
+
+    new_emitter = SCons.Builder.ListEmitter([base_emitter, new_emitter])
+    builder.emitter = new_emitter
+
+def generate(env):
+    if not exists(env):
+        return
+
+    # If we are generating bitcode, add the magic linker flags that
+    # hide the bitcode symbols, and override the name of the bitcode
+    # symbol map file so that it is determinstically known to SCons
+    # rather than being a UUID. We need this so that we can install it
+    # under a well known name. We leave it to the evergreen
+    # postprocessing to rename to the correct name. I'd like to do
+    # this better, but struggled for a long time and decided that
+    # later was a better time to address this. We should also consider
+    # moving all bitcode setup into a separate tool.
+    bitcode = False
+    if env.TargetOSIs('darwin') and any(flag == "-fembed-bitcode" for flag in env['LINKFLAGS']):
+        bitcode = True
+        env.AppendUnique(LINKFLAGS=[
+            "-Wl,-bitcode_hide_symbols",
+            "-Wl,-bitcode_symbol_map,${TARGET}.bcsymbolmap",
+        ])
+
+
+    # TODO: For now, not doing this for programs. Need to update
+    # auto_install_binaries to understand to install the debug symbol
+    # for target X to the same target location as X.
+    for builder in ['SharedLibrary', 'LoadableModule']:
+        _update_builder(env, env['BUILDERS'][builder], bitcode)
+
+def exists(env):
+    return True
diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 42e951a7009..4c32ec627d1 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -552,6 +552,10 @@ def failMissingObjCopy(env, target, source):
     env.FatalError("Generating debug symbols requires objcopy, please set the OBJCOPY variable.")
 
 def installBinary( e, name ):
+
+    if hygienic:
+        return
+
     debug_sym_name = name
     name = add_exe( name )
 
@@ -576,8 +580,7 @@ def installBinary( e, name ):
             name,
             debug_sym_cmd
         )
-        if not hygienic:
-            e.Install("#/", debug_sym)
+        e.Install("#/", debug_sym)
         e.Alias('debugsymbols', debug_sym)
         distDebugSymbols.append(debug_sym)
 
@@ -591,24 +594,26 @@ def installBinary( e, name ):
     else:
         distBinaries.append(name)
 
-    if not hygienic:
-        inst = e.Install( "$INSTALL_DIR/bin", name )
+    inst = e.Install( "$INSTALL_DIR/bin", name )
 
-        if env.TargetOSIs('posix'):
-            e.AddPostAction( inst, 'chmod 755 $TARGET' )
+    if env.TargetOSIs('posix'):
+        e.AddPostAction( inst, 'chmod 755 $TARGET' )
 
 def installExternalBinary( e, name_str ):
+
+    if hygienic:
+        return
+
     name = env.File("#/%s" % add_exe(name_str))
     if not name.isfile():
         env.FatalError("ERROR: external binary not found: {0}", name)
 
     distBinaries.append(name)
 
-    if not hygienic:
-        inst = e.Install( "$INSTALL_DIR/bin", name )
+    inst = e.Install( "$INSTALL_DIR/bin", name )
 
-        if env.TargetOSIs('posix'):
-            e.AddPostAction( inst, 'chmod 755 $TARGET' )
+    if env.TargetOSIs('posix'):
+        e.AddPostAction( inst, 'chmod 755 $TARGET' )
 
 # "--use-new-tools" adds dependencies for rewritten (Go) tools
 # It is required for "dist" but optional for "install"
diff --git a/src/mongo/embedded/mongo_embedded/SConscript b/src/mongo/embedded/mongo_embedded/SConscript
index 79cd14899db..fc5a86e75ed 100644
--- a/src/mongo/embedded/mongo_embedded/SConscript
+++ b/src/mongo/embedded/mongo_embedded/SConscript
@@ -62,7 +62,7 @@ if mongoEmbeddedEnv.TargetOSIs('darwin'):
     )
 
 
-mongoEmbeddedEnv.Library(
+mongoEmbeddedTargets = mongoEmbeddedEnv.Library(
     target='mongo_embedded',
     source=[
         'mongo_embedded.cpp',
@@ -124,8 +124,6 @@ if get_option('link-model') != 'dynamic-sdk':
 if get_option('link-model') != 'dynamic-sdk' or get_option('install-mode') != 'hygienic' or not env.TargetOSIs('darwin') or env.TargetOSIs('macOS'):
     Return()
 
-installHeaderRoot = env.Dir('$INSTALL_DIR/include/mongo_embedded/v1/mongo_embedded')
-
 frameworkDir = env.Dir('$INSTALL_DIR/Frameworks/mongo_embedded.framework')
 env.Alias('install-embedded-dev', frameworkDir)
 
@@ -147,7 +145,7 @@ env.Install(
 
 env.Install(
     target=frameworkDir.Dir('Headers'),
-    source=installHeaderRoot.File('mongo_embedded.h')
+    source=env.File('mongo_embedded.h')
 )
 
 env.InstallAs(
@@ -174,10 +172,30 @@ env.Install(
 
 mongoEmbeddedFwLib = env.InstallAs(
     target=frameworkDir.File('mongo_embedded'),
-    source='$INSTALL_DIR/lib/libmongo_embedded.dylib',
+    source=mongoEmbeddedTargets[0],
 )
 
 env.AddPostAction(
     files=mongoEmbeddedFwLib,
-    action="install_name_tool -id @rpath/mongo_embedded.framework/mongo_embedded $TARGET",
+    action=[
+        "install_name_tool -delete_rpath @loader_path/../lib $TARGET",
+        "install_name_tool -id @rpath/mongo_embedded.framework/mongo_embedded $TARGET",
+    ]
 )
+
+mongoEmbeddedDSYM = getattr(mongoEmbeddedTargets[0].attributes, "separate_debug_file", None)
+if mongoEmbeddedDSYM:
+    frameworkDSYMDir = '$INSTALL_DIR/Frameworks/mongo_embedded.framework.dSYM'
+    env.Alias('install-embedded-dev', frameworkDSYMDir)
+
+    env.InstallAs(
+        target=frameworkDSYMDir,
+        source=mongoEmbeddedDSYM,
+    )
+
+mongoEmbeddedBCSymbolMap = getattr(mongoEmbeddedTargets[0].attributes, "bcsymbolmap_file", None)
+if mongoEmbeddedBCSymbolMap:
+    env.Install(
+        target=frameworkDir.Dir('BCSymbolMaps'),
+        source=mongoEmbeddedBCSymbolMap,
+    )
diff --git a/src/mongo/embedded/mongo_embedded/mongo_embedded.podspec.in b/src/mongo/embedded/mongo_embedded/mongo_embedded.podspec.in
index 875ced84dc1..943b5862014 100644
--- a/src/mongo/embedded/mongo_embedded/mongo_embedded.podspec.in
+++ b/src/mongo/embedded/mongo_embedded/mongo_embedded.podspec.in
@@ -23,4 +23,10 @@ Pod::Spec.new do |s|
   s.watchos.vendored_frameworks = "WatchOS/Frameworks/mongo_embedded.framework"
   s.tvos.vendored_frameworks = "AppleTVOS/Frameworks/mongo_embedded.framework"
 
+  # My current best understanding is that apple uses these to inject
+  # back in when running dsymutil on their side after regenerating a
+  # binary from the bitcode slices. So we need to keep these available
+  # in the pod so they get archived into end user application.
+  s.preserve_path = '**/*.bcsymbolmap'
+
 end
diff --git a/src/mongo/embedded/mongoc_embedded/SConscript b/src/mongo/embedded/mongoc_embedded/SConscript
index 2bccef55ee3..63633e8220a 100644
--- a/src/mongo/embedded/mongoc_embedded/SConscript
+++ b/src/mongo/embedded/mongoc_embedded/SConscript
@@ -63,7 +63,7 @@ if mongocEmbeddedEnv.TargetOSIs('darwin'):
         ],
     )
 
-mongocEmbeddedEnv.Library(
+mongocEmbeddedTargets = mongocEmbeddedEnv.Library(
     target='mongoc_embedded',
     source=[
         'mongoc_embedded.cpp',
@@ -121,8 +121,6 @@ if get_option('link-model') != 'dynamic-sdk':
 if get_option('link-model') != 'dynamic-sdk' or get_option('install-mode') != 'hygienic' or not env.TargetOSIs('darwin') or env.TargetOSIs('macOS'):
     Return()
 
-installHeaderRoot = env.Dir('$INSTALL_DIR/include/mongoc_embedded/v1/mongoc_embedded')
-
 frameworkDir = env.Dir('$INSTALL_DIR/Frameworks/mongoc_embedded.framework')
 env.Alias('install-embedded-dev', frameworkDir)
 
@@ -143,7 +141,7 @@ env.Install(
 
 env.Install(
     target=frameworkDir.Dir('Headers'),
-    source=installHeaderRoot.File('mongoc_embedded.h'),
+    source=env.File('mongoc_embedded.h'),
 )
 
 env.InstallAs(
@@ -170,13 +168,31 @@ env.Install(
 
 mongocEmbeddedFwLib = env.InstallAs(
     target=frameworkDir.File('mongoc_embedded'),
-    source='$INSTALL_DIR/lib/libmongoc_embedded.dylib',
+    source=mongocEmbeddedTargets[0],
 )
 
 env.AddPostAction(
     files=mongocEmbeddedFwLib,
     action=[
+        "install_name_tool -delete_rpath @loader_path/../lib $TARGET",
         "install_name_tool -id @rpath/mongoc_embedded.framework/mongoc_embedded $TARGET",
         "install_name_tool -change @rpath/libmongo_embedded.dylib @rpath/mongo_embedded.framework/mongo_embedded $TARGET",
     ],
 )
+
+mongocEmbeddedDSYM = getattr(mongocEmbeddedTargets[0].attributes, "separate_debug_file", None)
+if mongocEmbeddedDSYM:
+    frameworkDSYMDir = '$INSTALL_DIR/Frameworks/mongoc_embedded.framework.dSYM'
+    env.Alias('install-embedded-dev', frameworkDSYMDir)
+
+    env.InstallAs(
+        target=frameworkDSYMDir,
+        source=mongocEmbeddedDSYM,
+    )
+
+mongocEmbeddedBCSymbolMap = getattr(mongocEmbeddedTargets[0].attributes, "bcsymbolmap_file", None)
+if mongocEmbeddedBCSymbolMap:
+    env.Install(
+        target=frameworkDir.Dir('BCSymbolMaps'),
+        source=mongocEmbeddedBCSymbolMap,
+    )
diff --git a/src/mongo/embedded/mongoc_embedded/mongoc_embedded.podspec.in b/src/mongo/embedded/mongoc_embedded/mongoc_embedded.podspec.in
index 2787777a60d..cb94dab5bc6 100644
--- a/src/mongo/embedded/mongoc_embedded/mongoc_embedded.podspec.in
+++ b/src/mongo/embedded/mongoc_embedded/mongoc_embedded.podspec.in
@@ -26,4 +26,7 @@ Pod::Spec.new do |s|
   s.dependency "mongo_embedded", "~> 4.0"
   s.dependency "mongo-c-driver", "~> 1.13"
 
+  # See notes in mongo_embedded.podspec.in.
+  s.preserve_path = '**/*.bcsymbolmap'
+
 end