SERVER-50363 Merge --build-tools=next into stable

Merging the following fixes into the stable version of the build tools
and migrate fully from --ninja=next to the --build-tools=next:

  * SERVER-47598
  * SERVER-50010
  * SERVER-47943
  * SERVER-50125
  * SERVER-50376
  * SERVER-49457
  * SERVER-49493
  * SERVER-49036
  * SERVER-48966

5 files changed, 684 insertions, 304 deletions

diff --git a/SConstruct b/SConstruct
index b73a80b6411..2bfe2a2b426 100644
--- a/SConstruct
+++ b/SConstruct
@@ -105,10 +105,10 @@ SetOption('random', 1)
 #
 
 add_option('ninja',
-    choices=['stable', 'next', 'disabled'],
+    choices=['enabled', 'disabled'],
     default='disabled',
     nargs='?',
-    const='stable',
+    const='enabled',
     type='choice',
     help='Enable the build.ninja generator tool stable or canary version',
 )
@@ -1107,7 +1107,7 @@ envDict = dict(BUILD_ROOT=buildDir,
 # user has opted into the next gen tools, add our experimental tool
 # directory into the default toolpath, ahead of whatever is already in
 # there so it overrides it.
-if get_option('build-tools') == 'next' or get_option('ninja') == 'next':
+if get_option('build-tools') == 'next':
     SCons.Tool.DefaultToolpath.insert(0, os.path.abspath('site_scons/site_tools/next'))
 
 env = Environment(variables=env_vars, **envDict)
@@ -4020,22 +4020,18 @@ elif env.ToolchainIs("gcc"):
 
 # Now that we are done with configure checks, enable ccache and
 # icecream if requested.
-if get_option('build-tools') == 'next' or get_option('ninja') == 'next':
-    if 'CCACHE' in env and env['CCACHE']:
-        ccache = Tool('ccache')
-        if not ccache.exists(env):
-            env.FatalError(f"Failed to load ccache tool with CCACHE={env['CCACHE']}")
-        ccache(env)
-    if 'ICECC' in env and env['ICECC']:
-        env['ICECREAM_VERBOSE'] = env.Verbose()
-        env['ICECREAM_TARGET_DIR'] = '$BUILD_ROOT/scons/icecream'
-        icecream = Tool('icecream')
-        if not icecream.exists(env):
-            env.FatalError(f"Failed to load icecream tool with ICECC={env['ICECC']}")
-        icecream(env)
-else:
-    env.Tool('ccache')
-    env.Tool('icecream')
+if 'CCACHE' in env and env['CCACHE']:
+    ccache = Tool('ccache')
+    if not ccache.exists(env):
+        env.FatalError(f"Failed to load ccache tool with CCACHE={env['CCACHE']}")
+    ccache(env)
+if 'ICECC' in env and env['ICECC']:
+    env['ICECREAM_VERBOSE'] = env.Verbose()
+    env['ICECREAM_TARGET_DIR'] = '$BUILD_ROOT/scons/icecream'
+    icecream = Tool('icecream')
+    if not icecream.exists(env):
+        env.FatalError(f"Failed to load icecream tool with ICECC={env['ICECC']}")
+    icecream(env)
 
 # Defaults for SCons provided flags. SetOption only sets the option to our value
 # if the user did not provide it. So for any flag here if it's explicitly passed
@@ -4090,34 +4086,31 @@ if get_option('ninja') != 'disabled':
             env.FatalError("Use of ccache is mandatory with --ninja and icecream older than 1.2. You are running {}.".format(env['ICECREAM_VERSION']))
 
     ninja_builder = Tool("ninja")
-    if get_option('build-tools') == 'next' or get_option('ninja') == 'next':
-        env["NINJA_BUILDDIR"] = env.Dir("$BUILD_DIR/ninja")
-        ninja_builder.generate(env)
-
-        ninjaConf = Configure(env, help=False, custom_tests = {
-            'CheckNinjaCompdbExpand': env.CheckNinjaCompdbExpand,
-        })
-        env['NINJA_COMPDB_EXPAND'] = ninjaConf.CheckNinjaCompdbExpand()
-        ninjaConf.Finish()
-
-        # TODO: API for getting the sconscripts programmatically
-        # exists upstream: https://github.com/SCons/scons/issues/3625
-        def ninja_generate_deps(env, target, source, for_signature):
-            dependencies = env.Flatten([
-                'SConstruct',
-                glob(os.path.join('src', '**', 'SConscript'), recursive=True),
-                glob(os.path.join(os.path.expanduser('~/.scons/'), '**', '*.py'), recursive=True),
-                glob(os.path.join('site_scons', '**', '*.py'), recursive=True),
-                glob(os.path.join('buildscripts', '**', '*.py'), recursive=True),
-                glob(os.path.join('src/third_party/scons-*', '**', '*.py'), recursive=True),
-                glob(os.path.join('src/mongo/db/modules', '**', '*.py'), recursive=True),
-            ])
+    env["NINJA_BUILDDIR"] = env.Dir("$BUILD_DIR/ninja")
+    ninja_builder.generate(env)
 
-            return dependencies
-
-        env['NINJA_REGENERATE_DEPS'] = ninja_generate_deps
-    else:
-        ninja_builder.generate(env)
+    ninjaConf = Configure(env, help=False, custom_tests = {
+        'CheckNinjaCompdbExpand': env.CheckNinjaCompdbExpand,
+    })
+    env['NINJA_COMPDB_EXPAND'] = ninjaConf.CheckNinjaCompdbExpand()
+    ninjaConf.Finish()
+
+    # TODO: API for getting the sconscripts programmatically
+    # exists upstream: https://github.com/SCons/scons/issues/3625
+    def ninja_generate_deps(env, target, source, for_signature):
+        dependencies = env.Flatten([
+            'SConstruct',
+            glob(os.path.join('src', '**', 'SConscript'), recursive=True),
+            glob(os.path.join(os.path.expanduser('~/.scons/'), '**', '*.py'), recursive=True),
+            glob(os.path.join('site_scons', '**', '*.py'), recursive=True),
+            glob(os.path.join('buildscripts', '**', '*.py'), recursive=True),
+            glob(os.path.join('src/third_party/scons-*', '**', '*.py'), recursive=True),
+            glob(os.path.join('src/mongo/db/modules', '**', '*.py'), recursive=True),
+        ])
+
+        return dependencies
+
+    env['NINJA_REGENERATE_DEPS'] = ninja_generate_deps
 
     # idlc.py has the ability to print it's implicit dependencies
     # while generating, Ninja can consume these prints using the
@@ -4134,15 +4127,9 @@ if get_option('ninja') != 'disabled':
     )
 
     def get_idlc_command(env, node, action, targets, sources, executor=None):
-        if get_option('build-tools') == 'next' or get_option('ninja') == 'next':
-            _, variables, _ = env.NinjaGetGenericShellCommand(node, action, targets, sources, executor=executor)
-        else:
-            _, variables = env.NinjaGetShellCommand(node, action, targets, sources, executor=executor)
+        _, variables, _ = env.NinjaGetGenericShellCommand(node, action, targets, sources, executor=executor)
         variables["msvc_deps_prefix"] = "import file:"
-        if get_option('build-tools') == 'next' or get_option('ninja') == 'next':
-            return "IDLC", variables, env.subst(env['IDLC']).split()
-        else:
-            return "IDLC", variables
+        return "IDLC", variables, env.subst(env['IDLC']).split()
 
     env.NinjaRuleMapping("$IDLCCOM", get_idlc_command)
     env.NinjaRuleMapping(env["IDLCCOM"], get_idlc_command)
@@ -4828,4 +4815,4 @@ for i, s in enumerate(BUILD_TARGETS):
 # Do any final checks the Libdeps linter may need to do once all
 # SConscripts have been read but before building begins.
 if get_option('build-tools') == 'next':
-    libdeps.LibdepLinter(env).final_checks()
\ No newline at end of file
+    libdeps.LibdepLinter(env).final_checks()
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 6e0048bbf30..3435b5d986a 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -3549,7 +3549,7 @@ tasks:
         task_install_action:
             default
         task_compile_flags: >-
-          --ninja=next
+          --ninja
         targets:
           build.ninja
     - command: shell.exec
diff --git a/site_scons/site_tools/ccache.py b/site_scons/site_tools/ccache.py
index 1a34571cc7e..2a894919f8d 100644
--- a/site_scons/site_tools/ccache.py
+++ b/site_scons/site_tools/ccache.py
@@ -30,24 +30,21 @@ from pkg_resources import parse_version
 
 # This is the oldest version of ccache that offers support for -gsplit-dwarf
 _ccache_version_min = parse_version("3.2.3")
-_ccache_version_found = None
 
 
 def exists(env):
     """Look for a viable ccache implementation that meets our version requirements."""
-
-    # If we already generated, we definitely exist
-    if "CCACHE_VERSION" in env:
-        return True
-
-    ccache = env.get("CCACHE", False)
-    if not ccache:
+    if not env.subst("$CCACHE"):
         return False
 
-    ccache = env.WhereIs(ccache)
+    ccache = env.WhereIs("$CCACHE")
     if not ccache:
+        print(f"Error: ccache not found at {env['CCACHE']}")
         return False
 
+    if 'CCACHE_VERSION' in env and env['CCACHE_VERSION'] >= _ccache_version_min:
+        return True
+
     pipe = SCons.Action._subproc(
         env,
         SCons.Util.CLVar(ccache) + ["--version"],
@@ -57,6 +54,7 @@ def exists(env):
     )
 
     if pipe.wait() != 0:
+        print(f"Error: failed to execute '{env['CCACHE']}'")
         return False
 
     validated = False
@@ -70,25 +68,23 @@ def exists(env):
         ccache_version = re.split("ccache version (.+)", line)
         if len(ccache_version) < 2:
             continue
-        global _ccache_version_found
-        _ccache_version_found = parse_version(ccache_version[1])
-        if _ccache_version_found >= _ccache_version_min:
+        ccache_version = parse_version(ccache_version[1])
+        if ccache_version >= _ccache_version_min:
             validated = True
 
+    if validated:
+        env['CCACHE_VERSION'] = ccache_version
+    else:
+        print(f"Error: failed to verify ccache version >= {_ccache_version_min}, found {ccache_version}")
+
     return validated
 
 
 def generate(env):
     """Add ccache support."""
 
-    # If we have already generated the tool, don't generate it again.
-    if "CCACHE_VERSION" in env:
-        return
-
-    # If we can't find ccache, or it is too old a version, don't
-    # generate.
-    if not exists(env):
-        return
+    # Absoluteify
+    env["CCACHE"] = env.WhereIs("$CCACHE")
 
     # Propagate CCACHE related variables into the command environment
     for var, host_value in os.environ.items():
@@ -104,23 +100,53 @@ def generate(env):
     if env.ToolchainIs("clang"):
         env.AppendUnique(CCFLAGS=["-Qunused-arguments"])
 
-    # Record our found CCACHE_VERSION. Other tools that need to know
-    # about ccache (like iecc) should query this variable to determine
-    # if ccache is active. Looking at the CCACHE variable in the
-    # environment is not sufficient, since the user may have set it,
-    # but it doesn't work or is out of date.
-    env["CCACHE_VERSION"] = _ccache_version_found
+    # Check whether icecream is requested and is a valid tool.
+    if "ICECC" in env:
+        icecream = SCons.Tool.Tool('icecream')
+        icecream_enabled = bool(icecream) and icecream.exists(env)
+    else:
+        icecream_enabled = False
 
     # Set up a performant ccache configuration. Here, we don't use a second preprocessor and
     # pass preprocessor arguments that deterministically expand source files so a stable
     # hash can be calculated on them. This both reduces the amount of work ccache needs to
     # do and increases the likelihood of a cache hit.
-    env["ENV"]["CCACHE_NOCPP2"] = 1
     if env.ToolchainIs("clang"):
+        env["ENV"].pop("CCACHE_CPP2", None)
+        env["ENV"]["CCACHE_NOCPP2"] = "1"
         env.AppendUnique(CCFLAGS=["-frewrite-includes"])
     elif env.ToolchainIs("gcc"):
-        env.AppendUnique(CCFLAGS=["-fdirectives-only"])
-
+        if icecream_enabled:
+            # Newer versions of Icecream will drop -fdirectives-only from
+            # preprocessor and compiler flags if it does not find a remote
+            # build host to build on. ccache, on the other hand, will not
+            # pass the flag to the compiler if CCACHE_NOCPP2=1, but it will
+            # pass it to the preprocessor. The combination of setting
+            # CCACHE_NOCPP2=1 and passing the flag can lead to build
+            # failures.
+
+            # See: https://jira.mongodb.org/browse/SERVER-48443
+            # We have an open issue with Icecream and ccache to resolve the
+            # cause of these build failures. Once the bug is resolved and
+            # the fix is deployed, we can remove this entire conditional
+            # branch and make it like the one for clang.
+            # TODO: https://github.com/icecc/icecream/issues/550
+            env["ENV"].pop("CCACHE_CPP2", None)
+            env["ENV"]["CCACHE_NOCPP2"] = "1"
+        else:
+            env["ENV"].pop("CCACHE_NOCPP2", None)
+            env["ENV"]["CCACHE_CPP2"] = "1"
+            env.AppendUnique(CCFLAGS=["-fdirectives-only"])
+
+    # Ensure ccache accounts for any extra files in use that affects the generated object
+    # file. This can be used for situations where a file is passed as an argument to a
+    # compiler parameter and differences in the file need to be accounted for in the
+    # hash result to prevent erroneous cache hits.
+    if "CCACHE_EXTRAFILES" in env and env["CCACHE_EXTRAFILES"]:
+        env["ENV"]["CCACHE_EXTRAFILES"] = ":".join([
+            blackfile.path
+            for blackfile in env["CCACHE_EXTRAFILES"]
+        ])
 
     # Make a generator to expand to CCACHE in the case where we are
     # not a conftest. We don't want to use ccache for configure tests
diff --git a/site_scons/site_tools/icecream.py b/site_scons/site_tools/icecream.py
index 979bba7e3b8..7456ed0cc8f 100644
--- a/site_scons/site_tools/icecream.py
+++ b/site_scons/site_tools/icecream.py
@@ -20,14 +20,15 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-import SCons
-
 import os
 import re
 import subprocess
+import urllib
 
 from pkg_resources import parse_version
 
+import SCons
+
 _icecream_version_min = parse_version("1.1rc2")
 _icecream_version_gcc_remote_cpp = parse_version("1.2")
 
@@ -50,34 +51,44 @@ class _BoundSubstitution:
 
 def icecc_create_env(env, target, source, for_signature):
     # Safe to assume unix here because icecream only works on Unix
-    mkdir = "mkdir -p ${ICECC_VERSION.Dir('').abspath}"
+    mkdir = "mkdir -p ${TARGET.dir}"
 
     # Create the env, use awk to get just the tarball name and we store it in
     # the shell variable $ICECC_VERSION_TMP so the subsequent mv command and
     # store it in a known location. Add any files requested from the user environment.
-    create_env = "ICECC_VERSION_TMP=$$($ICECC_CREATE_ENV --$ICECC_COMPILER_TYPE $CC $CXX"
+    create_env = "ICECC_VERSION_TMP=$$(${SOURCES[0]} --$ICECC_COMPILER_TYPE ${SOURCES[1]} ${SOURCES[2]}"
+
+    # TODO: It would be a little more elegant if things in
+    # ICECC_CREATE_ENV_ADDFILES were handled as sources, because we
+    # would get automatic dependency tracking. However, there are some
+    # wrinkles around the mapped case so we have opted to leave it as
+    # just interpreting the env for now.
     for addfile in env.get('ICECC_CREATE_ENV_ADDFILES', []):
-        if (type(addfile) == tuple
-            and len(addfile) == 2):
-            if env['ICECREAM_VERSION'] > parse_version('1.1'):
-                raise Exception("This version of icecream does not support addfile remapping.")
-            create_env += " --addfile {}={}".format(
-                env.File(addfile[0]).srcnode().abspath,
-                env.File(addfile[1]).srcnode().abspath)
-            env.Depends('$ICECC_VERSION', addfile[1])
-        elif type(addfile) == str:
-            create_env += " --addfile {}".format(env.File(addfile).srcnode().abspath)
-            env.Depends('$ICECC_VERSION', addfile)
+        if isinstance(addfile, tuple):
+            if len(addfile) == 2:
+                if env['ICECREAM_VERSION'] > parse_version('1.1'):
+                    raise Exception("This version of icecream does not support addfile remapping.")
+                create_env += " --addfile {}={}".format(
+                    env.File(addfile[0]).srcnode().abspath,
+                    env.File(addfile[1]).srcnode().abspath)
+                env.Depends(target, addfile[1])
+            else:
+                raise Exception(f"Found incorrect icecream addfile format: {str(addfile)}" +
+                                f"\ntuple must two elements of the form" +
+                                f"\n('chroot dest path', 'source file path')")
         else:
-            # NOTE: abspath is required by icecream because of
-            # this line in icecc-create-env:
-            # https://github.com/icecc/icecream/blob/10b9468f5bd30a0fdb058901e91e7a29f1bfbd42/client/icecc-create-env.in#L534
-            # which cuts out the two files based off the equals sign and
-            # starting slash of the second file
-            raise Exception("Found incorrect icecream addfile format: {}" +
-                "\nicecream addfiles must be a single path or tuple path format: " +
-                "('chroot dest path', 'source file path')".format(
-                str(addfile)))
+            try:
+                create_env += f" --addfile {env.File(addfile).srcnode().abspath}"
+                env.Depends(target, addfile)
+            except:
+                # NOTE: abspath is required by icecream because of
+                # this line in icecc-create-env:
+                # https://github.com/icecc/icecream/blob/10b9468f5bd30a0fdb058901e91e7a29f1bfbd42/client/icecc-create-env.in#L534
+                # which cuts out the two files based off the equals sign and
+                # starting slash of the second file
+                raise Exception(f"Found incorrect icecream addfile format: {type(addfile)}" +
+                                f"\nvalue provided cannot be converted to a file path")
+
     create_env += " | awk '/^creating .*\\.tar\\.gz/ { print $$2 }')"
 
     # Simply move our tarball to the expected locale.
@@ -89,10 +100,6 @@ def icecc_create_env(env, target, source, for_signature):
 
 
 def generate(env):
-
-    if not exists(env):
-        return
-
     # icecc lower then 1.1 supports addfile remapping accidentally
     # and above it adds an empty cpuinfo so handle cpuinfo issues for icecream
     # below version 1.1
@@ -101,77 +108,56 @@ def generate(env):
         and os.path.exists('/proc/cpuinfo')):
         env.AppendUnique(ICECC_CREATE_ENV_ADDFILES=[('/proc/cpuinfo', '/dev/null')])
 
-    env["ICECCENVCOMSTR"] = env.get("ICECCENVCOMSTR", "Generating environment: $TARGET")
-    env["ICECC_COMPILER_TYPE"] = env.get(
-        "ICECC_COMPILER_TYPE", os.path.basename(env.WhereIs("${CC}"))
-    )
-    env.Append(
-        BUILDERS={
-            "IcecreamEnv": SCons.Builder.Builder(
-                action=SCons.Action.CommandGeneratorAction(
-                    icecc_create_env, {"comstr": "$ICECCENVCOMSTR"},
-                )
-            )
-        }
-    )
-
-    # If we are going to load the ccache tool, but we haven't done so
-    # yet, then explicitly do it now. We need the ccache tool to be in
-    # place before we setup icecream because we need to do things a
-    # little differently if ccache is in play. If you don't use the
-    # TOOLS variable to configure your tools, you should explicitly
-    # load the ccache tool before you load icecream.
-    ccache_enabled = "CCACHE_VERSION" in env
-    if "ccache" in env["TOOLS"] and not ccache_enabled:
-        env.Tool("ccache")
-
     # Absoluteify, so we can derive ICERUN
     env["ICECC"] = env.WhereIs("$ICECC")
 
-    if not "ICERUN" in env:
-        env["ICERUN"] = env.File("$ICECC").File("icerun")
-
-    # Absoluteify, for parity with ICECC
-    env["ICERUN"] = env.WhereIs("$ICERUN")
+    if "ICERUN" in env:
+        # Absoluteify, for parity with ICECC
+        icerun = env.WhereIs("$ICERUN")
+    else:
+        icerun = env.File("$ICECC").File("icerun")
+    env["ICERUN"] = icerun
 
-    env["ICECC_CREATE_ENV"] = env.WhereIs(
-        env.get("ICECC_CREATE_ENV", "icecc-create-env")
-    )
+    if "ICECC_CREATE_ENV" in env:
+        icecc_create_env_bin = env.WhereIs("$ICECC_CREATE_ENV")
+    else:
+        icecc_create_env_bin = env.File("ICECC").File("icecc-create-env")
+    env["ICECC_CREATE_ENV"] = icecc_create_env_bin
 
-    # Make CC and CXX absolute paths too. It is better for icecc.
+    # Make CC and CXX absolute paths too. This ensures the correct paths to
+    # compilers get passed to icecc-create-env rather than letting it
+    # potentially discover something we don't expect via PATH.
     env["CC"] = env.WhereIs("$CC")
     env["CXX"] = env.WhereIs("$CXX")
 
-    have_explicit_icecc_version = 'ICECC_VERSION' in env and bool(env['ICECC_VERSION'])
-    have_icecc_version_url = have_explicit_icecc_version and env["ICECC_VERSION"].startswith("http")
+    # Set up defaults for configuration options
+    env['ICECREAM_TARGET_DIR'] = env.Dir(
+        env.get('ICECREAM_TARGET_DIR', '#./.icecream')
+    )
+    verbose = env.get('ICECREAM_VERBOSE', False)
+    env['ICECREAM_DEBUG'] = env.get('ICECREAM_DEBUG', False)
+
+    # We have a lot of things to build and run that the final user
+    # environment doesn't need to see or know about. Make a custom env
+    # that we use consistently from here to where we end up setting
+    # ICECREAM_RUN_ICECC in the user env.
+    setupEnv = env.Clone(
+        NINJA_SKIP=True
+    )
+
+    if 'ICECC_VERSION' in setupEnv and bool(setupEnv['ICECC_VERSION']):
 
-    if have_explicit_icecc_version and not have_icecc_version_url:
-        icecc_version_file = env.File('$ICECC_VERSION')
-        if not icecc_version_file.exists():
-            raise Exception(
-                'The ICECC_VERSION variable set set to {}, but this file does not exist'.format(icecc_version_file)
-            )
-        env['ICECC_VERSION'] = icecc_version_file
-    else:
-        # Generate the deterministic name for our tarball
-        icecc_version_target_filename = env.subst("${CC}${CXX}.tar.gz").replace("/", "_")[
-            1:
-        ]
-        icecc_version_dir = env.Dir("$BUILD_ROOT/scons/icecc")
-        icecc_known_version = icecc_version_dir.File(icecc_version_target_filename)
-
-        if have_icecc_version_url:
-            # We do the above weaker validation as opposed to
-            # urllib.urlparse (or similar). We really only support http
-            # URLs here and any other validation either requires a third
-            # party module or accepts things we don't.
-            env["ICECC_VERSION_URL"] = env["ICECC_VERSION"]
-            env["ICECC_VERSION"] = icecc_known_version
+        if setupEnv["ICECC_VERSION"].startswith("http"):
+
+            quoted = urllib.parse.quote(setupEnv['ICECC_VERSION'], safe=[])
 
             # Use curl / wget to download the toolchain because SCons (and ninja)
             # are better at running shell commands than Python functions.
-            curl = env.WhereIs("curl")
-            wget = env.WhereIs("wget")
+            #
+            # TODO: This all happens SCons side now. Should we just use python to
+            # fetch instead?
+            curl = setupEnv.WhereIs("curl")
+            wget = setupEnv.WhereIs("wget")
 
             if curl:
                 cmdstr = "curl -L"
@@ -182,99 +168,288 @@ def generate(env):
                     "You have specified an ICECC_VERSION that is a URL but you have neither wget nor curl installed."
                 )
 
-            env.Command(
-                target="$ICECC_VERSION",
-                source=["$CC", "$CXX"],
-                action=[
-                    cmdstr + " -o $TARGET $ICECC_VERSION_URL",
-                ],
-            )
+            # Copy ICECC_VERSION into ICECC_VERSION_URL so that we can
+            # change ICECC_VERSION without perturbing the effect of
+            # the action.
+            setupEnv['ICECC_VERSION_URL'] = setupEnv['ICECC_VERSION']
+            setupEnv['ICECC_VERSION'] = icecc_version_file = setupEnv.Command(
+                target=f"$ICECREAM_TARGET_DIR/{quoted}",
+                source=[setupEnv.Value(quoted)],
+                action=SCons.Action.Action(
+                    f"{cmdstr} -o $TARGET $ICECC_VERSION_URL",
+                    "Downloading compiler package from $ICECC_VERSION_URL" if not verbose else str(),
+                ),
+            )[0]
+
         else:
-            # Make a predictable name for the toolchain
-            env["ICECC_VERSION"] = env.File(icecc_known_version)
-            env.IcecreamEnv(
-                target="$ICECC_VERSION",
-                source=["$ICECC_CREATE_ENV", "$CC", "$CXX"],
+            # Convert the users selection into a File node and do some basic validation
+            setupEnv['ICECC_VERSION'] = icecc_version_file = setupEnv.File('$ICECC_VERSION')
+
+            if not icecc_version_file.exists():
+                raise Exception(
+                    'The ICECC_VERSION variable set set to {}, but this file does not exist'.format(icecc_version_file)
+                )
+
+        # This is what we are going to call the file names as known to SCons on disk
+        setupEnv["ICECC_VERSION_ID"] = "user_provided." + icecc_version_file.name
+
+    else:
+
+        setupEnv["ICECC_COMPILER_TYPE"] = setupEnv.get(
+            "ICECC_COMPILER_TYPE", os.path.basename(setupEnv.WhereIs("${CC}"))
+        )
+
+        # This is what we are going to call the file names as known to SCons on disk. We do the
+        # subst early so that we can call `replace` on the result.
+        setupEnv["ICECC_VERSION_ID"] = setupEnv.subst(
+            "icecc-create-env.${CC}${CXX}.tar.gz").replace("/", "_"
+        )
+
+        setupEnv["ICECC_VERSION"] = icecc_version_file = setupEnv.Command(
+            target="$ICECREAM_TARGET_DIR/$ICECC_VERSION_ID",
+            source=[
+                "$ICECC_CREATE_ENV",
+                "$CC",
+                "$CXX"
+            ],
+            action=SCons.Action.Action(
+                icecc_create_env,
+                "Generating icecream compiler package: $TARGET" if not verbose else str(),
+                generator=True,
             )
+        )[0]
 
-        # Our ICECC_VERSION isn't just a file, so we need to make
-        # things depend on it to ensure that it comes into being at
-        # the right time. Don't do that for conftests though: we never
-        # want to run them remote.
-        def icecc_toolchain_dependency_emitter(target, source, env):
-            if "conftest" not in str(target[0]):
-                env.Requires(target, "$ICECC_VERSION")
-            return target, source
-
-        # Cribbed from Tool/cc.py and Tool/c++.py. It would be better if
-        # we could obtain this from SCons.
-        _CSuffixes = [".c"]
-        if not SCons.Util.case_sensitive_suffixes(".c", ".C"):
-            _CSuffixes.append(".C")
-
-        _CXXSuffixes = [".cpp", ".cc", ".cxx", ".c++", ".C++"]
-        if SCons.Util.case_sensitive_suffixes(".c", ".C"):
-            _CXXSuffixes.append(".C")
-
-        suffixes = _CSuffixes + _CXXSuffixes
-        for object_builder in SCons.Tool.createObjBuilders(env):
-            emitterdict = object_builder.builder.emitter
-            for suffix in emitterdict.keys():
-                if not suffix in suffixes:
-                    continue
-                base = emitterdict[suffix]
-                emitterdict[suffix] = SCons.Builder.ListEmitter(
-                    [base, icecc_toolchain_dependency_emitter]
+    # At this point, all paths above have produced a file of some sort. We now move on
+    # to producing our own signature for this local file.
+
+    setupEnv.Append(
+        ICECREAM_TARGET_BASE_DIR='$ICECREAM_TARGET_DIR',
+        ICECREAM_TARGET_BASE_FILE='$ICECC_VERSION_ID',
+        ICECREAM_TARGET_BASE='$ICECREAM_TARGET_BASE_DIR/$ICECREAM_TARGET_BASE_FILE',
+    )
+
+    # If the file we are planning to use is not within
+    # ICECREAM_TARGET_DIR then make a local copy of it that is.
+    if icecc_version_file.dir != env['ICECREAM_TARGET_DIR']:
+        setupEnv["ICECC_VERSION"] = icecc_version_file = setupEnv.Command(
+            target=[
+                '${ICECREAM_TARGET_BASE}.local',
+            ],
+            source=icecc_version_file,
+            action=SCons.Defaults.Copy('$TARGET', '$SOURCE'),
+        )
+
+        # There is no point caching the copy.
+        setupEnv.NoCache(icecc_version_file)
+
+    # Now, we compute our own signature of the local compiler package,
+    # and create yet another link to the compiler package with a name
+    # containing our computed signature. Now we know that we can give
+    # this filename to icecc and it will be assured to really reflect
+    # the contents of the package, and not the arbitrary naming of the
+    # file as found on the users filesystem or from
+    # icecc-create-env. We put the absolute path to that filename into
+    # a file that we can read from.
+    icecc_version_info = setupEnv.File(setupEnv.Command(
+        target=[
+            '${ICECREAM_TARGET_BASE}.sha256',
+            '${ICECREAM_TARGET_BASE}.sha256.path',
+        ],
+        source=icecc_version_file,
+        action=SCons.Action.ListAction(
+            [
+
+                # icecc-create-env run twice with the same input will
+                # create files with identical contents, and identical
+                # filenames, but with different hashes because it
+                # includes timestamps. So we compute a new hash based
+                # on the actual stream contents of the file by
+                # untarring it into shasum.
+                SCons.Action.Action(
+                    "tar xfO ${SOURCES[0]} | shasum -b -a 256 - | awk '{ print $1 }' > ${TARGETS[0]}",
+                    "Calculating sha256 sum of ${SOURCES[0]}" if not verbose else str(),
+                ),
+
+                SCons.Action.Action(
+                    "ln -f ${SOURCES[0]} ${TARGETS[0].dir}/icecream_py_sha256_$$(cat ${TARGETS[0]}).tar.gz",
+                    "Linking ${SOURCES[0]} to its sha256 sum name" if not verbose else str(),
+                ),
+
+                SCons.Action.Action(
+                    "echo ${TARGETS[0].dir.abspath}/icecream_py_sha256_$$(cat ${TARGETS[0]}).tar.gz > ${TARGETS[1]}",
+                    "Storing sha256 sum name for ${SOURCES[0]} to ${TARGETS[1]}" if not verbose else str(),
                 )
+            ],
+        )
+    ))
+
+    # We can't allow these to interact with the cache because the
+    # second action produces a file unknown to SCons. If caching were
+    # permitted, the other two files could be retrieved from cache but
+    # the file produced by the second action could not (and would not)
+    # be. We would end up with a broken setup.
+    setupEnv.NoCache(icecc_version_info)
+
+    # Create a value node that, when built, contains the result of
+    # reading the contents of the sha256.path file. This way we can
+    # pull the value out of the file and substitute it into our
+    # wrapper script.
+    icecc_version_string_value = setupEnv.Command(
+        target=setupEnv.Value(None),
+        source=[
+            icecc_version_info[1]
+        ],
+        action=SCons.Action.Action(
+            lambda env, target, source: target[0].write(source[0].get_text_contents()),
+            "Reading compiler package sha256 sum path from $SOURCE" if not verbose else str(),
+        )
+    )[0]
+
+    def icecc_version_string_generator(source, target, env, for_signature):
+        if for_signature:
+            return icecc_version_string_value.get_csig()
+        return icecc_version_string_value.read()
+
+    # Set the values that will be interpolated into the run-icecc script.
+    setupEnv['ICECC_VERSION'] = icecc_version_string_generator
+
+    # If necessary, we include the users desired architecture in the
+    # interpolated file.
+    icecc_version_arch_string = str()
+    if "ICECC_VERSION_ARCH" in setupEnv:
+        icecc_version_arch_string = "${ICECC_VERSION_ARCH}:"
+
+    # Finally, create the run-icecc wrapper script. The contents will
+    # re-invoke icecc with our sha256 sum named file, ensuring that we
+    # trust the signature to be appropriate. In a pure SCons build, we
+    # actually wouldn't need this Substfile, we could just set
+    # env['ENV]['ICECC_VERSION'] to the Value node above. But that
+    # won't work for Ninja builds where we can't ask for the contents
+    # of such a node easily. Creating a Substfile means that SCons
+    # will take care of generating a file that Ninja can use.
+    run_icecc = setupEnv.Textfile(
+        target="$ICECREAM_TARGET_DIR/run-icecc.sh",
+        source=[
+            '#!/bin/sh',
+            'ICECC_VERSION=@icecc_version_arch@@icecc_version@ exec @icecc@ "$@"',
+            '',
+        ],
+        SUBST_DICT={
+            '@icecc@' : '$ICECC',
+            '@icecc_version@' : '$ICECC_VERSION',
+            '@icecc_version_arch@' : icecc_version_arch_string,
+        },
+
+        # Don't change around the suffixes
+        TEXTFILEPREFIX=str(),
+        TEXTFILESUFFIX=str(),
+
+        # Somewhat surprising, but even though Ninja will defer to
+        # SCons to invoke this, we still need ninja to be aware of it
+        # so that it knows to invoke SCons to produce it as part of
+        # TEMPLATE expansion. Since we have set NINJA_SKIP=True for
+        # setupEnv, we need to reverse that here.
+        NINJA_SKIP=False
+    )
+
+    setupEnv.AddPostAction(
+        run_icecc,
+        action=SCons.Defaults.Chmod('$TARGET', "u+x"),
+    )
+
+    setupEnv.Depends(
+        target=run_icecc,
+        dependency=[
+
+            # TODO: Without the ICECC dependency, changing ICECC doesn't cause the Substfile
+            # to regenerate. Why is this?
+            '$ICECC',
+
+            # This dependency is necessary so that we build into this
+            # string before we create the file.
+            icecc_version_string_value,
+
+            # TODO: SERVER-50587 We need to make explicit depends here because of NINJA_SKIP. Any
+            # dependencies in the nodes created in setupEnv with NINJA_SKIP would have
+            # that dependency chain hidden from ninja, so they won't be rebuilt unless
+            # added as dependencies here on this node that has NINJA_SKIP=False.
+            '$CC',
+            '$CXX',
+            icecc_version_file,
+        ],
+    )
+
+    # From here out, we make changes to the users `env`.
+    setupEnv = None
+
+    env['ICECREAM_RUN_ICECC'] = run_icecc[0]
+
+    def icecc_toolchain_dependency_emitter(target, source, env):
+        if "conftest" not in str(target[0]):
+            # Requires or Depends? There are trade-offs:
+            #
+            # If it is `Depends`, then enabling or disabling icecream
+            # will cause a global recompile. But, if you regenerate a
+            # new compiler package, you will get a rebuild. If it is
+            # `Requires`, then enabling or disabling icecream will not
+            # necessarily cause a global recompile (it depends if
+            # C[,C,XX]FLAGS get changed when you do so), but on the
+            # other hand if you regenerate a new compiler package you
+            # will *not* get a rebuild.
+            #
+            # For now, we are opting for `Requires`, because it seems
+            # preferable that opting in or out of icecream shouldn't
+            # force a rebuild.
+            env.Requires(target, "$ICECREAM_RUN_ICECC")
+        return target, source
+
+    # Cribbed from Tool/cc.py and Tool/c++.py. It would be better if
+    # we could obtain this from SCons.
+    _CSuffixes = [".c"]
+    if not SCons.Util.case_sensitive_suffixes(".c", ".C"):
+        _CSuffixes.append(".C")
+
+    _CXXSuffixes = [".cpp", ".cc", ".cxx", ".c++", ".C++"]
+    if SCons.Util.case_sensitive_suffixes(".c", ".C"):
+        _CXXSuffixes.append(".C")
+
+    suffixes = _CSuffixes + _CXXSuffixes
+    for object_builder in SCons.Tool.createObjBuilders(env):
+        emitterdict = object_builder.builder.emitter
+        for suffix in emitterdict.keys():
+            if not suffix in suffixes:
+                continue
+            base = emitterdict[suffix]
+            emitterdict[suffix] = SCons.Builder.ListEmitter(
+                [base, icecc_toolchain_dependency_emitter]
+            )
+
+    # Check whether ccache is requested and is a valid tool.
+    if "CCACHE" in env:
+        ccache = SCons.Tool.Tool('ccache')
+        ccache_enabled = bool(ccache) and ccache.exists(env)
+    else:
+        ccache_enabled = False
 
     if env.ToolchainIs("clang"):
         env["ENV"]["ICECC_CLANG_REMOTE_CPP"] = 1
     elif env.ToolchainIs("gcc"):
-        if env["ICECREAM_VERSION"] >= _icecream_version_gcc_remote_cpp:
-            if ccache_enabled:
-                # Newer versions of Icecream will drop -fdirectives-only from
-                # preprocessor and compiler flags if it does not find a remote
-                # build host to build on. ccache, on the other hand, will not
-                # pass the flag to the compiler if CCACHE_NOCPP2=1, but it will
-                # pass it to the preprocessor. The combination of setting
-                # CCACHE_NOCPP2=1 and passing the flag can lead to build
-                # failures.
-
-                # See: https://jira.mongodb.org/browse/SERVER-48443
-
-                # We have an open issue with Icecream and ccache to resolve the
-                # cause of these build failures. Once the bug is resolved and
-                # the fix is deployed, we can remove this entire conditional
-                # branch and make it like the one for clang.
-                # TODO: https://github.com/icecc/icecream/issues/550
-                env["ENV"].pop("CCACHE_NOCPP2", None)
-                env["ENV"]["CCACHE_CPP2"] = 1
-                try:
-                    env["CCFLAGS"].remove("-fdirectives-only")
-                except ValueError:
-                    pass
-            else:
-                # If we can, we should make Icecream do its own preprocessing
-                # to reduce concurrency on the local host. We should not do
-                # this when ccache is in use because ccache will execute
-                # Icecream to do its own preprocessing and then execute
-                # Icecream as the compiler on the preprocessed source.
-                env["ENV"]["ICECC_REMOTE_CPP"] = 1
+        if env["ICECREAM_VERSION"] < _icecream_version_gcc_remote_cpp:
+            # We aren't going to use ICECC_REMOTE_CPP because icecc
+            # 1.1 doesn't offer it. We disallow fallback to local
+            # builds because the fallback is serial execution.
+            env["ENV"]["ICECC_CARET_WORKAROUND"] = 0
+        elif not ccache_enabled:
+            # If we can, we should make Icecream do its own preprocessing
+            # to reduce concurrency on the local host. We should not do
+            # this when ccache is in use because ccache will execute
+            # Icecream to do its own preprocessing and then execute
+            # Icecream as the compiler on the preprocessed source.
+            env["ENV"]["ICECC_REMOTE_CPP"] = 1
 
     if "ICECC_SCHEDULER" in env:
         env["ENV"]["USE_SCHEDULER"] = env["ICECC_SCHEDULER"]
 
-    # Build up the string we will set in the environment to tell icecream
-    # about the compiler package.
-    icecc_version_string = "${ICECC_VERSION.abspath}"
-    if "ICECC_VERSION_ARCH" in env:
-        icecc_version_string = "${ICECC_VERSION_ARCH}:" + icecc_version_string
-
-    # Use our BoundSubstitition class to put ICECC_VERSION into env['ENV'] with
-    # substitution in play. This avoids an early subst which can behave
-    # strangely.
-    env["ENV"]["ICECC_VERSION"] = _BoundSubstitution(env, icecc_version_string)
-
     # If ccache is in play we actually want the icecc binary in the
     # CCACHE_PREFIX environment variable, not on the command line, per
     # the ccache documentation on compiler wrappers. Otherwise, just
@@ -285,19 +460,22 @@ def generate(env):
     # compiler flags (things like -fdirectives-only), but we still try
     # to do the right thing.
     if ccache_enabled:
-        env["ENV"]["CCACHE_PREFIX"] = _BoundSubstitution(env, "$ICECC")
+        # If the path to CCACHE_PREFIX isn't absolute, then it will
+        # look it up in PATH. That isn't what we want here, we make
+        # the path absolute.
+        env['ENV']['CCACHE_PREFIX'] = _BoundSubstitution(env, "${ICECREAM_RUN_ICECC.abspath}")
     else:
         # Make a generator to expand to ICECC in the case where we are
-        # not a conftest. We never want to run conftests
-        # remotely. Ideally, we would do this for the CCACHE_PREFIX
-        # case above, but unfortunately if we did we would never
-        # actually see the conftests, because the BoundSubst means
-        # that we will never have a meaningful `target` variable when
-        # we are in ENV. Instead, rely on the ccache.py tool to do
-        # it's own filtering out of conftests.
+        # not a conftest. We never want to run conftests remotely.
+        # Ideally, we would do this for the CCACHE_PREFIX case above,
+        # but unfortunately if we did we would never actually see the
+        # conftests, because the BoundSubst means that we will never
+        # have a meaningful `target` variable when we are in ENV.
+        # Instead, rely on the ccache.py tool to do it's own filtering
+        # out of conftests.
         def icecc_generator(target, source, env, for_signature):
             if "conftest" not in str(target[0]):
-                return '$ICECC'
+                return '$ICECREAM_RUN_ICECC'
             return ''
         env['ICECC_GENERATOR'] = icecc_generator
 
@@ -327,22 +505,29 @@ def generate(env):
             env[command] = " ".join(["$( $ICERUN $)", env[command]])
 
     # Uncomment these to debug your icecc integration
-    # env['ENV']['ICECC_DEBUG'] = 'debug'
-    # env['ENV']['ICECC_LOGFILE'] = 'icecc.log'
+    if env['ICECREAM_DEBUG']:
+        env['ENV']['ICECC_DEBUG'] = 'debug'
+        env['ENV']['ICECC_LOGFILE'] = 'icecc.log'
 
 
 def exists(env):
-    # Assume the tool has run if we already know the version.
-    if "ICECREAM_VERSION" in env:
-        return True
-
-    icecc = env.get("ICECC", False)
-    if not icecc:
+    if not env.subst("$ICECC"):
         return False
-    icecc = env.WhereIs(icecc)
+
+    icecc = env.WhereIs("$ICECC")
     if not icecc:
+        # TODO: We should not be printing here because we don't always know the
+        # use case for loading this tool. It may be that the user desires
+        # writing this output to a log file or not even displaying it at all.
+        # We should instead be invoking a callback to SConstruct that it can
+        # interpret as needed. Or better yet, we should use some SCons logging
+        # and error API, if and when one should emerge.
+        print(f"Error: icecc not found at {env['ICECC']}")
         return False
 
+    if 'ICECREAM_VERSION' in env and env['ICECREAM_VERSION'] >= _icecream_version_min:
+        return True
+
     pipe = SCons.Action._subproc(
         env,
         SCons.Util.CLVar(icecc) + ["--version"],
@@ -352,9 +537,26 @@ def exists(env):
     )
 
     if pipe.wait() != 0:
+        print(f"Error: failed to execute '{env['ICECC']}'")
         return False
 
     validated = False
+
+    if "ICERUN" in env:
+        # Absoluteify, for parity with ICECC
+        icerun = env.WhereIs("$ICERUN")
+    else:
+        icerun = env.File("$ICECC").File("icerun")
+    if not icerun:
+        print(f"Error: the icerun wrapper does not exist at {icerun} as expected")
+
+    if "ICECC_CREATE_ENV" in env:
+        icecc_create_env_bin = env.WhereIs("$ICECC_CREATE_ENV")
+    else:
+        icecc_create_env_bin = env.File("ICECC").File("icecc-create-env")
+    if not icecc_create_env_bin:
+        print(f"Error: the icecc-create-env utility does not exist at {icecc_create_env_bin} as expected")
+
     for line in pipe.stdout:
         line = line.decode("utf-8")
         if validated:
@@ -371,5 +573,7 @@ def exists(env):
 
     if validated:
         env['ICECREAM_VERSION'] = icecc_version
+    else:
+        print(f"Error: failed to verify icecream version >= {_icecream_version_min}, found {icecc_version}")
 
     return validated
diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py
index 0ce56407439..6448bf428b5 100644
--- a/site_scons/site_tools/ninja.py
+++ b/site_scons/site_tools/ninja.py
@@ -28,6 +28,7 @@ import importlib
 import io
 import shutil
 import shlex
+import textwrap
 
 from glob import glob
 from os.path import join as joinpath
@@ -138,7 +139,7 @@ def get_order_only(node):
     """Return a list of order only dependencies for node."""
     if node.prerequisites is None:
         return []
-    return [get_path(src_file(prereq)) for prereq in node.prerequisites]
+    return [get_path(src_file(prereq)) for prereq in node.prerequisites if is_valid_dependent_node(prereq)]
 
 
 def get_dependencies(node, skip_sources=False):
@@ -147,21 +148,44 @@ def get_dependencies(node, skip_sources=False):
         return [
             get_path(src_file(child))
             for child in node.children()
-            if child not in node.sources
+            if child not in node.sources and is_valid_dependent_node(child)
         ]
-    return [get_path(src_file(child)) for child in node.children()]
+    return [get_path(src_file(child)) for child in node.children() if is_valid_dependent_node(child)]
 
 
-def get_inputs(node):
-    """Collect the Ninja inputs for node."""
+def get_inputs(node, skip_unknown_types=False):
+    """
+    Collect the Ninja inputs for node.
+
+    If the given node has inputs which can not be converted into something
+    Ninja can process, this will throw an exception. Optionally, those nodes
+    that are not processable can be skipped as inputs with the
+    skip_unknown_types keyword arg.
+    """
     executor = node.get_executor()
     if executor is not None:
         inputs = executor.get_all_sources()
     else:
         inputs = node.sources
 
-    inputs = [get_path(src_file(o)) for o in inputs]
-    return inputs
+    # Some Nodes (e.g. Python.Value Nodes) won't have files associated. We allow these to be
+    # optionally skipped to enable the case where we will re-invoke SCons for things
+    # like TEMPLATE. Otherwise, we have no direct way to express the behavior for such
+    # Nodes in Ninja, so we raise a hard error
+    ninja_nodes = []
+    for input_node in inputs:
+        if isinstance(input_node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)):
+            ninja_nodes.append(input_node)
+        else:
+            if skip_unknown_types:
+                continue
+            raise Exception("Can't process {} node '{}' as an input for '{}'".format(
+                type(input_node),
+                str(input_node),
+                str(node)))
+
+    # convert node items into raw paths/aliases for ninja
+    return [get_path(src_file(o)) for o in ninja_nodes]
 
 
 def get_outputs(node):
@@ -179,6 +203,40 @@ def get_outputs(node):
 
     return outputs
 
+def generate_depfile(env, node, dependencies):
+    """
+    Ninja tool function for writing a depfile. The depfile should include
+    the node path followed by all the dependent files in a makefile format.
+
+    dependencies arg can be a list or a subst generator which returns a list.
+    """
+
+    depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile')
+
+    # subst_list will take in either a raw list or a subst callable which generates
+    # a list, and return a list of CmdStringHolders which can be converted into raw strings.
+    # If a raw list was passed in, then scons_list will make a list of lists from the original
+    # values and even subst items in the list if they are substitutable. Flatten will flatten
+    # the list in that case, to ensure for either input we have a list of CmdStringHolders.
+    deps_list = env.Flatten(env.subst_list(dependencies))
+
+    # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings
+    # and make sure to escape the strings to handle spaces in paths. We also will sort the result
+    # keep the order of the list consistent.
+    escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list])
+    depfile_contents = str(node) + ": " + ' '.join(escaped_depends)
+
+    need_rewrite = False
+    try:
+        with open(depfile, 'r') as f:
+            need_rewrite = (f.read() != depfile_contents)
+    except FileNotFoundError:
+        need_rewrite = True
+
+    if need_rewrite:
+        os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True)
+        with open(depfile, 'w') as f:
+            f.write(depfile_contents)
 
 class SConsToNinjaTranslator:
     """Translates SCons Actions into Ninja build objects."""
@@ -250,11 +308,14 @@ class SConsToNinjaTranslator:
         # dependencies don't really matter when we're going to shove these to
         # the bottom of ninja's DAG anyway and Textfile builders can have text
         # content as their source which doesn't work as an implicit dep in
-        # ninja.
+        # ninja. We suppress errors on input Nodes types that we cannot handle
+        # since we expect that the re-invocation of SCons will handle dependency
+        # tracking for those Nodes and their dependents.
         if name == "_action":
             return {
                 "rule": "TEMPLATE",
                 "outputs": get_outputs(node),
+                "inputs": get_inputs(node, skip_unknown_types=True),
                 "implicit": get_dependencies(node, skip_sources=True),
             }
 
@@ -425,8 +486,16 @@ class NinjaState:
                 "rspfile_content": "$rspc",
                 "pool": "local_pool",
             },
+            # Ninja does not automatically delete the archive before
+            # invoking ar. The ar utility will append to an existing archive, which
+            # can cause duplicate symbols if the symbols moved between object files.
+            # Native SCons will perform this operation so we need to force ninja
+            # to do the same. See related for more info:
+            # https://jira.mongodb.org/browse/SERVER-49457
             "AR": {
-                "command": "$env$AR @$out.rsp",
+                "command": "{}$env$AR @$out.rsp".format(
+                    '' if sys.platform == "win32" else "rm -f $out && "
+                ),
                 "description": "Archiving $out",
                 "rspfile": "$out.rsp",
                 "rspfile_content": "$rspc",
@@ -486,6 +555,7 @@ class NinjaState:
                 "command": "$SCONS_INVOCATION_W_TARGETS",
                 "description": "Regenerating $out",
                 "generator": 1,
+                "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'),
                 # Console pool restricts to 1 job running at a time,
                 # it additionally has some special handling about
                 # passing stdin, stdout, etc to process in this pool
@@ -564,6 +634,8 @@ class NinjaState:
 
         ninja.comment("Generated by scons. DO NOT EDIT.")
 
+        ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR']))
+
         for pool_name, size in self.pools.items():
             ninja.pool(pool_name, size)
 
@@ -673,6 +745,16 @@ class NinjaState:
 
                 build["outputs"] = first_output
 
+            # Optionally a rule can specify a depfile, and SCons can generate implicit
+            # dependencies into the depfile. This allows for dependencies to come and go
+            # without invalidating the ninja file. The depfile was created in ninja specifically
+            # for dealing with header files appearing and disappearing across rebuilds, but it can
+            # be repurposed for anything, as long as you have a way to regenerate the depfile.
+            # More specific info can be found here: https://ninja-build.org/manual.html#_depfile
+            if rule is not None and rule.get('depfile') and build.get('deps_files'):
+                path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']]
+                generate_depfile(self.env, path[0], build.pop('deps_files', []))
+
             if "inputs" in build:
                 build["inputs"].sort()
 
@@ -683,13 +765,13 @@ class NinjaState:
 
             # Special handling for outputs and implicit since we need to
             # aggregate not replace for each builder.
-            for agg_key in ["outputs", "implicit"]:
+            for agg_key in ["outputs", "implicit", "inputs"]:
                 new_val = template_builds.get(agg_key, [])
 
                 # Use pop so the key is removed and so the update
                 # below will not overwrite our aggregated values.
                 cur_val = template_builder.pop(agg_key, [])
-                if isinstance(cur_val, list):
+                if is_List(cur_val):
                     new_val += cur_val
                 else:
                     new_val.append(cur_val)
@@ -707,19 +789,25 @@ class NinjaState:
         # generate this rule even though SCons should know we're
         # dependent on SCons files.
         #
-        # TODO: We're working on getting an API into SCons that will
-        # allow us to query the actual SConscripts used. Right now
-        # this glob method has deficiencies like skipping
-        # jstests/SConscript and being specific to the MongoDB
-        # repository layout.
+        # The REGENERATE rule uses depfile, so we need to generate the depfile
+        # in case any of the SConscripts have changed. The depfile needs to be
+        # path with in the build and the passed ninja file is an abspath, so
+        # we will use SCons to give us the path within the build. Normally
+        # generate_depfile should not be called like this, but instead be called
+        # through the use of custom rules, and filtered out in the normal
+        # list of build generation about. However, because the generate rule
+        # is hardcoded here, we need to do this generate_depfile call manually.
+        ninja_file_path = self.env.File(ninja_file).path
+        generate_depfile(
+            self.env,
+            ninja_file_path,
+            self.env['NINJA_REGENERATE_DEPS']
+        )
+
         ninja.build(
-            self.env.File(ninja_file).path,
+            ninja_file_path,
             rule="REGENERATE",
-            implicit=[
-                self.env.File("#SConstruct").path,
-                __file__,
-            ]
-            + sorted(glob("src/**/SConscript", recursive=True)),
+            implicit=[__file__],
         )
 
         # If we ever change the name/s of the rules that include
@@ -731,8 +819,8 @@ class NinjaState:
             pool="console",
             implicit=[ninja_file],
             variables={
-                "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format(
-                    ninja_file
+                "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format(
+                    ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
                 )
             },
         )
@@ -848,7 +936,13 @@ def get_command_env(env):
         if windows:
             command_env += "set '{}={}' && ".format(key, value)
         else:
-            command_env += "{}={} ".format(key, value)
+            # We address here *only* the specific case that a user might have
+            # an environment variable which somehow gets included and has
+            # spaces in the value. These are escapes that Ninja handles. This
+            # doesn't make builds on paths with spaces (Ninja and SCons issues)
+            # nor expanding response file paths with spaces (Ninja issue) work.
+            value = value.replace(r' ', r'$ ')
+            command_env += "{}='{}' ".format(key, value)
 
     env["NINJA_ENV_VAR_CACHE"] = command_env
     return command_env
@@ -902,7 +996,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False):
         variables[rule] = cmd
         if use_command_env:
             variables["env"] = get_command_env(env)
-        return rule, variables
+        return rule, variables, [tool_command]
 
     return get_response_file_command
 
@@ -932,13 +1026,21 @@ def generate_command(env, node, action, targets, sources, executor=None):
     return cmd.replace("$", "$$")
 
 
-def get_shell_command(env, node, action, targets, sources, executor=None):
+def get_generic_shell_command(env, node, action, targets, sources, executor=None):
     return (
         "CMD",
         {
             "cmd": generate_command(env, node, action, targets, sources, executor=None),
             "env": get_command_env(env),
         },
+        # Since this function is a rule mapping provider, it must return a list of dependencies,
+        # and usually this would be the path to a tool, such as a compiler, used for this rule.
+        # However this function is to generic to be able to reliably extract such deps
+        # from the command, so we return a placeholder empty list. It should be noted that
+        # generally this function will not be used soley and is more like a template to generate
+        # the basics for a custom provider which may have more specific options for a provier
+        # function for a custom NinjaRuleMapping.
+        []
     )
 
 
@@ -974,12 +1076,39 @@ def get_command(env, node, action):  # pylint: disable=too-many-branches
     if not comstr:
         return None
 
-    provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command)
-    rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor)
+    provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command)
+    rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor)
 
     # Get the dependencies for all targets
     implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)})
 
+    # Now add in the other dependencies related to the command,
+    # e.g. the compiler binary. The ninja rule can be user provided so
+    # we must do some validation to resolve the dependency path for ninja.
+    for provider_dep in provider_deps:
+
+        provider_dep = sub_env.subst(provider_dep)
+        if not provider_dep:
+            continue
+
+        # If the tool is a node, then SCons will resolve the path later, if its not
+        # a node then we assume it generated from build and make sure it is existing.
+        if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep):
+            implicit.append(provider_dep)
+            continue
+
+        # Many commands will assume the binary is in the path, so
+        # we accept this as a possible input from a given command.
+        provider_dep_abspath = sub_env.WhereIs(provider_dep)
+        if provider_dep_abspath:
+            implicit.append(provider_dep_abspath)
+            continue
+
+        # Possibly these could be ignore and the build would still work, however it may not always
+        # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided
+        # ninja rule.
+        raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'")
+
     ninja_build = {
         "order_only": get_order_only(node),
         "outputs": get_outputs(node),
@@ -1042,18 +1171,21 @@ def register_custom_handler(env, name, handler):
 
 
 def register_custom_rule_mapping(env, pre_subst_string, rule):
-    """Register a custom handler for SCons function actions."""
+    """Register a function to call for a given rule."""
     global __NINJA_RULE_MAPPING
     __NINJA_RULE_MAPPING[pre_subst_string] = rule
 
 
-def register_custom_rule(env, rule, command, description="", deps=None, pool=None):
+def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False):
     """Allows specification of Ninja rules from inside SCons files."""
     rule_obj = {
         "command": command,
         "description": description if description else "{} $out".format(rule),
     }
 
+    if use_depfile:
+        rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile')
+
     if deps is not None:
         rule_obj["deps"] = deps
 
@@ -1091,6 +1223,27 @@ def ninja_contents(original):
 
     return wrapper
 
+def CheckNinjaCompdbExpand(env, context):
+    """ Configure check testing if ninja's compdb can expand response files"""
+
+    context.Message('Checking if ninja compdb can expand response files... ')
+    ret, output = context.TryAction(
+        action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET',
+        extension='.ninja',
+        text=textwrap.dedent("""
+            rule CMD_RSP
+              command = $cmd @$out.rsp > fake_output.txt
+              description = Building $out
+              rspfile = $out.rsp
+              rspfile_content = $rspc
+            build fake_output.txt: CMD_RSP fake_input.txt
+              cmd = echo
+              pool = console
+              rspc = "test"
+            """))
+    result = '@fake_output.txt.rsp' not in output
+    context.Result(result)
+    return result
 
 def ninja_stat(_self, path):
     """
@@ -1201,12 +1354,20 @@ def generate(env):
     env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build")
     env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja")
     env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja")
-
+    env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path)
     ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}")
     ninja_file = env.Ninja(target=ninja_file_name, source=[])
     env.AlwaysBuild(ninja_file)
     env.Alias("$NINJA_ALIAS_NAME", ninja_file)
 
+    # TODO: API for getting the SConscripts programmatically
+    # exists upstream: https://github.com/SCons/scons/issues/3625
+    def ninja_generate_deps(env):
+        return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True))
+    env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps
+
+    env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}')
+
     # This adds the required flags such that the generated compile
     # commands will create depfiles as appropriate in the Ninja file.
     if env["PLATFORM"] == "win32":
@@ -1214,9 +1375,11 @@ def generate(env):
     else:
         env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"])
 
+    env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand")
+
     # Provide a way for custom rule authors to easily access command
     # generation.
-    env.AddMethod(get_shell_command, "NinjaGetShellCommand")
+    env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand")
     env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider")
 
     # Provides a way for users to handle custom FunctionActions they