SERVER-48443 Fix builds with Icecream 1.2+ and gcc 4.4+

A bug spotted in Icecream 1.2+ can cause build failures when building
with gcc. This is, in turn, due to a bug in GCC where the preprocessor
executed via `gcc -E` has different behavior than the one used
internally during compilation. We are working with Icecream, and GCC
to address these problems. For now, we work around the bugs.

    * GCC bug report: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88475
    * Icecream bug report: icecc/icecream#550

2 files changed, 46 insertions, 9 deletions

diff --git a/site_scons/site_tools/ccache.py b/site_scons/site_tools/ccache.py
index 07de63f70a9..1a34571cc7e 100644
--- a/site_scons/site_tools/ccache.py
+++ b/site_scons/site_tools/ccache.py
@@ -111,6 +111,17 @@ def generate(env):
     # but it doesn't work or is out of date.
     env["CCACHE_VERSION"] = _ccache_version_found
 
+    # Set up a performant ccache configuration. Here, we don't use a second preprocessor and
+    # pass preprocessor arguments that deterministically expand source files so a stable
+    # hash can be calculated on them. This both reduces the amount of work ccache needs to
+    # do and increases the likelihood of a cache hit.
+    env["ENV"]["CCACHE_NOCPP2"] = 1
+    if env.ToolchainIs("clang"):
+        env.AppendUnique(CCFLAGS=["-frewrite-includes"])
+    elif env.ToolchainIs("gcc"):
+        env.AppendUnique(CCFLAGS=["-fdirectives-only"])
+
+
     # Make a generator to expand to CCACHE in the case where we are
     # not a conftest. We don't want to use ccache for configure tests
     # because we don't want to use icecream for configure tests, but
diff --git a/site_scons/site_tools/icecream.py b/site_scons/site_tools/icecream.py
index d77b53d1ea4..9a1768a8f80 100644
--- a/site_scons/site_tools/icecream.py
+++ b/site_scons/site_tools/icecream.py
@@ -29,7 +29,7 @@ import subprocess
 from pkg_resources import parse_version
 
 _icecream_version_min = parse_version("1.1rc2")
-_ccache_nocpp2_version = parse_version("3.4.1")
+_icecream_version_gcc_remote_cpp = parse_version("1.2")
 
 
 # I'd prefer to use value here, but amazingly, its __str__ returns the
@@ -238,14 +238,37 @@ def generate(env):
 
     if env.ToolchainIs("clang"):
         env["ENV"]["ICECC_CLANG_REMOTE_CPP"] = 1
-
-        if ccache_enabled and env["CCACHE_VERSION"] >= _ccache_nocpp2_version:
-            env.AppendUnique(CCFLAGS=["-frewrite-includes"])
-            env["ENV"]["CCACHE_NOCPP2"] = 1
-    else:
-        env.AppendUnique(CCFLAGS=["-fdirectives-only"])
-        if ccache_enabled:
-            env["ENV"]["CCACHE_NOCPP2"] = 1
+    elif env.ToolchainIs("gcc"):
+        if env["ICECREAM_VERSION"] >= _icecream_version_gcc_remote_cpp:
+            if ccache_enabled:
+                # Newer versions of Icecream will drop -fdirectives-only from
+                # preprocessor and compiler flags if it does not find a remote
+                # build host to build on. ccache, on the other hand, will not
+                # pass the flag to the compiler if CCACHE_NOCPP2=1, but it will
+                # pass it to the preprocessor. The combination of setting
+                # CCACHE_NOCPP2=1 and passing the flag can lead to build
+                # failures.
+
+                # See: https://jira.mongodb.org/browse/SERVER-48443
+
+                # We have an open issue with Icecream and ccache to resolve the
+                # cause of these build failures. Once the bug is resolved and
+                # the fix is deployed, we can remove this entire conditional
+                # branch and make it like the one for clang.
+                # TODO: https://github.com/icecc/icecream/issues/550
+                env["ENV"].pop("CCACHE_NOCPP2", None)
+                env["ENV"]["CCACHE_CPP2"] = 1
+                try:
+                    env["CCFLAGS"].remove("-fdirectives-only")
+                except ValueError:
+                    pass
+            else:
+                # If we can, we should make Icecream do its own preprocessing
+                # to reduce concurrency on the local host. We should not do
+                # this when ccache is in use because ccache will execute
+                # Icecream to do its own preprocessing and then execute
+                # Icecream as the compiler on the preprocessed source.
+                env["ENV"]["ICECC_REMOTE_CPP"] = 1
 
     if "ICECC_SCHEDULER" in env:
         env["ENV"]["USE_SCHEDULER"] = env["ICECC_SCHEDULER"]
@@ -318,6 +341,9 @@ def generate(env):
 
 
 def exists(env):
+    # Assume the tool has run if we already know the version.
+    if "ICECREAM_VERSION" in env:
+        return True
 
     icecc = env.get("ICECC", False)
     if not icecc: