SERVER-48691 added workaround for ninja response files line to long

(cherry picked from commit f4902aa0d320cfe1cc1d15d4e4623e4d24e32f39)

2 files changed, 92 insertions, 3 deletions

diff --git a/SConstruct b/SConstruct
index 0d6690f7226..9e94557ecf9 100644
--- a/SConstruct
+++ b/SConstruct
@@ -4040,6 +4040,80 @@ if get_option('ninja') != 'disabled':
 
     env['NINJA_REGENERATE_DEPS'] = ninja_generate_deps
 
+    if get_option('build-tools') == 'next' and env.TargetOSIs("windows"):
+        # This is a workaround on windows for SERVER-48691 where the line length
+        # in response files is too long:
+        # https://developercommunity.visualstudio.com/content/problem/441978/fatal-error-lnk1170-line-in-command-file-contains.html
+        #
+        # Ninja currently does not support
+        # storing a newline in the ninja file, and therefore you can not
+        # easily generate it to the response files. The only documented
+        # way to get newlines into the response file is to use the $in_newline
+        # variable in the rule.
+        #
+        # This workaround will move most of the object or lib links into the
+        # inputs and then make the respone file consist of the inputs plus
+        # whatever options are left in the original response content
+        # more info can be found here:
+        # https://github.com/ninja-build/ninja/pull/1223/files/e71bcceefb942f8355aab83ab447d702354ba272#r179526824
+        # https://github.com/ninja-build/ninja/issues/1000
+
+        # we are making a new special rule which will leverage
+        # the $in_newline to get newlines into our response file
+        env.NinjaRule(
+            "WINLINK",
+            "$env$WINLINK @$out.rsp",
+            description="Linking $out",
+            deps=None,
+            pool="local_pool",
+            use_depfile=False,
+            use_response_file=True,
+            response_file_content="$in_newline $rspc")
+
+        # Setup the response file content generation to use our workaround rule
+        # for LINK commands.
+        provider = env.NinjaGenResponseFileProvider(
+            "WINLINK",
+            "$LINK",
+        )
+        env.NinjaRuleMapping("${LINKCOM}", provider)
+        env.NinjaRuleMapping(env["LINKCOM"], provider)
+
+        # The workaround function will move some of the content from the rspc
+        # variable into the nodes inputs. We only want to move build nodes because
+        # inputs must be files, so we make sure the the option in the rspc
+        # file starts with the build directory.
+        def winlink_workaround(env, node, ninja_build):
+            if ninja_build and 'rspc' in ninja_build["variables"]:
+
+                rsp_content = []
+                ninja_build["inputs"] = []
+                for opt in ninja_build["variables"]["rspc"].split():
+
+                    # if its a candidate to go in the inputs add it, else keep it in the non-newline
+                    # rsp_content list
+                    if opt.startswith(str(env.Dir("$BUILD_DIR"))) and opt != str(node):
+                        ninja_build["inputs"].append(opt)
+                    else:
+                        rsp_content.append(opt)
+
+                ninja_build["variables"]["rspc"] = ' '.join(rsp_content)
+                ninja_build["inputs"] += [infile for infile in ninja_build["inputs"] if infile not in ninja_build["inputs"]]
+
+        # We apply the workaround to all Program nodes as they have potential
+        # response files that have lines that are too long.
+        # This will setup a callback function for a node
+        # so that when its been processed, we can make some final adjustments before
+        # its generated to the ninja file.
+        def winlink_workaround_emitter(target, source, env):
+            env.NinjaSetBuildNodeCallback(target[0], winlink_workaround)
+            return target, source
+
+        builder = env['BUILDERS']["Program"]
+        base_emitter = builder.emitter
+        new_emitter = SCons.Builder.ListEmitter([base_emitter, winlink_workaround_emitter])
+        builder.emitter = new_emitter
+
     # idlc.py has the ability to print it's implicit dependencies
     # while generating, Ninja can consume these prints using the
     # deps=msvc method.
diff --git a/site_scons/site_tools/next/ninja.py b/site_scons/site_tools/next/ninja.py
index dfd6d835980..acb85d42da4 100644
--- a/site_scons/site_tools/next/ninja.py
+++ b/site_scons/site_tools/next/ninja.py
@@ -266,6 +266,7 @@ class SConsToNinjaTranslator:
             return None
 
         build = {}
+        env = node.env if node.env else self.env
 
         # Ideally this should never happen, and we do try to filter
         # Ninja builders out of being sources of ninja builders but I
@@ -277,18 +278,23 @@ class SConsToNinjaTranslator:
             build = self.handle_func_action(node, action)
         elif isinstance(action, SCons.Action.LazyAction):
             # pylint: disable=protected-access
-            action = action._generate_cache(node.env if node.env else self.env)
+            action = action._generate_cache(env)
             build = self.action_to_ninja_build(node, action=action)
         elif isinstance(action, SCons.Action.ListAction):
             build = self.handle_list_action(node, action)
         elif isinstance(action, COMMAND_TYPES):
-            build = get_command(node.env if node.env else self.env, node, action)
+            build = get_command(env, node, action)
         else:
             raise Exception("Got an unbuildable ListAction for: {}".format(str(node)))
 
         if build is not None:
             build["order_only"] = get_order_only(node)
 
+        if 'conftest' not in str(node):
+            node_callback = getattr(node.attributes, "ninja_build_callback", None)
+            if callable(node_callback):
+                node_callback(env, node, build)
+
         return build
 
     def handle_func_action(self, node, action):
@@ -1176,7 +1182,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule):
     __NINJA_RULE_MAPPING[pre_subst_string] = rule
 
 
-def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False):
+def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"):
     """Allows specification of Ninja rules from inside SCons files."""
     rule_obj = {
         "command": command,
@@ -1192,6 +1198,10 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non
     if pool is not None:
         rule_obj["pool"] = pool
 
+    if use_response_file:
+        rule_obj["rspfile"] = "$out.rsp"
+        rule_obj["rspfile_content"] = response_file_content
+
     env[NINJA_RULES][rule] = rule_obj
 
 
@@ -1199,6 +1209,9 @@ def register_custom_pool(env, pool, size):
     """Allows the creation of custom Ninja pools"""
     env[NINJA_POOLS][pool] = size
 
+def set_build_node_callback(env, node, callback):
+    if 'conftest' not in str(node):
+        setattr(node.attributes, "ninja_build_callback", callback)
 
 def ninja_csig(original):
     """Return a dummy csig"""
@@ -1380,7 +1393,9 @@ def generate(env):
     # Provide a way for custom rule authors to easily access command
     # generation.
     env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand")
+    env.AddMethod(get_command, "NinjaGetCommand")
     env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider")
+    env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback")
 
     # Provides a way for users to handle custom FunctionActions they
     # want to translate to Ninja.