SERVER-48203 setup ninja install actions rule, make evergreen fail correctly on ninja, add precious handling

1 files changed, 103 insertions, 51 deletions

diff --git a/site_scons/site_tools/next/ninja.py b/site_scons/site_tools/next/ninja.py
index dd87ff17d40..c76c92a3ce1 100644
--- a/site_scons/site_tools/next/ninja.py
+++ b/site_scons/site_tools/next/ninja.py
@@ -64,6 +64,7 @@ def _install_action_function(_env, node):
         "rule": "INSTALL",
         "inputs": [get_path(src_file(s)) for s in node.sources],
         "implicit": get_dependencies(node),
+        "variables": {"precious": node.precious}
     }
 
 
@@ -81,6 +82,7 @@ def _mkdir_action_function(env, node):
             "cmd": "{mkdir} $out".format(
                 mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p",
             ),
+            "variables": {"precious": node.precious}
         },
     }
 
@@ -100,6 +102,7 @@ def _lib_symlink_action_function(_env, node):
         "inputs": inputs,
         "rule": "SYMLINK",
         "implicit": get_dependencies(node),
+        "variables": {"precious": node.precious}
     }
 
 
@@ -304,6 +307,11 @@ class SConsToNinjaTranslator:
             if callable(node_callback):
                 node_callback(env, node, build)
 
+        if build is not None and node.precious:
+            if not build.get('variables'):
+                build['variables'] = {}
+            build['variables']['precious'] = node.precious
+
         return build
 
     def handle_func_action(self, node, action):
@@ -408,14 +416,6 @@ class SConsToNinjaTranslator:
                 "implicit": dependencies,
             }
 
-        elif results[0]["rule"] == "INSTALL":
-            return {
-                "outputs": all_outputs,
-                "rule": "INSTALL",
-                "inputs": [get_path(src_file(s)) for s in node.sources],
-                "implicit": dependencies,
-            }
-
         raise Exception("Unhandled list action with rule: " + results[0]["rule"])
 
 
@@ -445,7 +445,10 @@ class NinjaState:
         escape = env.get("ESCAPE", lambda x: x)
 
         self.variables = {
-            "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp",
+            # The /b option here will make sure that windows updates the mtime
+            # when copying the file. This allows to not need to use restat for windows
+            # copy commands.
+            "COPY": "cmd.exe /c 1>NUL copy /b" if sys.platform == "win32" else "cp",
             "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format(
                 sys.executable,
                 " ".join(
@@ -492,16 +495,8 @@ class NinjaState:
                 "rspfile_content": "$rspc",
                 "pool": "local_pool",
             },
-            # Ninja does not automatically delete the archive before
-            # invoking ar. The ar utility will append to an existing archive, which
-            # can cause duplicate symbols if the symbols moved between object files.
-            # Native SCons will perform this operation so we need to force ninja
-            # to do the same. See related for more info:
-            # https://jira.mongodb.org/browse/SERVER-49457
             "AR": {
-                "command": "{}$env$AR @$out.rsp".format(
-                    '' if sys.platform == "win32" else "rm -f $out && "
-                ),
+                "command": "$env$AR @$out.rsp",
                 "description": "Archiving $out",
                 "rspfile": "$out.rsp",
                 "rspfile_content": "$rspc",
@@ -519,15 +514,6 @@ class NinjaState:
                 "command": "$COPY $in $out",
                 "description": "Install $out",
                 "pool": "install_pool",
-                # On Windows cmd.exe /c copy does not always correctly
-                # update the timestamp on the output file. This leads
-                # to a stuck constant timestamp in the Ninja database
-                # and needless rebuilds.
-                #
-                # Adding restat here ensures that Ninja always checks
-                # the copy updated the timestamp and that Ninja has
-                # the correct information.
-                "restat": 1,
             },
             "TEMPLATE": {
                 "command": "$SCONS_INVOCATION $out",
@@ -648,14 +634,39 @@ class NinjaState:
         for var, val in self.variables.items():
             ninja.variable(var, val)
 
+        # This is the command that is used to clean a target before building it,
+        # excluding precious targets. The windows command is split into two parts
+        # to handle files and directories separately. The rd command only accepts one
+        # one directory so we have to loop on multiple outputs. This command should not
+        # fail on error because there may be no files to delete.
+        if sys.platform == "win32":
+            rm_cmd = f'cmd.exe /c del /q $rm_outs >nul 2>&1 & cmd.exe /c (for %a in ($rm_outs) do rd /s /q "%~a") >nul 2>&1 &'
+        else:
+            rm_cmd = 'rm -rf $out;'
+
+        # Make two sets of rules to honor scons Precious setting. The build nodes themselves
+        # will then reselect their rule according to the precious being set for that node.
+        precious_rules = {}
         for rule, kwargs in self.rules.items():
-            ninja.rule(rule, **kwargs)
+
+            # Do not worry about precious for commands that don't have targets (phony)
+            # or that will callback to scons (which maintains its own precious).
+            if rule not in ['phony', 'TEMPLATE', 'REGENERATE']:
+                precious_rule = rule + "_PRECIOUS"
+                precious_rules[precious_rule] = kwargs.copy()
+                ninja.rule(precious_rule, **precious_rules[precious_rule])
+
+                kwargs['command'] = f"{rm_cmd} " + kwargs['command']
+                ninja.rule(rule, **kwargs)
+            else:
+                ninja.rule(rule, **kwargs)
+        self.rules.update(precious_rules)
 
         generated_source_files = sorted({
             output
             # First find builds which have header files in their outputs.
             for build in self.builds.values()
-            if self.has_generated_sources(build["outputs"])
+            if self.has_generated_sources(build.get("outputs",[]))
             for output in build["outputs"]
             # Collect only the header files from the builds with them
             # in their output. We do this because is_generated_source
@@ -674,6 +685,38 @@ class NinjaState:
 
         template_builders = []
 
+        # If we ever change the name/s of the rules that include
+        # compile commands (i.e. something like CC) we will need to
+        # update this build to reflect that complete list.
+        self.builds["compile_commands.json"] = {
+            'rule' : "CMD",
+            'outputs': ["compile_commands.json"],
+            'pool' : "console",
+            'implicit' : [ninja_file],
+            'variables' : {
+                "cmd" : "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format(
+                    ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
+                )
+            },
+        }
+        self.builds["compiledb"] = {
+            'rule' : "phony",
+            "outputs" : ["compiledb"],
+            'implicit' : ["compile_commands.json"],
+        }
+
+        # Now for all build nodes, we want to select the precious rule or not.
+        # If it's not precious, we need to save all the outputs into a variable
+        # on that node. Later we will be removing outputs and switching them to
+        # phonies so that we can generate response and depfiles correctly.
+        for build, kwargs in self.builds.items():
+            if kwargs.get('variables') and kwargs['variables'].get('precious'):
+                kwargs['rule'] = kwargs['rule'] + '_PRECIOUS'
+            elif kwargs['rule'] not in ['phony', 'TEMPLATE', 'REGENERATE']:
+                if not kwargs.get('variables'):
+                    kwargs['variables'] = {}
+                kwargs['variables']['rm_outs'] = kwargs['outputs'].copy()
+
         for build in [self.builds[key] for key in sorted(self.builds.keys())]:
             if build["rule"] == "TEMPLATE":
                 template_builders.append(build)
@@ -690,7 +733,7 @@ class NinjaState:
             if (
                 generated_source_files
                 and not build["rule"] == "INSTALL"
-                and set(build["outputs"]).isdisjoint(generated_source_files)
+                and set(build.get("outputs", [])).isdisjoint(generated_source_files)
                 and set(build.get("implicit", [])).isdisjoint(generated_source_files)
             ):
 
@@ -816,24 +859,6 @@ class NinjaState:
             implicit=[__file__],
         )
 
-        # If we ever change the name/s of the rules that include
-        # compile commands (i.e. something like CC) we will need to
-        # update this build to reflect that complete list.
-        ninja.build(
-            "compile_commands.json",
-            rule="CMD",
-            pool="console",
-            implicit=[ninja_file],
-            variables={
-                "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format(
-                    ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
-                )
-            },
-        )
-
-        ninja.build(
-            "compiledb", rule="phony", implicit=["compile_commands.json"],
-        )
 
         # Look in SCons's list of DEFAULT_TARGETS, find the ones that
         # we generated a ninja build rule for.
@@ -998,7 +1023,10 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom
         cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:]
         rsp_content = " ".join(rsp_content)
 
-        variables = {"rspc": rsp_content}
+        variables = {
+            "rspc": rsp_content
+        }
+
         variables[rule] = cmd
         if use_command_env:
             variables["env"] = get_command_env(env)
@@ -1043,6 +1071,7 @@ def get_generic_shell_command(env, node, action, targets, sources, executor=None
         {
             "cmd": generate_command(env, node, action, targets, sources, executor=None),
             "env": get_command_env(env),
+            "precious": node.precious
         },
         # Since this function is a rule mapping provider, it must return a list of dependencies,
         # and usually this would be the path to a tool, such as a compiler, used for this rule.
@@ -1089,7 +1118,7 @@ def get_command(env, node, action):  # pylint: disable=too-many-branches
 
     provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command)
     rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor)
-
+    variables['precious'] = node.precious
     # Get the dependencies for all targets
     implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)})
 
@@ -1196,7 +1225,18 @@ def register_custom_rule_mapping(env, pre_subst_string, rule):
     __NINJA_RULE_MAPPING[pre_subst_string] = rule
 
 
-def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"):
+def register_custom_rule(
+    env,
+    rule,
+    command,
+    description="",
+    deps=None,
+    pool=None,
+    use_depfile=False,
+    use_response_file=False,
+    response_file_content="$rspc",
+    restat=False):
+
     """Allows specification of Ninja rules from inside SCons files."""
     rule_obj = {
         "command": command,
@@ -1214,8 +1254,13 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non
 
     if use_response_file:
         rule_obj["rspfile"] = "$out.rsp"
+        if rule_obj["rspfile"] not in command:
+            raise Exception(f'Bad Ninja Custom Rule: response file requested, but {rule_obj["rspfile"]} not in in command: {command}')
         rule_obj["rspfile_content"] = response_file_content
 
+    if restat:
+        rule_obj["restat"] = 1
+
     env[NINJA_RULES][rule] = rule_obj
 
 
@@ -1411,6 +1456,13 @@ def generate(env):
     env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider")
     env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback")
 
+    # Expose ninja node path converstion functions to make writing
+    # custom function action handlers easier.
+    env.AddMethod(lambda env, node: get_outputs(node), "NinjaGetOutputs")
+    env.AddMethod(lambda env, node, skip_unknown_types=False: get_inputs(node, skip_unknown_types), "NinjaGetInputs")
+    env.AddMethod(lambda env, node, skip_sources=False: get_dependencies(node), "NinjaGetDependencies")
+    env.AddMethod(lambda env, node: get_order_only(node), "NinjaGetOrderOnly")
+
     # Provides a way for users to handle custom FunctionActions they
     # want to translate to Ninja.
     env[NINJA_CUSTOM_HANDLERS] = {}