Revert "SERVER-48203 setup ninja install actions rule, make evergreen fail correctly on ninja, add precious handling"

This reverts commit 7739da6997795c20924580087a0e09db0a8cf929.

1 files changed, 51 insertions, 103 deletions

diff --git a/site_scons/site_tools/next/ninja.py b/site_scons/site_tools/next/ninja.py
index c76c92a3ce1..dd87ff17d40 100644
--- a/site_scons/site_tools/next/ninja.py
+++ b/site_scons/site_tools/next/ninja.py
@@ -64,7 +64,6 @@ def _install_action_function(_env, node):
         "rule": "INSTALL",
         "inputs": [get_path(src_file(s)) for s in node.sources],
         "implicit": get_dependencies(node),
-        "variables": {"precious": node.precious}
     }
 
 
@@ -82,7 +81,6 @@ def _mkdir_action_function(env, node):
             "cmd": "{mkdir} $out".format(
                 mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p",
             ),
-            "variables": {"precious": node.precious}
         },
     }
 
@@ -102,7 +100,6 @@ def _lib_symlink_action_function(_env, node):
         "inputs": inputs,
         "rule": "SYMLINK",
         "implicit": get_dependencies(node),
-        "variables": {"precious": node.precious}
     }
 
 
@@ -307,11 +304,6 @@ class SConsToNinjaTranslator:
             if callable(node_callback):
                 node_callback(env, node, build)
 
-        if build is not None and node.precious:
-            if not build.get('variables'):
-                build['variables'] = {}
-            build['variables']['precious'] = node.precious
-
         return build
 
     def handle_func_action(self, node, action):
@@ -416,6 +408,14 @@ class SConsToNinjaTranslator:
                 "implicit": dependencies,
             }
 
+        elif results[0]["rule"] == "INSTALL":
+            return {
+                "outputs": all_outputs,
+                "rule": "INSTALL",
+                "inputs": [get_path(src_file(s)) for s in node.sources],
+                "implicit": dependencies,
+            }
+
         raise Exception("Unhandled list action with rule: " + results[0]["rule"])
 
 
@@ -445,10 +445,7 @@ class NinjaState:
         escape = env.get("ESCAPE", lambda x: x)
 
         self.variables = {
-            # The /b option here will make sure that windows updates the mtime
-            # when copying the file. This allows to not need to use restat for windows
-            # copy commands.
-            "COPY": "cmd.exe /c 1>NUL copy /b" if sys.platform == "win32" else "cp",
+            "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp",
             "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format(
                 sys.executable,
                 " ".join(
@@ -495,8 +492,16 @@ class NinjaState:
                 "rspfile_content": "$rspc",
                 "pool": "local_pool",
             },
+            # Ninja does not automatically delete the archive before
+            # invoking ar. The ar utility will append to an existing archive, which
+            # can cause duplicate symbols if the symbols moved between object files.
+            # Native SCons will perform this operation so we need to force ninja
+            # to do the same. See related for more info:
+            # https://jira.mongodb.org/browse/SERVER-49457
             "AR": {
-                "command": "$env$AR @$out.rsp",
+                "command": "{}$env$AR @$out.rsp".format(
+                    '' if sys.platform == "win32" else "rm -f $out && "
+                ),
                 "description": "Archiving $out",
                 "rspfile": "$out.rsp",
                 "rspfile_content": "$rspc",
@@ -514,6 +519,15 @@ class NinjaState:
                 "command": "$COPY $in $out",
                 "description": "Install $out",
                 "pool": "install_pool",
+                # On Windows cmd.exe /c copy does not always correctly
+                # update the timestamp on the output file. This leads
+                # to a stuck constant timestamp in the Ninja database
+                # and needless rebuilds.
+                #
+                # Adding restat here ensures that Ninja always checks
+                # the copy updated the timestamp and that Ninja has
+                # the correct information.
+                "restat": 1,
             },
             "TEMPLATE": {
                 "command": "$SCONS_INVOCATION $out",
@@ -634,39 +648,14 @@ class NinjaState:
         for var, val in self.variables.items():
             ninja.variable(var, val)
 
-        # This is the command that is used to clean a target before building it,
-        # excluding precious targets. The windows command is split into two parts
-        # to handle files and directories separately. The rd command only accepts one
-        # one directory so we have to loop on multiple outputs. This command should not
-        # fail on error because there may be no files to delete.
-        if sys.platform == "win32":
-            rm_cmd = f'cmd.exe /c del /q $rm_outs >nul 2>&1 & cmd.exe /c (for %a in ($rm_outs) do rd /s /q "%~a") >nul 2>&1 &'
-        else:
-            rm_cmd = 'rm -rf $out;'
-
-        # Make two sets of rules to honor scons Precious setting. The build nodes themselves
-        # will then reselect their rule according to the precious being set for that node.
-        precious_rules = {}
         for rule, kwargs in self.rules.items():
-
-            # Do not worry about precious for commands that don't have targets (phony)
-            # or that will callback to scons (which maintains its own precious).
-            if rule not in ['phony', 'TEMPLATE', 'REGENERATE']:
-                precious_rule = rule + "_PRECIOUS"
-                precious_rules[precious_rule] = kwargs.copy()
-                ninja.rule(precious_rule, **precious_rules[precious_rule])
-
-                kwargs['command'] = f"{rm_cmd} " + kwargs['command']
-                ninja.rule(rule, **kwargs)
-            else:
-                ninja.rule(rule, **kwargs)
-        self.rules.update(precious_rules)
+            ninja.rule(rule, **kwargs)
 
         generated_source_files = sorted({
             output
             # First find builds which have header files in their outputs.
             for build in self.builds.values()
-            if self.has_generated_sources(build.get("outputs",[]))
+            if self.has_generated_sources(build["outputs"])
             for output in build["outputs"]
             # Collect only the header files from the builds with them
             # in their output. We do this because is_generated_source
@@ -685,38 +674,6 @@ class NinjaState:
 
         template_builders = []
 
-        # If we ever change the name/s of the rules that include
-        # compile commands (i.e. something like CC) we will need to
-        # update this build to reflect that complete list.
-        self.builds["compile_commands.json"] = {
-            'rule' : "CMD",
-            'outputs': ["compile_commands.json"],
-            'pool' : "console",
-            'implicit' : [ninja_file],
-            'variables' : {
-                "cmd" : "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format(
-                    ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
-                )
-            },
-        }
-        self.builds["compiledb"] = {
-            'rule' : "phony",
-            "outputs" : ["compiledb"],
-            'implicit' : ["compile_commands.json"],
-        }
-
-        # Now for all build nodes, we want to select the precious rule or not.
-        # If it's not precious, we need to save all the outputs into a variable
-        # on that node. Later we will be removing outputs and switching them to
-        # phonies so that we can generate response and depfiles correctly.
-        for build, kwargs in self.builds.items():
-            if kwargs.get('variables') and kwargs['variables'].get('precious'):
-                kwargs['rule'] = kwargs['rule'] + '_PRECIOUS'
-            elif kwargs['rule'] not in ['phony', 'TEMPLATE', 'REGENERATE']:
-                if not kwargs.get('variables'):
-                    kwargs['variables'] = {}
-                kwargs['variables']['rm_outs'] = kwargs['outputs'].copy()
-
         for build in [self.builds[key] for key in sorted(self.builds.keys())]:
             if build["rule"] == "TEMPLATE":
                 template_builders.append(build)
@@ -733,7 +690,7 @@ class NinjaState:
             if (
                 generated_source_files
                 and not build["rule"] == "INSTALL"
-                and set(build.get("outputs", [])).isdisjoint(generated_source_files)
+                and set(build["outputs"]).isdisjoint(generated_source_files)
                 and set(build.get("implicit", [])).isdisjoint(generated_source_files)
             ):
 
@@ -859,6 +816,24 @@ class NinjaState:
             implicit=[__file__],
         )
 
+        # If we ever change the name/s of the rules that include
+        # compile commands (i.e. something like CC) we will need to
+        # update this build to reflect that complete list.
+        ninja.build(
+            "compile_commands.json",
+            rule="CMD",
+            pool="console",
+            implicit=[ninja_file],
+            variables={
+                "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format(
+                    ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
+                )
+            },
+        )
+
+        ninja.build(
+            "compiledb", rule="phony", implicit=["compile_commands.json"],
+        )
 
         # Look in SCons's list of DEFAULT_TARGETS, find the ones that
         # we generated a ninja build rule for.
@@ -1023,10 +998,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom
         cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:]
         rsp_content = " ".join(rsp_content)
 
-        variables = {
-            "rspc": rsp_content
-        }
-
+        variables = {"rspc": rsp_content}
         variables[rule] = cmd
         if use_command_env:
             variables["env"] = get_command_env(env)
@@ -1071,7 +1043,6 @@ def get_generic_shell_command(env, node, action, targets, sources, executor=None
         {
             "cmd": generate_command(env, node, action, targets, sources, executor=None),
             "env": get_command_env(env),
-            "precious": node.precious
         },
         # Since this function is a rule mapping provider, it must return a list of dependencies,
         # and usually this would be the path to a tool, such as a compiler, used for this rule.
@@ -1118,7 +1089,7 @@ def get_command(env, node, action):  # pylint: disable=too-many-branches
 
     provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command)
     rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor)
-    variables['precious'] = node.precious
+
     # Get the dependencies for all targets
     implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)})
 
@@ -1225,18 +1196,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule):
     __NINJA_RULE_MAPPING[pre_subst_string] = rule
 
 
-def register_custom_rule(
-    env,
-    rule,
-    command,
-    description="",
-    deps=None,
-    pool=None,
-    use_depfile=False,
-    use_response_file=False,
-    response_file_content="$rspc",
-    restat=False):
-
+def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"):
     """Allows specification of Ninja rules from inside SCons files."""
     rule_obj = {
         "command": command,
@@ -1254,13 +1214,8 @@ def register_custom_rule(
 
     if use_response_file:
         rule_obj["rspfile"] = "$out.rsp"
-        if rule_obj["rspfile"] not in command:
-            raise Exception(f'Bad Ninja Custom Rule: response file requested, but {rule_obj["rspfile"]} not in in command: {command}')
         rule_obj["rspfile_content"] = response_file_content
 
-    if restat:
-        rule_obj["restat"] = 1
-
     env[NINJA_RULES][rule] = rule_obj
 
 
@@ -1456,13 +1411,6 @@ def generate(env):
     env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider")
     env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback")
 
-    # Expose ninja node path converstion functions to make writing
-    # custom function action handlers easier.
-    env.AddMethod(lambda env, node: get_outputs(node), "NinjaGetOutputs")
-    env.AddMethod(lambda env, node, skip_unknown_types=False: get_inputs(node, skip_unknown_types), "NinjaGetInputs")
-    env.AddMethod(lambda env, node, skip_sources=False: get_dependencies(node), "NinjaGetDependencies")
-    env.AddMethod(lambda env, node: get_order_only(node), "NinjaGetOrderOnly")
-
     # Provides a way for users to handle custom FunctionActions they
     # want to translate to Ninja.
     env[NINJA_CUSTOM_HANDLERS] = {}