SERVER-48203 add precious and link-type install actions to ninja

1 files changed, 143 insertions, 61 deletions

diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py
index 5c1af72db2b..be46bd34646 100644
--- a/site_scons/site_tools/ninja.py
+++ b/site_scons/site_tools/ninja.py
@@ -57,6 +57,24 @@ COMMAND_TYPES = (
     SCons.Action.CommandGeneratorAction,
 )
 
+ninja_compdb_adjust = """\
+import json
+import sys
+
+compdb = {}
+with open(sys.argv[1]) as f:
+    compdb = json.load(f)
+
+for command in compdb:
+    if command['output'].endswith('.compdb'):
+        command['output'] = command['output'][:-(len('.compdb'))]
+    else:
+        print(f"compdb entry does not contain '.compdb': {command['output']}")
+
+with open(sys.argv[1], 'w') as f:
+    json.dump(compdb, f, indent=2)
+"""
+
 
 def _install_action_function(_env, node):
     """Install files using the install or copy commands"""
@@ -65,6 +83,7 @@ def _install_action_function(_env, node):
         "rule": "INSTALL",
         "inputs": [get_path(src_file(s)) for s in node.sources],
         "implicit": get_dependencies(node),
+        "variables": {"precious": node.precious},
     }
 
 
@@ -83,6 +102,7 @@ def _mkdir_action_function(env, node):
                 "mkdir {args}".format(
                     args=' '.join(get_outputs(node)) + " & exit /b 0"
                     if env["PLATFORM"] == "win32" else "-p " + ' '.join(get_outputs(node)), ),
+            "variables": {"precious": node.precious},
         },
     }
 
@@ -102,6 +122,7 @@ def _lib_symlink_action_function(_env, node):
         "inputs": inputs,
         "rule": "SYMLINK",
         "implicit": get_dependencies(node),
+        "variables": {"precious": node.precious},
     }
 
 
@@ -312,6 +333,11 @@ class SConsToNinjaTranslator:
             if callable(node_callback):
                 node_callback(env, node, build)
 
+        if build is not None and node.precious:
+            if not build.get('variables'):
+                build['variables'] = {}
+            build['variables']['precious'] = node.precious
+
         return build
 
     def handle_func_action(self, node, action):
@@ -414,14 +440,6 @@ class SConsToNinjaTranslator:
                 "implicit": dependencies,
             }
 
-        elif results[0]["rule"] == "INSTALL":
-            return {
-                "outputs": all_outputs,
-                "rule": "INSTALL",
-                "inputs": [get_path(src_file(s)) for s in node.sources],
-                "implicit": dependencies,
-            }
-
         raise Exception("Unhandled list action with rule: " + results[0]["rule"])
 
 
@@ -451,8 +469,11 @@ class NinjaState:
         scons_escape = env.get("ESCAPE", lambda x: x)
 
         self.variables = {
+            # The /b option here will make sure that windows updates the mtime
+            # when copying the file. This allows to not need to use restat for windows
+            # copy commands.
             "COPY":
-                "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp",
+                "cmd.exe /c 1>NUL copy /b" if sys.platform == "win32" else "cp",
             "NOOP":
                 "cmd.exe /c 1>NUL echo 0" if sys.platform == "win32" else "echo 0 >/dev/null",
             "SCONS_INVOCATION":
@@ -498,6 +519,18 @@ class NinjaState:
                 "rspfile": "$out.rsp",
                 "rspfile_content": "$rspc",
             },
+            "COMPDB_CC": {
+                "command": "$CC @$out.rsp",
+                "description": "Compiling $out",
+                "rspfile": "$out.rsp",
+                "rspfile_content": "$rspc",
+            },
+            "COMPDB_CXX": {
+                "command": "$CXX @$out.rsp",
+                "description": "Compiling $out",
+                "rspfile": "$out.rsp",
+                "rspfile_content": "$rspc",
+            },
             "LINK": {
                 "command": "$env$LINK @$out.rsp",
                 "description": "Linked $out",
@@ -505,24 +538,12 @@ class NinjaState:
                 "rspfile_content": "$rspc",
                 "pool": "local_pool",
             },
-            # Ninja does not automatically delete the archive before
-            # invoking ar. The ar utility will append to an existing archive, which
-            # can cause duplicate symbols if the symbols moved between object files.
-            # Native SCons will perform this operation so we need to force ninja
-            # to do the same. See related for more info:
-            # https://jira.mongodb.org/browse/SERVER-49457
             "AR": {
-                "command":
-                    "{}$env$AR @$out.rsp".format('' if sys.platform == "win32" else "rm -f $out && "
-                                                 ),
-                "description":
-                    "Archived $out",
-                "rspfile":
-                    "$out.rsp",
-                "rspfile_content":
-                    "$rspc",
-                "pool":
-                    "local_pool",
+                "command": "$env$AR @$out.rsp",
+                "description": "Archived $out",
+                "rspfile": "$out.rsp",
+                "rspfile_content": "$rspc",
+                "pool": "local_pool",
             },
             "SYMLINK": {
                 "command": (
@@ -538,15 +559,6 @@ class NinjaState:
                 "command": "$COPY $in $out",
                 "description": "Installed $out",
                 "pool": "install_pool",
-                # On Windows cmd.exe /c copy does not always correctly
-                # update the timestamp on the output file. This leads
-                # to a stuck constant timestamp in the Ninja database
-                # and needless rebuilds.
-                #
-                # Adding restat here ensures that Ninja always checks
-                # the copy updated the timestamp and that Ninja has
-                # the correct information.
-                "restat": 1,
             },
             "TEMPLATE": {
                 "command": "$SCONS_INVOCATION $out",
@@ -667,10 +679,34 @@ class NinjaState:
         for var, val in self.variables.items():
             ninja.variable(var, val)
 
+        # This is the command that is used to clean a target before building it,
+        # excluding precious targets.
+        if sys.platform == "win32":
+            rm_cmd = f'cmd.exe /c del /q $rm_outs >nul 2>&1 &'
+        else:
+            rm_cmd = 'rm -f $rm_outs;'
+
+        precious_rule_suffix = "_PRECIOUS"
+
+        # Make two sets of rules to honor scons Precious setting. The build nodes themselves
+        # will then reselect their rule according to the precious being set for that node.
+        precious_rules = {}
         for rule, kwargs in self.rules.items():
             if self.env.get('NINJA_MAX_JOBS') is not None and 'pool' not in kwargs:
                 kwargs['pool'] = 'local_pool'
-            ninja.rule(rule, **kwargs)
+            # Do not worry about precious for commands that don't have targets (phony)
+            # or that will callback to scons (which maintains its own precious).
+            if rule not in ['phony', 'TEMPLATE', 'REGENERATE', 'COMPDB_CC', 'COMPDB_CXX']:
+                precious_rule = rule + precious_rule_suffix
+                precious_rules[precious_rule] = kwargs.copy()
+                ninja.rule(precious_rule, **precious_rules[precious_rule])
+
+                kwargs['command'] = f"{rm_cmd} " + kwargs['command']
+                ninja.rule(rule, **kwargs)
+            else:
+
+                ninja.rule(rule, **kwargs)
+        self.rules.update(precious_rules)
 
         # If the user supplied an alias to determine generated sources, use that, otherwise
         # determine what the generated sources are dynamically.
@@ -722,6 +758,45 @@ class NinjaState:
 
         template_builders = []
 
+        # If we ever change the name/s of the rules that include
+        # compile commands (i.e. something like CC) we will need to
+        # update this build to reflect that complete list.
+        compile_commands = "compile_commands.json"
+        compdb_expand = '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
+        adjust_script_out = os.path.join(
+            get_path(self.env['NINJA_BUILDDIR']), 'ninja_compdb_adjust.py')
+        os.makedirs(os.path.dirname(adjust_script_out), exist_ok=True)
+        with open(adjust_script_out, 'w') as f:
+            f.write(ninja_compdb_adjust)
+        self.builds[compile_commands] = {
+            'rule': "CMD",
+            'outputs': [compile_commands],
+            'pool': "console",
+            'implicit': [ninja_file],
+            'variables': {
+                "cmd":
+                    f"ninja -f {ninja_file} -t compdb {compdb_expand}COMPDB_CC COMPDB_CXX > {compile_commands};"
+                    + f"{sys.executable} {adjust_script_out} {compile_commands}"
+            },
+        }
+        self.builds["compiledb"] = {
+            'rule': "phony",
+            "outputs": ["compiledb"],
+            'implicit': [compile_commands],
+        }
+
+        # Now for all build nodes, we want to select the precious rule or not.
+        # If it's not precious, we need to save all the outputs into a variable
+        # on that node. Later we will be removing outputs and switching them to
+        # phonies so that we can generate response and depfiles correctly.
+        for build, kwargs in self.builds.items():
+            if kwargs.get('variables') and kwargs['variables'].get('precious'):
+                kwargs['rule'] = kwargs['rule'] + precious_rule_suffix
+            elif kwargs['rule'] not in ['phony', 'TEMPLATE', 'REGENERATE']:
+                if not kwargs.get('variables'):
+                    kwargs['variables'] = {}
+                kwargs['variables']['rm_outs'] = kwargs['outputs'].copy()
+
         for build in [self.builds[key] for key in sorted(self.builds.keys())]:
             if build["rule"] == "TEMPLATE":
                 template_builders.append(build)
@@ -813,6 +888,20 @@ class NinjaState:
 
             ninja_sorted_build(ninja, **build)
 
+        for build, kwargs in self.builds.items():
+            if kwargs['rule'] in [
+                    'CC', f'CC{precious_rule_suffix}', 'CXX', f'CXX{precious_rule_suffix}'
+            ]:
+                rule = kwargs['rule'].replace(
+                    precious_rule_suffix
+                ) if precious_rule_suffix in kwargs['rule'] else kwargs['rule']
+                rule = "COMPDB_" + rule
+                compdb_build = kwargs.copy()
+
+                compdb_build['rule'] = rule
+                compdb_build['outputs'] = [kwargs['outputs'] + ".compdb"]
+                ninja.build(**compdb_build)
+
         template_builds = {'rule': "TEMPLATE"}
         for template_builder in template_builders:
 
@@ -876,30 +965,6 @@ class NinjaState:
             implicit=[__file__],
         )
 
-        # If we ever change the name/s of the rules that include
-        # compile commands (i.e. something like CC) we will need to
-        # update this build to reflect that complete list.
-        ninja_sorted_build(
-            ninja,
-            outputs="compile_commands.json",
-            rule="CMD",
-            pool="console",
-            implicit=[ninja_file],
-            variables={
-                "cmd":
-                    "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format(
-                        ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else '')
-            },
-            order_only=[generated_sources_alias],
-        )
-
-        ninja_sorted_build(
-            ninja,
-            outputs="compiledb",
-            rule="phony",
-            implicit=["compile_commands.json"],
-        )
-
         # Look in SCons's list of DEFAULT_TARGETS, find the ones that
         # we generated a ninja build rule for.
         scons_default_targets = [
@@ -1297,7 +1362,8 @@ def register_custom_rule_mapping(env, pre_subst_string, rule):
 
 
 def register_custom_rule(env, rule, command, description="", deps=None, pool=None,
-                         use_depfile=False, use_response_file=False, response_file_content="$rspc"):
+                         use_depfile=False, use_response_file=False, response_file_content="$rspc",
+                         restat=False):
     """Allows specification of Ninja rules from inside SCons files."""
     rule_obj = {
         "command": command,
@@ -1315,8 +1381,15 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non
 
     if use_response_file:
         rule_obj["rspfile"] = "$out.rsp"
+        if rule_obj["rspfile"] not in command:
+            raise Exception(
+                f'Bad Ninja Custom Rule: response file requested, but {rule_obj["rspfile"]} not in in command: {command}'
+            )
         rule_obj["rspfile_content"] = response_file_content
 
+    if restat:
+        rule_obj["restat"] = 1
+
     env[NINJA_RULES][rule] = rule_obj
 
 
@@ -1527,6 +1600,15 @@ def generate(env):
     env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider")
     env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback")
 
+    # Expose ninja node path converstion functions to make writing
+    # custom function action handlers easier.
+    env.AddMethod(lambda _env, node: get_outputs(node), "NinjaGetOutputs")
+    env.AddMethod(lambda _env, node, skip_unknown_types=False: get_inputs(node, skip_unknown_types),
+                  "NinjaGetInputs")
+    env.AddMethod(lambda _env, node, skip_sources=False: get_dependencies(node),
+                  "NinjaGetDependencies")
+    env.AddMethod(lambda _env, node: get_order_only(node), "NinjaGetOrderOnly")
+
     # Provides a way for users to handle custom FunctionActions they
     # want to translate to Ninja.
     env[NINJA_CUSTOM_HANDLERS] = {}