From 0054d395a55c60470f5747647a8a7988530d1a5f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 24 Aug 2022 17:05:26 +0000 Subject: SERVER-48203 add precious and link-type install actions to ninja --- site_scons/site_tools/ninja.py | 204 +++++++++++++++++++++++++++++------------ 1 file changed, 143 insertions(+), 61 deletions(-) (limited to 'site_scons') diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py index 5c1af72db2b..be46bd34646 100644 --- a/site_scons/site_tools/ninja.py +++ b/site_scons/site_tools/ninja.py @@ -57,6 +57,24 @@ COMMAND_TYPES = ( SCons.Action.CommandGeneratorAction, ) +ninja_compdb_adjust = """\ +import json +import sys + +compdb = {} +with open(sys.argv[1]) as f: + compdb = json.load(f) + +for command in compdb: + if command['output'].endswith('.compdb'): + command['output'] = command['output'][:-(len('.compdb'))] + else: + print(f"compdb entry does not contain '.compdb': {command['output']}") + +with open(sys.argv[1], 'w') as f: + json.dump(compdb, f, indent=2) +""" + def _install_action_function(_env, node): """Install files using the install or copy commands""" @@ -65,6 +83,7 @@ def _install_action_function(_env, node): "rule": "INSTALL", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": get_dependencies(node), + "variables": {"precious": node.precious}, } @@ -83,6 +102,7 @@ def _mkdir_action_function(env, node): "mkdir {args}".format( args=' '.join(get_outputs(node)) + " & exit /b 0" if env["PLATFORM"] == "win32" else "-p " + ' '.join(get_outputs(node)), ), + "variables": {"precious": node.precious}, }, } @@ -102,6 +122,7 @@ def _lib_symlink_action_function(_env, node): "inputs": inputs, "rule": "SYMLINK", "implicit": get_dependencies(node), + "variables": {"precious": node.precious}, } @@ -312,6 +333,11 @@ class SConsToNinjaTranslator: if callable(node_callback): node_callback(env, node, build) + if build is not None and node.precious: + if not build.get('variables'): + build['variables'] = {} + build['variables']['precious'] = node.precious + return build def handle_func_action(self, node, action): @@ -414,14 +440,6 @@ class SConsToNinjaTranslator: "implicit": dependencies, } - elif results[0]["rule"] == "INSTALL": - return { - "outputs": all_outputs, - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], - "implicit": dependencies, - } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) @@ -451,8 +469,11 @@ class NinjaState: scons_escape = env.get("ESCAPE", lambda x: x) self.variables = { + # The /b option here will make sure that windows updates the mtime + # when copying the file. This allows to not need to use restat for windows + # copy commands. "COPY": - "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", + "cmd.exe /c 1>NUL copy /b" if sys.platform == "win32" else "cp", "NOOP": "cmd.exe /c 1>NUL echo 0" if sys.platform == "win32" else "echo 0 >/dev/null", "SCONS_INVOCATION": @@ -498,6 +519,18 @@ class NinjaState: "rspfile": "$out.rsp", "rspfile_content": "$rspc", }, + "COMPDB_CC": { + "command": "$CC @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "COMPDB_CXX": { + "command": "$CXX @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, "LINK": { "command": "$env$LINK @$out.rsp", "description": "Linked $out", @@ -505,24 +538,12 @@ class NinjaState: "rspfile_content": "$rspc", "pool": "local_pool", }, - # Ninja does not automatically delete the archive before - # invoking ar. The ar utility will append to an existing archive, which - # can cause duplicate symbols if the symbols moved between object files. - # Native SCons will perform this operation so we need to force ninja - # to do the same. See related for more info: - # https://jira.mongodb.org/browse/SERVER-49457 "AR": { - "command": - "{}$env$AR @$out.rsp".format('' if sys.platform == "win32" else "rm -f $out && " - ), - "description": - "Archived $out", - "rspfile": - "$out.rsp", - "rspfile_content": - "$rspc", - "pool": - "local_pool", + "command": "$env$AR @$out.rsp", + "description": "Archived $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", }, "SYMLINK": { "command": ( @@ -538,15 +559,6 @@ class NinjaState: "command": "$COPY $in $out", "description": "Installed $out", "pool": "install_pool", - # On Windows cmd.exe /c copy does not always correctly - # update the timestamp on the output file. This leads - # to a stuck constant timestamp in the Ninja database - # and needless rebuilds. - # - # Adding restat here ensures that Ninja always checks - # the copy updated the timestamp and that Ninja has - # the correct information. - "restat": 1, }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", @@ -667,10 +679,34 @@ class NinjaState: for var, val in self.variables.items(): ninja.variable(var, val) + # This is the command that is used to clean a target before building it, + # excluding precious targets. + if sys.platform == "win32": + rm_cmd = f'cmd.exe /c del /q $rm_outs >nul 2>&1 &' + else: + rm_cmd = 'rm -f $rm_outs;' + + precious_rule_suffix = "_PRECIOUS" + + # Make two sets of rules to honor scons Precious setting. The build nodes themselves + # will then reselect their rule according to the precious being set for that node. + precious_rules = {} for rule, kwargs in self.rules.items(): if self.env.get('NINJA_MAX_JOBS') is not None and 'pool' not in kwargs: kwargs['pool'] = 'local_pool' - ninja.rule(rule, **kwargs) + # Do not worry about precious for commands that don't have targets (phony) + # or that will callback to scons (which maintains its own precious). + if rule not in ['phony', 'TEMPLATE', 'REGENERATE', 'COMPDB_CC', 'COMPDB_CXX']: + precious_rule = rule + precious_rule_suffix + precious_rules[precious_rule] = kwargs.copy() + ninja.rule(precious_rule, **precious_rules[precious_rule]) + + kwargs['command'] = f"{rm_cmd} " + kwargs['command'] + ninja.rule(rule, **kwargs) + else: + + ninja.rule(rule, **kwargs) + self.rules.update(precious_rules) # If the user supplied an alias to determine generated sources, use that, otherwise # determine what the generated sources are dynamically. @@ -722,6 +758,45 @@ class NinjaState: template_builders = [] + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + compile_commands = "compile_commands.json" + compdb_expand = '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else '' + adjust_script_out = os.path.join( + get_path(self.env['NINJA_BUILDDIR']), 'ninja_compdb_adjust.py') + os.makedirs(os.path.dirname(adjust_script_out), exist_ok=True) + with open(adjust_script_out, 'w') as f: + f.write(ninja_compdb_adjust) + self.builds[compile_commands] = { + 'rule': "CMD", + 'outputs': [compile_commands], + 'pool': "console", + 'implicit': [ninja_file], + 'variables': { + "cmd": + f"ninja -f {ninja_file} -t compdb {compdb_expand}COMPDB_CC COMPDB_CXX > {compile_commands};" + + f"{sys.executable} {adjust_script_out} {compile_commands}" + }, + } + self.builds["compiledb"] = { + 'rule': "phony", + "outputs": ["compiledb"], + 'implicit': [compile_commands], + } + + # Now for all build nodes, we want to select the precious rule or not. + # If it's not precious, we need to save all the outputs into a variable + # on that node. Later we will be removing outputs and switching them to + # phonies so that we can generate response and depfiles correctly. + for build, kwargs in self.builds.items(): + if kwargs.get('variables') and kwargs['variables'].get('precious'): + kwargs['rule'] = kwargs['rule'] + precious_rule_suffix + elif kwargs['rule'] not in ['phony', 'TEMPLATE', 'REGENERATE']: + if not kwargs.get('variables'): + kwargs['variables'] = {} + kwargs['variables']['rm_outs'] = kwargs['outputs'].copy() + for build in [self.builds[key] for key in sorted(self.builds.keys())]: if build["rule"] == "TEMPLATE": template_builders.append(build) @@ -813,6 +888,20 @@ class NinjaState: ninja_sorted_build(ninja, **build) + for build, kwargs in self.builds.items(): + if kwargs['rule'] in [ + 'CC', f'CC{precious_rule_suffix}', 'CXX', f'CXX{precious_rule_suffix}' + ]: + rule = kwargs['rule'].replace( + precious_rule_suffix + ) if precious_rule_suffix in kwargs['rule'] else kwargs['rule'] + rule = "COMPDB_" + rule + compdb_build = kwargs.copy() + + compdb_build['rule'] = rule + compdb_build['outputs'] = [kwargs['outputs'] + ".compdb"] + ninja.build(**compdb_build) + template_builds = {'rule': "TEMPLATE"} for template_builder in template_builders: @@ -876,30 +965,6 @@ class NinjaState: implicit=[__file__], ) - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja_sorted_build( - ninja, - outputs="compile_commands.json", - rule="CMD", - pool="console", - implicit=[ninja_file], - variables={ - "cmd": - "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( - ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else '') - }, - order_only=[generated_sources_alias], - ) - - ninja_sorted_build( - ninja, - outputs="compiledb", - rule="phony", - implicit=["compile_commands.json"], - ) - # Look in SCons's list of DEFAULT_TARGETS, find the ones that # we generated a ninja build rule for. scons_default_targets = [ @@ -1297,7 +1362,8 @@ def register_custom_rule_mapping(env, pre_subst_string, rule): def register_custom_rule(env, rule, command, description="", deps=None, pool=None, - use_depfile=False, use_response_file=False, response_file_content="$rspc"): + use_depfile=False, use_response_file=False, response_file_content="$rspc", + restat=False): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, @@ -1315,8 +1381,15 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non if use_response_file: rule_obj["rspfile"] = "$out.rsp" + if rule_obj["rspfile"] not in command: + raise Exception( + f'Bad Ninja Custom Rule: response file requested, but {rule_obj["rspfile"]} not in in command: {command}' + ) rule_obj["rspfile_content"] = response_file_content + if restat: + rule_obj["restat"] = 1 + env[NINJA_RULES][rule] = rule_obj @@ -1527,6 +1600,15 @@ def generate(env): env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") + # Expose ninja node path converstion functions to make writing + # custom function action handlers easier. + env.AddMethod(lambda _env, node: get_outputs(node), "NinjaGetOutputs") + env.AddMethod(lambda _env, node, skip_unknown_types=False: get_inputs(node, skip_unknown_types), + "NinjaGetInputs") + env.AddMethod(lambda _env, node, skip_sources=False: get_dependencies(node), + "NinjaGetDependencies") + env.AddMethod(lambda _env, node: get_order_only(node), "NinjaGetOrderOnly") + # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. env[NINJA_CUSTOM_HANDLERS] = {} -- cgit v1.2.1