diff options
author | Andrew Morrow <acm@mongodb.com> | 2021-03-07 22:19:42 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-03-08 03:55:47 +0000 |
commit | f30dbae001e83286c6aa18d566d1628d335b9d74 (patch) | |
tree | 74880ceb820e17ace0b8ae3d5a090bd68f6cddb2 /site_scons | |
parent | bea36f3f9c5d33b47545deb3761a3dcc9ae0c2cc (diff) | |
download | mongo-f30dbae001e83286c6aa18d566d1628d335b9d74.tar.gz |
Revert "SERVER-48203 setup ninja install actions rule, make evergreen fail correctly on ninja, add precious handling"
This reverts commit 7739da6997795c20924580087a0e09db0a8cf929.
Diffstat (limited to 'site_scons')
-rw-r--r-- | site_scons/site_tools/next/ninja.py | 154 |
1 files changed, 51 insertions, 103 deletions
diff --git a/site_scons/site_tools/next/ninja.py b/site_scons/site_tools/next/ninja.py index c76c92a3ce1..dd87ff17d40 100644 --- a/site_scons/site_tools/next/ninja.py +++ b/site_scons/site_tools/next/ninja.py @@ -64,7 +64,6 @@ def _install_action_function(_env, node): "rule": "INSTALL", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": get_dependencies(node), - "variables": {"precious": node.precious} } @@ -82,7 +81,6 @@ def _mkdir_action_function(env, node): "cmd": "{mkdir} $out".format( mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", ), - "variables": {"precious": node.precious} }, } @@ -102,7 +100,6 @@ def _lib_symlink_action_function(_env, node): "inputs": inputs, "rule": "SYMLINK", "implicit": get_dependencies(node), - "variables": {"precious": node.precious} } @@ -307,11 +304,6 @@ class SConsToNinjaTranslator: if callable(node_callback): node_callback(env, node, build) - if build is not None and node.precious: - if not build.get('variables'): - build['variables'] = {} - build['variables']['precious'] = node.precious - return build def handle_func_action(self, node, action): @@ -416,6 +408,14 @@ class SConsToNinjaTranslator: "implicit": dependencies, } + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": "INSTALL", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": dependencies, + } + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) @@ -445,10 +445,7 @@ class NinjaState: escape = env.get("ESCAPE", lambda x: x) self.variables = { - # The /b option here will make sure that windows updates the mtime - # when copying the file. This allows to not need to use restat for windows - # copy commands. - "COPY": "cmd.exe /c 1>NUL copy /b" if sys.platform == "win32" else "cp", + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( sys.executable, " ".join( @@ -495,8 +492,16 @@ class NinjaState: "rspfile_content": "$rspc", "pool": "local_pool", }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 "AR": { - "command": "$env$AR @$out.rsp", + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), "description": "Archiving $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", @@ -514,6 +519,15 @@ class NinjaState: "command": "$COPY $in $out", "description": "Install $out", "pool": "install_pool", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", @@ -634,39 +648,14 @@ class NinjaState: for var, val in self.variables.items(): ninja.variable(var, val) - # This is the command that is used to clean a target before building it, - # excluding precious targets. The windows command is split into two parts - # to handle files and directories separately. The rd command only accepts one - # one directory so we have to loop on multiple outputs. This command should not - # fail on error because there may be no files to delete. - if sys.platform == "win32": - rm_cmd = f'cmd.exe /c del /q $rm_outs >nul 2>&1 & cmd.exe /c (for %a in ($rm_outs) do rd /s /q "%~a") >nul 2>&1 &' - else: - rm_cmd = 'rm -rf $out;' - - # Make two sets of rules to honor scons Precious setting. The build nodes themselves - # will then reselect their rule according to the precious being set for that node. - precious_rules = {} for rule, kwargs in self.rules.items(): - - # Do not worry about precious for commands that don't have targets (phony) - # or that will callback to scons (which maintains its own precious). - if rule not in ['phony', 'TEMPLATE', 'REGENERATE']: - precious_rule = rule + "_PRECIOUS" - precious_rules[precious_rule] = kwargs.copy() - ninja.rule(precious_rule, **precious_rules[precious_rule]) - - kwargs['command'] = f"{rm_cmd} " + kwargs['command'] - ninja.rule(rule, **kwargs) - else: - ninja.rule(rule, **kwargs) - self.rules.update(precious_rules) + ninja.rule(rule, **kwargs) generated_source_files = sorted({ output # First find builds which have header files in their outputs. for build in self.builds.values() - if self.has_generated_sources(build.get("outputs",[])) + if self.has_generated_sources(build["outputs"]) for output in build["outputs"] # Collect only the header files from the builds with them # in their output. We do this because is_generated_source @@ -685,38 +674,6 @@ class NinjaState: template_builders = [] - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - self.builds["compile_commands.json"] = { - 'rule' : "CMD", - 'outputs': ["compile_commands.json"], - 'pool' : "console", - 'implicit' : [ninja_file], - 'variables' : { - "cmd" : "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( - ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else '' - ) - }, - } - self.builds["compiledb"] = { - 'rule' : "phony", - "outputs" : ["compiledb"], - 'implicit' : ["compile_commands.json"], - } - - # Now for all build nodes, we want to select the precious rule or not. - # If it's not precious, we need to save all the outputs into a variable - # on that node. Later we will be removing outputs and switching them to - # phonies so that we can generate response and depfiles correctly. - for build, kwargs in self.builds.items(): - if kwargs.get('variables') and kwargs['variables'].get('precious'): - kwargs['rule'] = kwargs['rule'] + '_PRECIOUS' - elif kwargs['rule'] not in ['phony', 'TEMPLATE', 'REGENERATE']: - if not kwargs.get('variables'): - kwargs['variables'] = {} - kwargs['variables']['rm_outs'] = kwargs['outputs'].copy() - for build in [self.builds[key] for key in sorted(self.builds.keys())]: if build["rule"] == "TEMPLATE": template_builders.append(build) @@ -733,7 +690,7 @@ class NinjaState: if ( generated_source_files and not build["rule"] == "INSTALL" - and set(build.get("outputs", [])).isdisjoint(generated_source_files) + and set(build["outputs"]).isdisjoint(generated_source_files) and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): @@ -859,6 +816,24 @@ class NinjaState: implicit=[__file__], ) + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[ninja_file], + variables={ + "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( + ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) # Look in SCons's list of DEFAULT_TARGETS, find the ones that # we generated a ninja build rule for. @@ -1023,10 +998,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] rsp_content = " ".join(rsp_content) - variables = { - "rspc": rsp_content - } - + variables = {"rspc": rsp_content} variables[rule] = cmd if use_command_env: variables["env"] = get_command_env(env) @@ -1071,7 +1043,6 @@ def get_generic_shell_command(env, node, action, targets, sources, executor=None { "cmd": generate_command(env, node, action, targets, sources, executor=None), "env": get_command_env(env), - "precious": node.precious }, # Since this function is a rule mapping provider, it must return a list of dependencies, # and usually this would be the path to a tool, such as a compiler, used for this rule. @@ -1118,7 +1089,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) - variables['precious'] = node.precious + # Get the dependencies for all targets implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) @@ -1225,18 +1196,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule): __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule( - env, - rule, - command, - description="", - deps=None, - pool=None, - use_depfile=False, - use_response_file=False, - response_file_content="$rspc", - restat=False): - +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, @@ -1254,13 +1214,8 @@ def register_custom_rule( if use_response_file: rule_obj["rspfile"] = "$out.rsp" - if rule_obj["rspfile"] not in command: - raise Exception(f'Bad Ninja Custom Rule: response file requested, but {rule_obj["rspfile"]} not in in command: {command}') rule_obj["rspfile_content"] = response_file_content - if restat: - rule_obj["restat"] = 1 - env[NINJA_RULES][rule] = rule_obj @@ -1456,13 +1411,6 @@ def generate(env): env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") - # Expose ninja node path converstion functions to make writing - # custom function action handlers easier. - env.AddMethod(lambda env, node: get_outputs(node), "NinjaGetOutputs") - env.AddMethod(lambda env, node, skip_unknown_types=False: get_inputs(node, skip_unknown_types), "NinjaGetInputs") - env.AddMethod(lambda env, node, skip_sources=False: get_dependencies(node), "NinjaGetDependencies") - env.AddMethod(lambda env, node: get_order_only(node), "NinjaGetOrderOnly") - # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. env[NINJA_CUSTOM_HANDLERS] = {} |