diff options
Diffstat (limited to 'site_scons')
-rw-r--r-- | site_scons/site_tools/distsrc.py | 4 | ||||
-rw-r--r-- | site_scons/site_tools/gziptool.py | 4 | ||||
-rw-r--r-- | site_scons/site_tools/ninja.py | 8 | ||||
-rw-r--r-- | site_scons/site_tools/ninja_next.py | 308 |
4 files changed, 157 insertions, 167 deletions
diff --git a/site_scons/site_tools/distsrc.py b/site_scons/site_tools/distsrc.py index 6fab5559521..0ce88f938be 100644 --- a/site_scons/site_tools/distsrc.py +++ b/site_scons/site_tools/distsrc.py @@ -198,8 +198,8 @@ def generate(env, **kwargs): generator=distsrc_action_generator, ) - def DistSrc(env, target): - result = env.__DISTSRC(target=target, source=[]) + def DistSrc(env, target, **kwargs): + result = env.__DISTSRC(target=target, source=[], **kwargs) env.AlwaysBuild(result) env.NoCache(result) return result diff --git a/site_scons/site_tools/gziptool.py b/site_scons/site_tools/gziptool.py index dce6062c993..7d7ee7cc418 100644 --- a/site_scons/site_tools/gziptool.py +++ b/site_scons/site_tools/gziptool.py @@ -32,8 +32,8 @@ def generate(env, **kwargs): "GZIPTOOL_COMSTR", "Compressing $TARGET with gzip" ) - def GZipTool(env, target, source): - result = env.__GZIPTOOL(target=target, source=source) + def GZipTool(env, target, source, **kwargs): + result = env.__GZIPTOOL(target=target, source=source, **kwargs) env.AlwaysBuild(result) return result diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py index 3c7e3f8c824..0f5b89c2a40 100644 --- a/site_scons/site_tools/ninja.py +++ b/site_scons/site_tools/ninja.py @@ -837,7 +837,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): def get_response_file_command(env, node, action, targets, sources, executor=None): if hasattr(action, "process"): cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd_list = [str(c) for c in cmd_list[0]] + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] else: command = generate_command( env, node, action, targets, sources, executor=executor @@ -862,11 +862,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] - - # Some commands may have a need to pass a literal $ to a tool - # like ld (think $ORIGIN). We need to protect that $ from - # Ninja by using $$. - rsp_content = " ".join(rsp_content).replace('$', '$$') + rsp_content = " ".join(rsp_content) variables = {"rspc": rsp_content} variables[rule] = cmd diff --git a/site_scons/site_tools/ninja_next.py b/site_scons/site_tools/ninja_next.py index a865cc6dee3..f4f20d16290 100644 --- a/site_scons/site_tools/ninja_next.py +++ b/site_scons/site_tools/ninja_next.py @@ -29,6 +29,7 @@ from SCons.Action import _string_from_cmd_list, get_default_ENV from SCons.Util import is_List, flatten_sequence from SCons.Script import COMMAND_LINE_TARGETS +NINJA_STATE = None NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" @@ -100,8 +101,21 @@ def alias_to_ninja_build(node): } -def get_dependencies(node): +def get_order_only(node): + """Return a list of order only dependencies for node.""" + if node.prerequisites is None: + return [] + return [get_path(src_file(prereq)) for prereq in node.prerequisites] + + +def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + if skip_sources: + return [ + get_path(src_file(child)) + for child in node.children() + if child not in node.sources + ] return [get_path(src_file(child)) for child in node.children()] @@ -149,15 +163,14 @@ class SConsToNinjaTranslator: "LibSymlinksActionFunction": _lib_symlink_action_function, } - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = False # pylint: disable=too-many-return-statements def action_to_ninja_build(self, node, action=None): """Generate build arguments dictionary for node.""" - # Use False since None is a valid value for this Attribute - build = getattr(node.attributes, NINJA_BUILD, False) - if build is not False: - return build + if not self.loaded_custom: + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = True if node.builder is None: return None @@ -165,39 +178,49 @@ class SConsToNinjaTranslator: if action is None: action = node.builder.action + if node.env and node.env.get("NINJA_SKIP"): + return None + + build = {} + # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one if node.builder == self.env["BUILDERS"]["Ninja"]: - return None - - if isinstance(action, SCons.Action.FunctionAction): - return self.handle_func_action(node, action) - - if isinstance(action, SCons.Action.LazyAction): + build = None + elif isinstance(action, SCons.Action.FunctionAction): + build = self.handle_func_action(node, action) + elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access action = action._generate_cache(node.env if node.env else self.env) - return self.action_to_ninja_build(node, action=action) - - if isinstance(action, SCons.Action.ListAction): - return self.handle_list_action(node, action) + build = self.action_to_ninja_build(node, action=action) + elif isinstance(action, SCons.Action.ListAction): + build = self.handle_list_action(node, action) + elif isinstance(action, COMMAND_TYPES): + build = get_command(node.env if node.env else self.env, node, action) + else: + raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - if isinstance(action, COMMAND_TYPES): - return get_command(node.env if node.env else self.env, node, action) + if build is not None: + build["order_only"] = get_order_only(node) - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) + return build def handle_func_action(self, node, action): """Determine how to handle the function action.""" name = action.function_name() # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required + # node to indicate that SCons is required. We skip sources here because + # dependencies don't really matter when we're going to shove these to + # the bottom of ninja's DAG anyway and Textfile builders can have text + # content as their source which doesn't work as an implicit dep in + # ninja. if name == "_action": return { "rule": "TEMPLATE", "outputs": get_outputs(node), - "implicit": get_dependencies(node), + "implicit": get_dependencies(node, skip_sources=True), } handler = self.func_handlers.get(name, None) @@ -454,48 +477,21 @@ class NinjaState: self.rules[rule]["deps"] = "gcc" self.rules[rule]["depfile"] = "$out.d" - self.rules.update(env.get(NINJA_RULES, {})) - self.pools.update(env.get(NINJA_POOLS, {})) + def add_build(self, node): + if not node.has_builder(): + return False - def generate_builds(self, node): - """Generate a ninja build rule for node and it's children.""" - # Filter out nodes with no builder. They are likely source files - # and so no work needs to be done, it will be used in the - # generation for some real target. - # - # Note that all nodes have a builder attribute but it is sometimes - # set to None. So we cannot use a simpler hasattr check here. - if getattr(node, "builder", None) is None: - return - - stack = [[node]] - while stack: - frame = stack.pop() - for child in frame: - outputs = set(get_outputs(child)) - # Check if all the outputs are in self.built, if they - # are we've already seen this node and it's children. - if not outputs.isdisjoint(self.built): - continue - - self.built = self.built.union(outputs) - stack.append(child.children()) + if isinstance(node, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(node) + else: + build = self.translator.action_to_ninja_build(node) - if isinstance(child, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(child) - elif node.builder is not None: - # Use False since None is a valid value for this attribute - build = getattr(child.attributes, NINJA_BUILD, False) - if build is False: - build = self.translator.action_to_ninja_build(child) - else: - build = None + # Some things are unbuild-able or need not be built in Ninja + if build is None: + return False - # Some things are unbuild-able or need not be built in Ninja - if build is None or build == 0: - continue - - self.builds.append(build) + self.builds.append(build) + return True def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" @@ -512,7 +508,7 @@ class NinjaState: return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file, fallback_default_target=None): + def generate(self, ninja_file): """ Generate the build.ninja. @@ -521,6 +517,9 @@ class NinjaState: if self.__generated: return + self.rules.update(self.env.get(NINJA_RULES, {})) + self.pools.update(self.env.get(NINJA_POOLS, {})) + content = io.StringIO() ninja = self.writer_class(content, width=100) @@ -585,7 +584,9 @@ class NinjaState: # generated source was rebuilt. We just need to make # sure that all of these sources are generated before # other builds. - build["order_only"] = "_generated_sources" + order_only = build.get("order_only", []) + order_only.append("_generated_sources") + build["order_only"] = order_only # When using a depfile Ninja can only have a single output # but SCons will usually have emitted an output for every @@ -668,13 +669,6 @@ class NinjaState: + glob("src/**/SConscript", recursive=True), ) - ninja.build( - "scons-invocation", - rule="CMD", - pool="console", - variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, - ) - # If we ever change the name/s of the rules that include # compile commands (i.e. something like CC) we will need to # update this build to reflect that complete list. @@ -707,11 +701,6 @@ class NinjaState: if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - # If not then set the default to the fallback_default_target we were given. - # Otherwise we won't create a default ninja target. - elif fallback_default_target is not None: - ninja.default(fallback_default_target) - with open(ninja_file, "w") as build_ninja: build_ninja.write(content.getvalue()) @@ -828,7 +817,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): def get_response_file_command(env, node, action, targets, sources, executor=None): if hasattr(action, "process"): cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd_list = [str(c) for c in cmd_list[0]] + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] else: command = generate_command( env, node, action, targets, sources, executor=executor @@ -853,11 +842,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] - - # Some commands may have a need to pass a literal $ to a tool - # like ld (think $ORIGIN). We need to protect that $ from - # Ninja by using $$. - rsp_content = " ".join(rsp_content).replace('$', '$$') + rsp_content = " ".join(rsp_content) variables = {"rspc": rsp_content} variables[rule] = cmd @@ -889,7 +874,8 @@ def generate_command(env, node, action, targets, sources, executor=None): if cmd.endswith("&&"): cmd = cmd[0:-2].strip() - return cmd + # Escape dollars as necessary + return cmd.replace("$", "$$") def get_shell_command(env, node, action, targets, sources, executor=None): @@ -941,6 +927,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) ninja_build = { + "order_only": get_order_only(node), "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, @@ -980,34 +967,8 @@ def ninja_builder(env, target, source): # here. print("Generating:", str(target[0])) - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) - ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) - - suffix = env.get("NINJA_SUFFIX", "") - if suffix and not suffix[0] == ".": - suffix = "." + suffix - - generated_build_ninja = target[0].get_abspath() + suffix - ninja_state = NinjaState(env, ninja_syntax.Writer) - - for src in source: - ninja_state.generate_builds(src) - - ninja_state.generate(generated_build_ninja, str(source[0])) + generated_build_ninja = target[0].get_abspath() + NINJA_STATE.generate(generated_build_ninja) return 0 @@ -1021,25 +982,6 @@ class AlwaysExecAction(SCons.Action.FunctionAction): return super().__call__(*args, **kwargs) -def ninja_print(_cmd, target, _source, env): - """Tag targets with the commands to build them.""" - if target: - for tgt in target: - if ( - tgt.has_builder() - # Use 'is False' because not would still trigger on - # None's which we don't want to regenerate - and getattr(tgt.attributes, NINJA_BUILD, False) is False - and isinstance(tgt.builder.action, COMMAND_TYPES) - ): - ninja_action = get_command(env, tgt, tgt.builder.action) - setattr(tgt.attributes, NINJA_BUILD, ninja_action) - # Preload the attributes dependencies while we're still running - # multithreaded - get_dependencies(tgt) - return 0 - - def register_custom_handler(env, name, handler): """Register a custom handler for SCons function actions.""" env[NINJA_CUSTOM_HANDLERS][name] = handler @@ -1202,6 +1144,15 @@ def generate(env): ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") + env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") + env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") + + ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1245,8 +1196,12 @@ def generate(env): from SCons.Tool.mslink import compositeLinkAction if env["LINKCOM"] == compositeLinkAction: - env["LINKCOM"] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env["SHLINKCOM"] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' # Normally in SCons actions for the Program and *Library builders # will return "${*COM}" as their pre-subst'd command line. However @@ -1315,6 +1270,7 @@ def generate(env): SCons.Node.FS.File.prepare = ninja_noop SCons.Node.FS.File.push_to_cache = ninja_noop SCons.Executor.Executor.prepare = ninja_noop + SCons.Taskmaster.Task.prepare = ninja_noop SCons.Node.FS.File.built = ninja_noop # We make lstat a no-op because it is only used for SONAME @@ -1347,20 +1303,8 @@ def generate(env): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) - # Replace false Compiling* messages with a more accurate output - # - # We also use this to tag all Nodes with Builders using - # CommandActions with the final command that was used to compile - # it for passing to Ninja. If we don't inject this behavior at - # this stage in the build too much state is lost to generate the - # command at the actual ninja_builder execution time for most - # commands. - # - # We do attempt command generation again in ninja_builder if it - # hasn't been tagged and it seems to work for anything that - # doesn't represent as a non-FunctionAction during the print_func - # call. - env["PRINT_CMD_LINE_FUNC"] = ninja_print + # Replace false action messages with nothing. + env["PRINT_CMD_LINE_FUNC"] = ninja_noop # This reduces unnecessary subst_list calls to add the compiler to # the implicit dependencies of targets. Since we encode full paths @@ -1369,11 +1313,6 @@ def generate(env): # where we expect it. env["IMPLICIT_COMMAND_DEPENDENCIES"] = False - # Set build to no_exec, our sublcass of FunctionAction will force - # an execution for ninja_builder so this simply effects all other - # Builders. - env.SetOption("no_exec", True) - # This makes SCons more aggressively cache MD5 signatures in the # SConsign file. env.SetOption("max_drift", 1) @@ -1383,6 +1322,69 @@ def generate(env): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) + ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) + + global NINJA_STATE + NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + + # Here we will force every builder to use an emitter which makes the ninja + # file depend on it's target. This forces the ninja file to the bottom of + # the DAG which is required so that we walk every target, and therefore add + # it to the global NINJA_STATE, before we try to write the ninja file. + def ninja_file_depends_on_all(target, source, env): + env.Depends(ninja_file, target) + return target, source + + for _, builder in env["BUILDERS"].items(): + emitter = builder.emitter + if emitter is not None: + builder.emitter = SCons.Builder.ListEmitter( + [emitter, ninja_file_depends_on_all] + ) + else: + builder.emitter = ninja_file_depends_on_all + + # Here we monkey patch the Task.execute method to not do a bunch of + # unnecessary work. If a build is a regular builder (i.e not a conftest and + # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we + # build it like normal. This skips all of the caching work that this method + # would normally do since we aren't pulling any of these targets from the + # cache. + # + # In the future we may be able to use this to actually cache the build.ninja + # file once we have the upstream support for referencing SConscripts as File + # nodes. + def ninja_execute(self): + global NINJA_STATE + + target = self.targets[0] + target_name = str(target) + if target_name != ninja_file_name and "conftest" not in target_name: + NINJA_STATE.add_build(target) + else: + target.build() + + SCons.Taskmaster.Task.execute = ninja_execute + + # Make needs_execute always return true instead of determining out of + # date-ness. + # SCons.Script.Main.BuildTask.needs_execute = lambda x: True + # We will eventually need to overwrite TempFileMunge to make it # handle persistent tempfiles or get an upstreamed change to add # some configurability to it's behavior in regards to tempfiles. @@ -1397,11 +1399,3 @@ def generate(env): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) env["TEMPFILE"] = NinjaNoResponseFiles - - # Force the SConsign to be written, we benefit from SCons caching of - # implicit dependencies and conftests. Unfortunately, we have to do this - # using an atexit handler because SCons will not write the file when in a - # no_exec build. - import atexit - - atexit.register(SCons.SConsign.write) |