Process options in includes files with the options of their junctionvalentindavid/include-options-from-junction

Unfortunately the options from main project cannot always be processed
in the include processing since project configuration might load
option declarations from a separate file. For that reason the result
of `Include.process` should still be passed through the option
processor. But all options files included from junctioned are already
evaluated.

1 files changed, 72 insertions, 6 deletions

diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py
index b49560947..b9a1c0d22 100644
--- a/src/buildstream/_includes.py
+++ b/src/buildstream/_includes.py
@@ -26,13 +26,51 @@ class Includes:
     #
     # Args:
     #    node (dict): A YAML node
+    #    only_local (bool): Whether to ignore junction files
+    #    process_project_options (bool): Whether to process options from current project
+    def process(self, node, *, only_local=False, process_project_options=True):
+        self._process(node, only_local=only_local, process_project_options=process_project_options)
+
+    # _process()
+    #
+    # Process recursively include directives in a YAML node. This
+    # method is a recursively called on loaded nodes from files.
+    #
+    # Args:
+    #    node (dict): A YAML node
     #    included (set): Fail for recursion if trying to load any files in this set
     #    current_loader (Loader): Use alternative loader (for junction files)
     #    only_local (bool): Whether to ignore junction files
-    def process(self, node, *, included=set(), current_loader=None, only_local=False):
+    #    process_project_options (bool): Whether to process options from current project
+    def _process(self, node, *, included=set(), current_loader=None, only_local=False, process_project_options=True):
         if current_loader is None:
             current_loader = self._loader
 
+        if process_project_options:
+            current_loader.project.options.process_node(node)
+
+        self._process_node(
+            node,
+            included=included,
+            only_local=only_local,
+            current_loader=current_loader,
+            process_project_options=process_project_options,
+        )
+
+    # _process_node()
+    #
+    # Process recursively include directives in a YAML node. This
+    # method is recursively called on all nodes.
+    #
+    # Args:
+    #    node (dict): A YAML node
+    #    included (set): Fail for recursion if trying to load any files in this set
+    #    current_loader (Loader): Use alternative loader (for junction files)
+    #    only_local (bool): Whether to ignore junction files
+    #    process_project_options (bool): Whether to process options from current project
+    def _process_node(
+        self, node, *, included=set(), current_loader=None, only_local=False, process_project_options=True
+    ):
         includes_node = node.get_node("(@)", allowed_types=[ScalarNode, SequenceNode], allow_none=True)
 
         if includes_node:
@@ -77,14 +115,26 @@ class Includes:
 
                 try:
                     included.add(file_path)
-                    self.process(include_node, included=included, current_loader=sub_loader, only_local=only_local)
+                    self._process(
+                        include_node,
+                        included=included,
+                        current_loader=sub_loader,
+                        only_local=only_local,
+                        process_project_options=process_project_options or current_loader != sub_loader,
+                    )
                 finally:
                     included.remove(file_path)
 
                 include_node._composite_under(node)
 
         for value in node.values():
-            self._process_value(value, included=included, current_loader=current_loader, only_local=only_local)
+            self._process_value(
+                value,
+                included=included,
+                current_loader=current_loader,
+                only_local=only_local,
+                process_project_options=process_project_options,
+            )
 
     # _include_file()
     #
@@ -100,6 +150,7 @@ class Includes:
             junction, include = include.split(":", 1)
             junction_loader = loader._get_loader(junction)
             current_loader = junction_loader
+            current_loader.project.ensure_fully_loaded()
         else:
             current_loader = loader
         project = current_loader.project
@@ -119,11 +170,26 @@ class Includes:
     #    included (set): Fail for recursion if trying to load any files in this set
     #    current_loader (Loader): Use alternative loader (for junction files)
     #    only_local (bool): Whether to ignore junction files
-    def _process_value(self, value, *, included=set(), current_loader=None, only_local=False):
+    #    process_project_options (bool): Whether to process options from current project
+    def _process_value(
+        self, value, *, included=set(), current_loader=None, only_local=False, process_project_options=True
+    ):
         value_type = type(value)
 
         if value_type is MappingNode:
-            self.process(value, included=included, current_loader=current_loader, only_local=only_local)
+            self._process_node(
+                value,
+                included=included,
+                current_loader=current_loader,
+                only_local=only_local,
+                process_project_options=process_project_options,
+            )
         elif value_type is SequenceNode:
             for v in value:
-                self._process_value(v, included=included, current_loader=current_loader, only_local=only_local)
+                self._process_value(
+                    v,
+                    included=included,
+                    current_loader=current_loader,
+                    only_local=only_local,
+                    process_project_options=process_project_options,
+                )