Add support for include '(@)' in project.conf and .bst files

Fixes #331.

35 files changed, 562 insertions, 94 deletions

diff --git a/buildstream/_includes.py b/buildstream/_includes.py
new file mode 100644
index 000000000..718cd8283
--- /dev/null
+++ b/buildstream/_includes.py
@@ -0,0 +1,48 @@
+import os
+from collections import Mapping
+from . import _yaml
+
+
+class Includes:
+
+    def __init__(self, loader, valid_keys=None):
+        self._loader = loader
+        self._valid_keys = valid_keys
+
+    def process(self, node):
+        while True:
+            includes = _yaml.node_get(node, list, '(@)', default_value=None)
+            if '(@)' in node:
+                del node['(@)']
+
+            if not includes:
+                break
+
+            for include in includes:
+                include_node = self._include_file(include)
+                if self._valid_keys:
+                    _yaml.node_validate(include_node, self._valid_keys)
+
+                _yaml.composite(node, include_node)
+
+        for _, value in _yaml.node_items(node):
+            self._process_value(value)
+
+    def _include_file(self, include):
+        if ':' in include:
+            junction, include = include.split(':', 1)
+            junction_loader = self._loader._get_loader(junction, fetch_subprojects=True)
+            directory = junction_loader.project.directory
+        else:
+            directory = self._loader.project.directory
+        return _yaml.load(os.path.join(directory, include))
+
+    def _process_value(self, value):
+        if isinstance(value, Mapping):
+            self.process(value)
+        elif isinstance(value, list):
+            self._process_list(value)
+
+    def _process_list(self, values):
+        for value in values:
+            self._process_value(value)
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index a190f17dc..c5bce624b 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -23,12 +23,13 @@ from collections import Mapping, namedtuple
 import tempfile
 import shutil
 
-from .._exceptions import LoadError, LoadErrorReason
+from .._exceptions import LoadError, LoadErrorReason, PluginError
 from .. import Consistency
 from .. import _yaml
 from ..element import Element
 from .._profile import Topics, profile_start, profile_end
 from .._platform import Platform
+from .._includes import Includes
 
 from .types import Symbol, Dependency
 from .loadelement import LoadElement
@@ -69,6 +70,7 @@ class Loader():
         self._context = context
         self._options = project.options      # Project options (OptionPool)
         self._basedir = basedir              # Base project directory
+        self._first_pass_options = project.first_pass_config.options  # Project options (OptionPool)
         self._tempdir = tempdir              # A directory to cleanup
         self._parent = parent                # The parent loader
 
@@ -76,6 +78,8 @@ class Loader():
         self._elements = {}       # Dict of elements
         self._loaders = {}        # Dict of junction loaders
 
+        self._includes = Includes(self)
+
     # load():
     #
     # Loads the project based on the parameters given to the constructor
@@ -241,7 +245,22 @@ class Loader():
                                 message, detail=detail) from e
             else:
                 raise
-        self._options.process_node(node)
+        kind = _yaml.node_get(node, str, Symbol.KIND)
+        try:
+            kind_type, _ = self.project.first_pass_config.plugins.get_element_type(kind)
+        except PluginError:
+            kind_type = None
+        if kind_type and hasattr(kind_type, 'BST_NO_PROJECT_DEFAULTS') and kind_type.BST_NO_PROJECT_DEFAULTS:
+            self._first_pass_options.process_node(node)
+        else:
+            if not self.project.is_loaded():
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "{}: Cannot pre-load. Element depends on project defaults."
+                                .format(filename))
+
+            self._includes.process(node)
+
+            self._options.process_node(node)
 
         element = LoadElement(node, filename, self)
 
@@ -506,7 +525,8 @@ class Loader():
                             "{}: Expected junction but element kind is {}".format(filename, meta_element.kind))
 
         platform = Platform.get_platform()
-        element = Element._new_from_meta(meta_element, platform.artifactcache)
+        element = Element._new_from_meta(meta_element, platform.artifactcache,
+                                         first_pass=True)
         element._preflight()
 
         for source in element.sources():
diff --git a/buildstream/_loader/metasource.py b/buildstream/_loader/metasource.py
index 3bcc21ec6..4241ae50e 100644
--- a/buildstream/_loader/metasource.py
+++ b/buildstream/_loader/metasource.py
@@ -38,3 +38,4 @@ class MetaSource():
         self.kind = kind
         self.config = config
         self.directory = directory
+        self.first_pass = False
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
index f90fd820c..83a202f80 100644
--- a/buildstream/_options/optionpool.py
+++ b/buildstream/_options/optionpool.py
@@ -108,15 +108,17 @@ class OptionPool():
     # Args:
     #    cli_options (list): A list of (str, str) tuples
     #
-    def load_cli_values(self, cli_options):
+    def load_cli_values(self, cli_options, ignore_unknown=False):
         for option_name, option_value in cli_options:
             try:
                 option = self._options[option_name]
             except KeyError as e:
-                raise LoadError(LoadErrorReason.INVALID_DATA,
-                                "Unknown option '{}' specified on the command line"
-                                .format(option_name)) from e
-            option.set_value(option_value)
+                if not ignore_unknown:
+                    raise LoadError(LoadErrorReason.INVALID_DATA,
+                                    "Unknown option '{}' specified on the command line"
+                                    .format(option_name)) from e
+            else:
+                option.set_value(option_value)
 
     # resolve()
     #
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 0668adc75..b4aa1efe6 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -34,6 +34,7 @@ from ._sourcefactory import SourceFactory
 from ._projectrefs import ProjectRefs, ProjectRefStorage
 from ._versions import BST_FORMAT_VERSION
 from ._loader import Loader
+from ._includes import Includes
 
 
 # The separator we use for user specified aliases
@@ -199,12 +200,33 @@ class PluginCollection:
                             .format(plugin, plugin.BST_FORMAT_VERSION, version))
 
 
+class ProjectConfig:
+    def __init__(self):
+        self.plugins = None
+        self.options = None                      # OptionPool
+        self.base_variables = {}                 # The base set of variables
+        self.element_overrides = {}              # Element specific configurations
+        self.source_overrides = {}               # Source specific configurations
+
+
 # Project()
 #
 # The Project Configuration
 #
 class Project():
 
+    INCLUDE_CONFIG_KEYS = ['variables',
+                           'environment', 'environment-nocache',
+                           'split-rules', 'elements', 'plugins',
+                           'aliases', 'artifacts',
+                           'fail-on-overlap', 'shell',
+                           'ref-storage', 'sandbox',
+                           'options']
+
+    MAIN_FILE_CONFIG_KEYS = ['format-version',
+                             'element-path',
+                             'name']
+
     def __init__(self, directory, context, *, junction=None, cli_options=None,
                  parent_loader=None, tempdir=None):
 
@@ -221,16 +243,14 @@ class Project():
         self.refs = ProjectRefs(self.directory, 'project.refs')
         self.junction_refs = ProjectRefs(self.directory, 'junction.refs')
 
-        self.plugins = None                      # PluginCollection
-        self.options = None                      # OptionPool
+        self.config = ProjectConfig()
+        self.first_pass_config = ProjectConfig()
+
         self.junction = junction                 # The junction Element object, if this is a subproject
         self.fail_on_overlap = False             # Whether overlaps are treated as errors
         self.ref_storage = None                  # ProjectRefStorage setting
-        self.base_variables = {}                 # The base set of variables
         self.base_environment = {}               # The base set of environment variables
         self.base_env_nocache = None             # The base nocache mask (list) for the environment
-        self.element_overrides = {}              # Element specific configurations
-        self.source_overrides = {}               # Source specific configurations
 
         #
         # Private Members
@@ -245,15 +265,42 @@ class Project():
         self._shell_environment = {}  # Statically set environment vars
         self._shell_host_files = []   # A list of HostMount objects
 
+        self.artifact_cache_specs = None
+        self._sandbox = None
+        self._splits = None
+
+        self._context.add_project(self)
+
+        self._loaded = False
+
         profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
-        self._load()
+        self._load(parent_loader=parent_loader, tempdir=tempdir)
         profile_end(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
 
-        self._context.add_project(self)
+        self._loaded = True
 
-        self.loader = Loader(self._context, self,
-                             parent=parent_loader,
-                             tempdir=tempdir)
+    @property
+    def plugins(self):
+        return self.config.plugins
+
+    @property
+    def options(self):
+        return self.config.options
+
+    @property
+    def base_variables(self):
+        return self.config.base_variables
+
+    @property
+    def element_overrides(self):
+        return self.config.element_overrides
+
+    @property
+    def source_overrides(self):
+        return self.config.source_overrides
+
+    def is_loaded(self):
+        return self._loaded
 
     # translate_url():
     #
@@ -312,7 +359,7 @@ class Project():
     #
     # Raises: LoadError if there was a problem with the project.conf
     #
-    def _load(self):
+    def _load(self, parent_loader=None, tempdir=None):
 
         # Load builtin default
         projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE)
@@ -327,15 +374,6 @@ class Project():
 
         _yaml.composite(config, project_conf)
 
-        # Element and Source  type configurations will be composited later onto
-        # element/source types, so we delete it from here and run our final
-        # assertion after.
-        self.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
-        self.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
-        config.pop('elements', None)
-        config.pop('sources', None)
-        _yaml.node_final_assertions(config)
-
         # Assert project's format version early, before validating toplevel keys
         format_version = _yaml.node_get(config, int, 'format-version')
         if BST_FORMAT_VERSION < format_version:
@@ -345,17 +383,6 @@ class Project():
                 "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
                 .format(format_version, major, minor, BST_FORMAT_VERSION))
 
-        _yaml.node_validate(config, [
-            'format-version',
-            'element-path', 'variables',
-            'environment', 'environment-nocache',
-            'split-rules', 'elements', 'plugins',
-            'aliases', 'name',
-            'artifacts', 'options',
-            'fail-on-overlap', 'shell',
-            'ref-storage', 'sandbox'
-        ])
-
         # The project name, element path and option declarations
         # are constant and cannot be overridden by option conditional statements
         self.name = _yaml.node_get(config, str, 'name')
@@ -369,30 +396,21 @@ class Project():
             _yaml.node_get(config, str, 'element-path')
         )
 
-        # Load project options
-        options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
-        self.options = OptionPool(self.element_path)
-        self.options.load(options_node)
-        if self.junction:
-            # load before user configuration
-            self.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)
+        self.config.options = OptionPool(self.element_path)
+        self.first_pass_config.options = OptionPool(self.element_path)
 
-        # Collect option values specified in the user configuration
-        overrides = self._context.get_overrides(self.name)
-        override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
-        self.options.load_yaml_values(override_options)
-        if self._cli_options:
-            self.options.load_cli_values(self._cli_options)
+        self.loader = Loader(self._context, self,
+                             parent=parent_loader,
+                             tempdir=tempdir)
 
-        # We're done modifying options, now we can use them for substitutions
-        self.options.resolve()
+        self._load_pass(_yaml.node_copy(config), self.first_pass_config, True)
 
-        #
-        # Now resolve any conditionals in the remaining configuration,
-        # any conditionals specified for project option declarations,
-        # or conditionally specifying the project name; will be ignored.
-        #
-        self.options.process_node(config)
+        project_includes = Includes(self.loader, self.INCLUDE_CONFIG_KEYS + ['elements', 'sources'])
+        project_includes.process(config)
+
+        self._load_pass(config, self.config, False)
+
+        _yaml.node_validate(config, self.INCLUDE_CONFIG_KEYS + self.MAIN_FILE_CONFIG_KEYS)
 
         #
         # Now all YAML composition is done, from here on we just load
@@ -402,23 +420,9 @@ class Project():
         # Load artifacts pull/push configuration for this project
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config)
 
-        self.plugins = PluginCollection(self, self._context, self.directory, config)
         # Source url aliases
         self._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
 
-        # Load base variables
-        self.base_variables = _yaml.node_get(config, Mapping, 'variables')
-
-        # Add the project name as a default variable
-        self.base_variables['project-name'] = self.name
-
-        # Extend variables with automatic variables and option exports
-        # Initialize it as a string as all variables are processed as strings.
-        self.base_variables['max-jobs'] = str(multiprocessing.cpu_count())
-
-        # Export options into variables, if that was requested
-        self.options.export_variables(self.base_variables)
-
         # Load sandbox environment variables
         self.base_environment = _yaml.node_get(config, Mapping, 'environment')
         self.base_env_nocache = _yaml.node_get(config, list, 'environment-nocache')
@@ -475,6 +479,56 @@ class Project():
 
             self._shell_host_files.append(mount)
 
+    def _load_pass(self, config, output, ignore_unknown):
+
+        # Element and Source  type configurations will be composited later onto
+        # element/source types, so we delete it from here and run our final
+        # assertion after.
+        output.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
+        output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
+        config.pop('elements', None)
+        config.pop('sources', None)
+        _yaml.node_final_assertions(config)
+
+        output.plugins = PluginCollection(self, self._context, self.directory, config)
+
+        # Load project options
+        options_node = _yaml.node_get(config, Mapping, 'options', default_value={})
+        output.options.load(options_node)
+        if self.junction:
+            # load before user configuration
+            output.options.load_yaml_values(self.junction.options, transform=self.junction._subst_string)
+
+        # Collect option values specified in the user configuration
+        overrides = self._context.get_overrides(self.name)
+        override_options = _yaml.node_get(overrides, Mapping, 'options', default_value={})
+        output.options.load_yaml_values(override_options)
+        if self._cli_options:
+            output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown)
+
+        # We're done modifying options, now we can use them for substitutions
+        output.options.resolve()
+
+        #
+        # Now resolve any conditionals in the remaining configuration,
+        # any conditionals specified for project option declarations,
+        # or conditionally specifying the project name; will be ignored.
+        #
+        output.options.process_node(config)
+
+        # Load base variables
+        output.base_variables = _yaml.node_get(config, Mapping, 'variables')
+
+        # Add the project name as a default variable
+        output.base_variables['project-name'] = self.name
+
+        # Extend variables with automatic variables and option exports
+        # Initialize it as a string as all variables are processed as strings.
+        output.base_variables['max-jobs'] = str(multiprocessing.cpu_count())
+
+        # Export options into variables, if that was requested
+        output.options.export_variables(output.base_variables)
+
     # _ensure_project_dir()
     #
     # Returns path of the project directory, if a configuration file is found
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index 48d3571d6..4801ecc10 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -75,6 +75,7 @@ class Stream():
         self._artifacts = self._platform.artifactcache
         self._context = context
         self._project = project
+
         self._pipeline = Pipeline(context, project, self._artifacts)
         self._scheduler = Scheduler(context, session_start,
                                     interrupt_callback=interrupt_callback,
diff --git a/buildstream/element.py b/buildstream/element.py
index f8a993a0c..ee523ff76 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -191,10 +191,19 @@ class Element(Plugin):
     *Since: 1.2*
     """
 
+    BST_NO_PROJECT_DEFAULTS = False
+    """
+
+    """
+
     def __init__(self, context, project, artifacts, meta, plugin_conf):
 
         super().__init__(meta.name, context, project, meta.provenance, "element")
 
+        if not project.is_loaded() and not self.BST_NO_PROJECT_DEFAULTS:
+            raise ElementError("{}: Cannot load element before project"
+                               .format(self), reason="project-not-loaded")
+
         self.normal_name = os.path.splitext(self.name.replace(os.sep, '-'))[0]
         """A normalized element name
 
@@ -885,9 +894,12 @@ class Element(Plugin):
     #    (Element): A newly created Element instance
     #
     @classmethod
-    def _new_from_meta(cls, meta, artifacts):
+    def _new_from_meta(cls, meta, artifacts, first_pass=False):
 
-        plugins = meta.project.plugins
+        if first_pass:
+            plugins = meta.project.first_pass_config.plugins
+        else:
+            plugins = meta.project.plugins
 
         if meta in cls.__instantiated_elements:
             return cls.__instantiated_elements[meta]
@@ -897,6 +909,7 @@ class Element(Plugin):
 
         # Instantiate sources
         for meta_source in meta.sources:
+            meta_source.first_pass = element.BST_NO_PROJECT_DEFAULTS
             source = plugins.create_source(meta_source)
             redundant_ref = source._load_ref()
             element.__sources.append(source)
@@ -907,10 +920,10 @@ class Element(Plugin):
 
         # Instantiate dependencies
         for meta_dep in meta.dependencies:
-            dependency = Element._new_from_meta(meta_dep, artifacts)
+            dependency = Element._new_from_meta(meta_dep, artifacts, first_pass=first_pass)
             element.__runtime_dependencies.append(dependency)
         for meta_dep in meta.build_dependencies:
-            dependency = Element._new_from_meta(meta_dep, artifacts)
+            dependency = Element._new_from_meta(meta_dep, artifacts, first_pass=first_pass)
             element.__build_dependencies.append(dependency)
 
         return element
@@ -2095,16 +2108,24 @@ class Element(Plugin):
 
     def __compose_default_splits(self, defaults):
         project = self._get_project()
-        project_splits = _yaml.node_chain_copy(project._splits)
 
         element_public = _yaml.node_get(defaults, Mapping, 'public', default_value={})
         element_bst = _yaml.node_get(element_public, Mapping, 'bst', default_value={})
         element_splits = _yaml.node_get(element_bst, Mapping, 'split-rules', default_value={})
 
-        # Extend project wide split rules with any split rules defined by the element
-        _yaml.composite(project_splits, element_splits)
+        if self.BST_NO_PROJECT_DEFAULTS:
+            splits = _yaml.node_chain_copy(element_splits)
+        elif project._splits is None:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: Project was not fully loaded while loading element. "
+                            "Only non-artifact elements (e.g. junctions) are allowed in this context."
+                            .format(self.name))
+        else:
+            splits = _yaml.node_chain_copy(project._splits)
+            # Extend project wide split rules with any split rules defined by the element
+            _yaml.composite(splits, element_splits)
 
-        element_bst['split-rules'] = project_splits
+        element_bst['split-rules'] = splits
         element_public['bst'] = element_bst
         defaults['public'] = element_public
 
@@ -2128,7 +2149,11 @@ class Element(Plugin):
             # Override the element's defaults with element specific
             # overrides from the project.conf
             project = self._get_project()
-            elements = project.element_overrides
+            if self.BST_NO_PROJECT_DEFAULTS:
+                elements = project.first_pass_config.element_overrides
+            else:
+                elements = project.element_overrides
+
             overrides = elements.get(self.get_kind())
             if overrides:
                 _yaml.composite(defaults, overrides)
@@ -2141,10 +2166,14 @@ class Element(Plugin):
     # creating sandboxes for this element
     #
     def __extract_environment(self, meta):
-        project = self._get_project()
         default_env = _yaml.node_get(self.__defaults, Mapping, 'environment', default_value={})
 
-        environment = _yaml.node_chain_copy(project.base_environment)
+        if self.BST_NO_PROJECT_DEFAULTS:
+            environment = {}
+        else:
+            project = self._get_project()
+            environment = _yaml.node_chain_copy(project.base_environment)
+
         _yaml.composite(environment, default_env)
         _yaml.composite(environment, meta.environment)
         _yaml.node_final_assertions(environment)
@@ -2157,8 +2186,13 @@ class Element(Plugin):
         return final_env
 
     def __extract_env_nocache(self, meta):
-        project = self._get_project()
-        project_nocache = project.base_env_nocache
+        if self.BST_NO_PROJECT_DEFAULTS:
+            project_nocache = []
+        else:
+            project = self._get_project()
+            assert project.is_loaded()
+            project_nocache = project.base_env_nocache
+
         default_nocache = _yaml.node_get(self.__defaults, list, 'environment-nocache', default_value=[])
         element_nocache = meta.env_nocache
 
@@ -2173,10 +2207,15 @@ class Element(Plugin):
     # substituting command strings to be run in the sandbox
     #
     def __extract_variables(self, meta):
-        project = self._get_project()
         default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables', default_value={})
 
-        variables = _yaml.node_chain_copy(project.base_variables)
+        project = self._get_project()
+        if self.BST_NO_PROJECT_DEFAULTS:
+            variables = _yaml.node_chain_copy(project.first_pass_config.base_variables)
+        else:
+            assert project.is_loaded()
+            variables = _yaml.node_chain_copy(project.base_variables)
+
         _yaml.composite(variables, default_vars)
         _yaml.composite(variables, meta.variables)
         _yaml.node_final_assertions(variables)
@@ -2200,13 +2239,18 @@ class Element(Plugin):
     # Sandbox-specific configuration data, to be passed to the sandbox's constructor.
     #
     def __extract_sandbox_config(self, meta):
-        project = self._get_project()
+        if self.BST_NO_PROJECT_DEFAULTS:
+            sandbox_config = {'build-uid': 0,
+                              'build-gid': 0}
+        else:
+            project = self._get_project()
+            assert project.is_loaded()
+            sandbox_config = _yaml.node_chain_copy(project._sandbox)
 
         # The default config is already composited with the project overrides
         sandbox_defaults = _yaml.node_get(self.__defaults, Mapping, 'sandbox', default_value={})
         sandbox_defaults = _yaml.node_chain_copy(sandbox_defaults)
 
-        sandbox_config = _yaml.node_chain_copy(project._sandbox)
         _yaml.composite(sandbox_config, sandbox_defaults)
         _yaml.composite(sandbox_config, meta.sandbox)
         _yaml.node_final_assertions(sandbox_config)
diff --git a/buildstream/plugins/elements/junction.py b/buildstream/plugins/elements/junction.py
index ee5ed24d5..2f81f4631 100644
--- a/buildstream/plugins/elements/junction.py
+++ b/buildstream/plugins/elements/junction.py
@@ -136,6 +136,7 @@ class JunctionElement(Element):
     # Junctions are not allowed any dependencies
     BST_FORBID_BDEPENDS = True
     BST_FORBID_RDEPENDS = True
+    BST_NO_PROJECT_DEFAULTS = True
 
     def configure(self, node):
         self.path = self.node_get_member(node, str, 'path', default='')
diff --git a/buildstream/source.py b/buildstream/source.py
index ec38ae8f2..c01993429 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -137,8 +137,9 @@ class Source(Plugin):
 
         # Collect the composited element configuration and
         # ask the element to configure itself.
-        self.__init_defaults()
+        self.__init_defaults(meta)
         self.__config = self.__extract_config(meta)
+
         self.configure(self.__config)
 
     COMMON_CONFIG_KEYS = ['kind', 'directory']
@@ -611,10 +612,13 @@ class Source(Plugin):
                               reason="ensure-stage-dir-fail") from e
         return directory
 
-    def __init_defaults(self):
+    def __init_defaults(self, meta):
         if not self.__defaults_set:
             project = self._get_project()
-            sources = project.source_overrides
+            if meta.first_pass:
+                sources = project.first_pass_config.source_overrides
+            else:
+                sources = project.source_overrides
             type(self).__defaults = sources.get(self.get_kind(), {})
             type(self).__defaults_set = True
 
diff --git a/tests/format/include.py b/tests/format/include.py
new file mode 100644
index 000000000..ca6eaab08
--- /dev/null
+++ b/tests/format/include.py
@@ -0,0 +1,200 @@
+import os
+import pytest
+from buildstream import _yaml
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from tests.testutils import cli, generate_junction, create_repo
+
+
+# Project directory
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'include'
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_project_file(cli, datafiles):
+    project = os.path.join(str(datafiles), 'file')
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_junction_file(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'junction')
+
+    generate_junction(tmpdir,
+                      os.path.join(project, 'subproject'),
+                      os.path.join(project, 'junction.bst'),
+                      store_ref=True)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_junction_options(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'options')
+
+    result = cli.run(project=project, args=[
+        '-o', 'build_arch', 'x86_64',
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['build_arch'] == 'x86_64'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_project_defines_name(cli, datafiles):
+    project = os.path.join(str(datafiles), 'defines_name')
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_element_partial_project_project(cli, tmpdir, datafiles):
+    """
+    Junction elements never depend on fully include processed project.
+    """
+
+    project = os.path.join(str(datafiles), 'junction')
+
+    subproject_path = os.path.join(project, 'subproject')
+    junction_path = os.path.join(project, 'junction.bst')
+
+    repo = create_repo('git', str(tmpdir))
+
+    ref = repo.create(subproject_path)
+
+    element = {
+        'kind': 'junction',
+        'sources': [
+            repo.source_config(ref=ref)
+        ]
+    }
+    _yaml.dump(element, junction_path)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'junction.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'included' not in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_element_partial_project_file(cli, tmpdir, datafiles):
+    """
+    Junction elements never depend on fully include processed project.
+    """
+
+    project = os.path.join(str(datafiles), 'file_with_subproject')
+
+    subproject_path = os.path.join(project, 'subproject')
+    junction_path = os.path.join(project, 'junction.bst')
+
+    repo = create_repo('git', str(tmpdir))
+
+    ref = repo.create(subproject_path)
+
+    element = {
+        'kind': 'junction',
+        'sources': [
+            repo.source_config(ref=ref)
+        ]
+    }
+    _yaml.dump(element, junction_path)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'junction.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'included' not in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'overrides')
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'manual_main_override' in loaded
+    assert 'manual_included_override' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides_composition(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'overrides')
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{config}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'build-commands' in loaded
+    assert loaded['build-commands'] == ['first', 'second']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides_sub_include(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'sub-include')
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'included' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'overrides')
+
+    generate_junction(tmpdir,
+                      os.path.join(project, 'subproject'),
+                      os.path.join(project, 'junction.bst'),
+                      store_ref=True)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'junction.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'main_override' in loaded
+    assert 'included_override' not in loaded
diff --git a/tests/format/include/defines_name/element.bst b/tests/format/include/defines_name/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/defines_name/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/defines_name/extra_conf.yml b/tests/format/include/defines_name/extra_conf.yml
new file mode 100644
index 000000000..84e8c6a65
--- /dev/null
+++ b/tests/format/include/defines_name/extra_conf.yml
@@ -0,0 +1 @@
+name: othername
diff --git a/tests/format/include/defines_name/project.conf b/tests/format/include/defines_name/project.conf
new file mode 100644
index 000000000..a7791a416
--- /dev/null
+++ b/tests/format/include/defines_name/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/file/element.bst b/tests/format/include/file/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/file/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/file/extra_conf.yml b/tests/format/include/file/extra_conf.yml
new file mode 100644
index 000000000..404ecd6dd
--- /dev/null
+++ b/tests/format/include/file/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: 'True'
diff --git a/tests/format/include/file/project.conf b/tests/format/include/file/project.conf
new file mode 100644
index 000000000..a7791a416
--- /dev/null
+++ b/tests/format/include/file/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/element.bst b/tests/format/include/file_with_subproject/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/file_with_subproject/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/file_with_subproject/extra_conf.yml b/tests/format/include/file_with_subproject/extra_conf.yml
new file mode 100644
index 000000000..404ecd6dd
--- /dev/null
+++ b/tests/format/include/file_with_subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: 'True'
diff --git a/tests/format/include/file_with_subproject/project.bst b/tests/format/include/file_with_subproject/project.bst
new file mode 100644
index 000000000..4836c5f8b
--- /dev/null
+++ b/tests/format/include/file_with_subproject/project.bst
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - junction.bst:extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/project.conf b/tests/format/include/file_with_subproject/project.conf
new file mode 100644
index 000000000..a7791a416
--- /dev/null
+++ b/tests/format/include/file_with_subproject/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/subproject/project.conf b/tests/format/include/file_with_subproject/subproject/project.conf
new file mode 100644
index 000000000..7a6655421
--- /dev/null
+++ b/tests/format/include/file_with_subproject/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/junction/element.bst b/tests/format/include/junction/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/junction/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/junction/project.conf b/tests/format/include/junction/project.conf
new file mode 100644
index 000000000..4836c5f8b
--- /dev/null
+++ b/tests/format/include/junction/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - junction.bst:extra_conf.yml
diff --git a/tests/format/include/junction/subproject/extra_conf.yml b/tests/format/include/junction/subproject/extra_conf.yml
new file mode 100644
index 000000000..404ecd6dd
--- /dev/null
+++ b/tests/format/include/junction/subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: 'True'
diff --git a/tests/format/include/junction/subproject/project.conf b/tests/format/include/junction/subproject/project.conf
new file mode 100644
index 000000000..7a6655421
--- /dev/null
+++ b/tests/format/include/junction/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/options/element.bst b/tests/format/include/options/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/options/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/options/extra_conf.yml b/tests/format/include/options/extra_conf.yml
new file mode 100644
index 000000000..ad1401e0a
--- /dev/null
+++ b/tests/format/include/options/extra_conf.yml
@@ -0,0 +1,8 @@
+options:
+  build_arch:
+    type: arch
+    description: Architecture
+    variable: build_arch
+    values:
+      - i586
+      - x86_64
diff --git a/tests/format/include/options/project.conf b/tests/format/include/options/project.conf
new file mode 100644
index 000000000..a7791a416
--- /dev/null
+++ b/tests/format/include/options/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/overrides/element.bst b/tests/format/include/overrides/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/overrides/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/overrides/extra_conf.yml b/tests/format/include/overrides/extra_conf.yml
new file mode 100644
index 000000000..3cd3530c5
--- /dev/null
+++ b/tests/format/include/overrides/extra_conf.yml
@@ -0,0 +1,16 @@
+elements:
+  junction:
+    variables:
+      included_override: True
+  manual:
+    variables:
+      manual_included_override: True
+    config:
+      build-commands:
+        (>):
+          - "second"
+
+sources:
+  git:
+    variables:
+      from_included: True
diff --git a/tests/format/include/overrides/project.conf b/tests/format/include/overrides/project.conf
new file mode 100644
index 000000000..9285b9d59
--- /dev/null
+++ b/tests/format/include/overrides/project.conf
@@ -0,0 +1,20 @@
+name: test
+
+elements:
+  junction:
+    variables:
+      main_override: True
+  manual:
+    variables:
+      manual_main_override: True
+    config:
+      build-commands:
+        - "first"
+
+sources:
+  git:
+    variables:
+      from_main: True
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/overrides/subproject/project.conf b/tests/format/include/overrides/subproject/project.conf
new file mode 100644
index 000000000..7a6655421
--- /dev/null
+++ b/tests/format/include/overrides/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/sub-include/element.bst b/tests/format/include/sub-include/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/sub-include/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/sub-include/manual_conf.yml b/tests/format/include/sub-include/manual_conf.yml
new file mode 100644
index 000000000..9c2c0dd34
--- /dev/null
+++ b/tests/format/include/sub-include/manual_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: True
diff --git a/tests/format/include/sub-include/project.conf b/tests/format/include/sub-include/project.conf
new file mode 100644
index 000000000..7f7df84c8
--- /dev/null
+++ b/tests/format/include/sub-include/project.conf
@@ -0,0 +1,6 @@
+name: test
+
+elements:
+  manual:
+    (@):
+      - manual_conf.yml